From ad867e8e3c74f6bc90acc61facb3e139fa5779e4 Mon Sep 17 00:00:00 2001 From: Igor Katson Date: Sat, 3 Jul 2021 19:10:59 +0100 Subject: [PATCH] Move everything to workspaces --- Cargo.lock | 69 +++++++++++++++++++ Cargo.toml | 16 ++++- crates/bencode/Cargo.toml | 13 ++++ .../src/bencode_value.rs | 8 +-- crates/bencode/src/lib.rs | 7 ++ .../src/serde_bencode_de.rs | 11 +-- .../src/serde_bencode_ser.rs | 2 +- crates/buffers/Cargo.toml | 10 +++ .../src/buffers.rs => buffers/src/lib.rs} | 12 +++- crates/clone_to_owned/Cargo.toml | 8 +++ .../src/lib.rs} | 0 crates/librqbit/Cargo.toml | 16 ++--- crates/librqbit/src/chunk_tracker.rs | 14 ++-- crates/librqbit/src/dht/inforead.rs | 10 ++- crates/librqbit/src/file_ops.rs | 16 ++--- crates/librqbit/src/http_api.rs | 2 +- crates/librqbit/src/lib.rs | 21 ++---- crates/librqbit/src/peer_connection.rs | 20 +++--- crates/librqbit/src/peer_info_reader/mod.rs | 29 ++++---- crates/librqbit/src/peer_state.rs | 6 +- crates/librqbit/src/torrent_manager.rs | 17 ++--- crates/librqbit/src/torrent_state.rs | 27 +++++--- crates/librqbit/src/tracker_comms.rs | 3 +- crates/librqbit/src/type_aliases.rs | 9 --- crates/librqbit_core/Cargo.toml | 18 +++++ .../src/constants.rs | 0 .../src/info_hash.rs | 0 .../src/lengths.rs | 19 ++--- crates/librqbit_core/src/lib.rs | 7 ++ .../{librqbit => librqbit_core}/src/magnet.rs | 0 .../src/peer_id.rs | 0 .../src/speed_estimator.rs | 0 .../src/torrent_metainfo.rs | 9 +-- crates/peer_binary_protocol/Cargo.toml | 17 +++++ .../src}/extended/handshake.rs | 6 +- .../src}/extended/mod.rs | 13 ++-- .../src}/extended/ut_metadata.rs | 9 +-- .../src/lib.rs} | 10 +-- crates/rqbit/Cargo.toml | 8 +-- crates/rqbit/src/main.rs | 13 ++-- crates/sha1w/Cargo.toml | 22 ++++++ .../src/sha1w.rs => sha1w/src/lib.rs} | 9 +++ 42 files changed, 338 insertions(+), 168 deletions(-) create mode 100644 crates/bencode/Cargo.toml rename crates/{librqbit => bencode}/src/bencode_value.rs (98%) create mode 100644 crates/bencode/src/lib.rs rename crates/{librqbit => bencode}/src/serde_bencode_de.rs (98%) rename crates/{librqbit => bencode}/src/serde_bencode_ser.rs (99%) create mode 100644 crates/buffers/Cargo.toml rename crates/{librqbit/src/buffers.rs => buffers/src/lib.rs} (94%) create mode 100644 crates/clone_to_owned/Cargo.toml rename crates/{librqbit/src/clone_to_owned.rs => clone_to_owned/src/lib.rs} (100%) create mode 100644 crates/librqbit_core/Cargo.toml rename crates/{librqbit => librqbit_core}/src/constants.rs (100%) rename crates/{librqbit => librqbit_core}/src/info_hash.rs (100%) rename crates/{librqbit => librqbit_core}/src/lengths.rs (95%) create mode 100644 crates/librqbit_core/src/lib.rs rename crates/{librqbit => librqbit_core}/src/magnet.rs (100%) rename crates/{librqbit => librqbit_core}/src/peer_id.rs (100%) rename crates/{librqbit => librqbit_core}/src/speed_estimator.rs (100%) rename crates/{librqbit => librqbit_core}/src/torrent_metainfo.rs (98%) create mode 100644 crates/peer_binary_protocol/Cargo.toml rename crates/{librqbit/src/peer_binary_protocol => peer_binary_protocol/src}/extended/handshake.rs (97%) rename crates/{librqbit/src/peer_binary_protocol => peer_binary_protocol/src}/extended/mod.rs (90%) rename crates/{librqbit/src/peer_binary_protocol => peer_binary_protocol/src}/extended/ut_metadata.rs (95%) rename crates/{librqbit/src/peer_binary_protocol/mod.rs => peer_binary_protocol/src/lib.rs} (99%) create mode 100644 crates/sha1w/Cargo.toml rename crates/{librqbit/src/sha1w.rs => sha1w/src/lib.rs} (89%) diff --git a/Cargo.lock b/Cargo.lock index 881093d..3523cac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,6 +40,17 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bencode" +version = "0.1.0" +dependencies = [ + "anyhow", + "buffers", + "clone_to_owned", + "serde", + "sha1w", +] + [[package]] name = "bincode" version = "1.3.3" @@ -86,6 +97,14 @@ dependencies = [ "safemem", ] +[[package]] +name = "buffers" +version = "0.1.0" +dependencies = [ + "clone_to_owned", + "serde", +] + [[package]] name = "bumpalo" version = "3.7.0" @@ -148,6 +167,10 @@ dependencies = [ "syn", ] +[[package]] +name = "clone_to_owned" +version = "0.1.0" + [[package]] name = "commoncrypto" version = "0.2.0" @@ -643,20 +666,26 @@ name = "librqbit" version = "0.1.0" dependencies = [ "anyhow", + "bencode", "bincode", "bitvec", + "buffers", "byteorder", + "clone_to_owned", "crypto-hash", "futures", "hex 0.4.3", + "librqbit_core", "log", "openssl", "parking_lot", + "peer_binary_protocol", "pretty_env_logger", "rand 0.8.4", "reqwest", "serde", "sha1", + "sha1w", "size_format", "tokio", "url", @@ -665,6 +694,22 @@ dependencies = [ "warp", ] +[[package]] +name = "librqbit_core" +version = "0.1.0" +dependencies = [ + "anyhow", + "bencode", + "buffers", + "clone_to_owned", + "hex 0.4.3", + "log", + "parking_lot", + "serde", + "url", + "uuid", +] + [[package]] name = "lock_api" version = "0.4.4" @@ -928,6 +973,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "peer_binary_protocol" +version = "0.1.0" +dependencies = [ + "anyhow", + "bencode", + "bincode", + "bitvec", + "buffers", + "byteorder", + "clone_to_owned", + "librqbit_core", + "serde", +] + [[package]] name = "percent-encoding" version = "2.1.0" @@ -1338,6 +1398,15 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" +[[package]] +name = "sha1w" +version = "0.1.0" +dependencies = [ + "crypto-hash", + "openssl", + "sha1", +] + [[package]] name = "size_format" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 8a82d20..bdba329 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,17 @@ [workspace] members = [ "crates/librqbit", - "crates/rqbit" -] \ No newline at end of file + "crates/rqbit", + "crates/buffers", + "crates/clone_to_owned", + "crates/bencode", + "crates/sha1w", + "crates/librqbit_core", + "crates/peer_binary_protocol" +] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" \ No newline at end of file diff --git a/crates/bencode/Cargo.toml b/crates/bencode/Cargo.toml new file mode 100644 index 0000000..2f8504f --- /dev/null +++ b/crates/bencode/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "bencode" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = {version = "1", features=["derive"]} +buffers = {path = "../buffers"} +clone_to_owned = {path = "../clone_to_owned"} +anyhow = "1" +sha1w = {path="../sha1w"} \ No newline at end of file diff --git a/crates/librqbit/src/bencode_value.rs b/crates/bencode/src/bencode_value.rs similarity index 98% rename from crates/librqbit/src/bencode_value.rs rename to crates/bencode/src/bencode_value.rs index f56042e..9904d3e 100644 --- a/crates/librqbit/src/bencode_value.rs +++ b/crates/bencode/src/bencode_value.rs @@ -1,12 +1,10 @@ use std::{collections::HashMap, marker::PhantomData}; +use buffers::{ByteBuf, ByteString}; +use clone_to_owned::CloneToOwned; use serde::{Deserialize, Deserializer}; -use crate::{ - buffers::{ByteBuf, ByteString}, - clone_to_owned::CloneToOwned, - serde_bencode_de::from_bytes, -}; +use crate::serde_bencode_de::from_bytes; pub fn dyn_from_bytes<'de, ByteBuf>(buf: &'de [u8]) -> anyhow::Result> where diff --git a/crates/bencode/src/lib.rs b/crates/bencode/src/lib.rs new file mode 100644 index 0000000..431bed5 --- /dev/null +++ b/crates/bencode/src/lib.rs @@ -0,0 +1,7 @@ +mod bencode_value; +mod serde_bencode_de; +mod serde_bencode_ser; + +pub use bencode_value::*; +pub use serde_bencode_de::*; +pub use serde_bencode_ser::*; diff --git a/crates/librqbit/src/serde_bencode_de.rs b/crates/bencode/src/serde_bencode_de.rs similarity index 98% rename from crates/librqbit/src/serde_bencode_de.rs rename to crates/bencode/src/serde_bencode_de.rs index 18c15c0..8a1be64 100644 --- a/crates/librqbit/src/serde_bencode_de.rs +++ b/crates/bencode/src/serde_bencode_de.rs @@ -1,14 +1,15 @@ -use crate::buffers::ByteBuf; -use crate::sha1w::ISha1; -use crate::type_aliases::Sha1; +use buffers::ByteBuf; use serde::de::Error as DeError; +use sha1w::{ISha1, Sha1}; pub struct BencodeDeserializer<'de> { buf: &'de [u8], field_context: Vec>, parsing_key: bool, - pub(crate) is_torrent_info: bool, - pub(crate) torrent_info_digest: Option<[u8; 20]>, + + // This is a f**ing hack + pub is_torrent_info: bool, + pub torrent_info_digest: Option<[u8; 20]>, } impl<'de> BencodeDeserializer<'de> { diff --git a/crates/librqbit/src/serde_bencode_ser.rs b/crates/bencode/src/serde_bencode_ser.rs similarity index 99% rename from crates/librqbit/src/serde_bencode_ser.rs rename to crates/bencode/src/serde_bencode_ser.rs index 036d7c9..60e9b19 100644 --- a/crates/librqbit/src/serde_bencode_ser.rs +++ b/crates/bencode/src/serde_bencode_ser.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use serde::{Serialize, Serializer}; -use crate::buffers::ByteString; +use buffers::ByteString; #[derive(Debug)] pub enum SerErrorKind { diff --git a/crates/buffers/Cargo.toml b/crates/buffers/Cargo.toml new file mode 100644 index 0000000..d4f332b --- /dev/null +++ b/crates/buffers/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "buffers" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = {version = "1", features=["derive"]} +clone_to_owned = {path="../clone_to_owned"} \ No newline at end of file diff --git a/crates/librqbit/src/buffers.rs b/crates/buffers/src/lib.rs similarity index 94% rename from crates/librqbit/src/buffers.rs rename to crates/buffers/src/lib.rs index 30994e1..18d0874 100644 --- a/crates/librqbit/src/buffers.rs +++ b/crates/buffers/src/lib.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Deserializer}; -use crate::clone_to_owned::CloneToOwned; +use clone_to_owned::CloneToOwned; #[derive(Default, PartialEq, Eq, Hash, Clone, PartialOrd, Ord)] pub struct ByteString(pub Vec); @@ -69,7 +69,15 @@ impl std::fmt::Display for ByteString { } } -impl CloneToOwned for B { +impl<'a> CloneToOwned for ByteBuf<'a> { + type Target = ByteString; + + fn clone_to_owned(&self) -> Self::Target { + ByteString(self.as_slice().to_owned()) + } +} + +impl CloneToOwned for ByteString { type Target = ByteString; fn clone_to_owned(&self) -> Self::Target { diff --git a/crates/clone_to_owned/Cargo.toml b/crates/clone_to_owned/Cargo.toml new file mode 100644 index 0000000..911fc8b --- /dev/null +++ b/crates/clone_to_owned/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "clone_to_owned" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/librqbit/src/clone_to_owned.rs b/crates/clone_to_owned/src/lib.rs similarity index 100% rename from crates/librqbit/src/clone_to_owned.rs rename to crates/clone_to_owned/src/lib.rs diff --git a/crates/librqbit/Cargo.toml b/crates/librqbit/Cargo.toml index 606e99c..37034e0 100644 --- a/crates/librqbit/Cargo.toml +++ b/crates/librqbit/Cargo.toml @@ -13,6 +13,13 @@ sha1-rust = ["sha1"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +bencode = {path = "../bencode"} +buffers = {path = "../buffers"} +librqbit_core = {path = "../librqbit_core"} +clone_to_owned = {path = "../clone_to_owned"} +peer_binary_protocol = {path = "../peer_binary_protocol"} +sha1w = {path = "../sha1w"} + tokio = {version = "1", features = ["macros", "rt-multi-thread"]} serde = {version = "1", features=["derive"]} anyhow = "1" @@ -39,11 +46,4 @@ hex = "0.4" [dev-dependencies] futures = {version = "0.3"} -pretty_env_logger = "0.4" -hex = "0.4" - -[profile.dev] -panic = "abort" - -[profile.release] -panic = "abort" \ No newline at end of file +pretty_env_logger = "0.4" \ No newline at end of file diff --git a/crates/librqbit/src/chunk_tracker.rs b/crates/librqbit/src/chunk_tracker.rs index a092f7a..1385a4d 100644 --- a/crates/librqbit/src/chunk_tracker.rs +++ b/crates/librqbit/src/chunk_tracker.rs @@ -1,10 +1,8 @@ +use librqbit_core::lengths::{ChunkInfo, Lengths, ValidPieceIndex}; use log::{debug, info}; +use peer_binary_protocol::Piece; -use crate::{ - lengths::{ChunkInfo, Lengths, ValidPieceIndex}, - peer_binary_protocol::Piece, - type_aliases::BF, -}; +use crate::type_aliases::BF; pub struct ChunkTracker { // This forms the basis of a "queue" to pull from. @@ -131,7 +129,11 @@ impl ChunkTracker { where ByteBuf: AsRef<[u8]>, { - let chunk_info = self.lengths.chunk_info_from_received_piece(piece)?; + let chunk_info = self.lengths.chunk_info_from_received_piece( + piece.index, + piece.begin, + piece.block.as_ref().len() as u32, + )?; let chunk_range = self.lengths.chunk_range(chunk_info.piece_index); let chunk_range = self.chunk_status.get_mut(chunk_range).unwrap(); if chunk_range.all() { diff --git a/crates/librqbit/src/dht/inforead.rs b/crates/librqbit/src/dht/inforead.rs index 4a13997..999fbc2 100644 --- a/crates/librqbit/src/dht/inforead.rs +++ b/crates/librqbit/src/dht/inforead.rs @@ -1,10 +1,12 @@ use std::net::SocketAddr; +use buffers::ByteString; use futures::{stream::FuturesUnordered, StreamExt}; +use librqbit_core::torrent_metainfo::TorrentMetaV1Info; use log::debug; use tokio::sync::mpsc::UnboundedReceiver; -use crate::{buffers::ByteString, peer_info_reader, torrent_metainfo::TorrentMetaV1Info}; +use crate::peer_info_reader; #[derive(Debug)] pub enum ReadMetainfoResult { @@ -61,9 +63,11 @@ pub async fn read_metainfo_from_peer_receiver( #[cfg(test)] mod tests { + use librqbit_core::{info_hash::decode_info_hash, peer_id::generate_peer_id}; + + use crate::dht::jsdht::JsDht; + use super::*; - use crate::info_hash::decode_info_hash; - use crate::{dht::jsdht::JsDht, peer_id::generate_peer_id}; use std::sync::Once; static LOG_INIT: Once = Once::new(); diff --git a/crates/librqbit/src/file_ops.rs b/crates/librqbit/src/file_ops.rs index 7014a56..b690b8f 100644 --- a/crates/librqbit/src/file_ops.rs +++ b/crates/librqbit/src/file_ops.rs @@ -6,17 +6,17 @@ use std::{ }; use anyhow::Context; +use buffers::ByteString; +use librqbit_core::{ + lengths::{ChunkInfo, Lengths, ValidPieceIndex}, + torrent_metainfo::{FileIteratorName, TorrentMetaV1Info}, +}; use log::{debug, trace, warn}; use parking_lot::Mutex; +use peer_binary_protocol::Piece; +use sha1w::ISha1; -use crate::{ - buffers::ByteString, - lengths::{ChunkInfo, Lengths, ValidPieceIndex}, - peer_binary_protocol::Piece, - sha1w::ISha1, - torrent_metainfo::{FileIteratorName, TorrentMetaV1Info}, - type_aliases::{PeerHandle, BF}, -}; +use crate::type_aliases::{PeerHandle, BF}; pub struct InitialCheckResults { pub needed_pieces: BF, diff --git a/crates/librqbit/src/http_api.rs b/crates/librqbit/src/http_api.rs index f4b96ec..d983bcb 100644 --- a/crates/librqbit/src/http_api.rs +++ b/crates/librqbit/src/http_api.rs @@ -1,11 +1,11 @@ use std::sync::Arc; +use librqbit_core::speed_estimator::SpeedEstimator; use std::io::Write; use std::sync::atomic::Ordering; use std::time::Instant; use warp::Filter; -use crate::speed_estimator::SpeedEstimator; use crate::torrent_state::TorrentState; // This is just a stub for debugging, nothing useful here. diff --git a/crates/librqbit/src/lib.rs b/crates/librqbit/src/lib.rs index 979c2dc..9a6fbc5 100644 --- a/crates/librqbit/src/lib.rs +++ b/crates/librqbit/src/lib.rs @@ -1,27 +1,20 @@ -pub mod bencode_value; -pub mod buffers; pub mod chunk_tracker; -pub mod clone_to_owned; -pub mod constants; pub mod dht; pub mod file_ops; pub mod http_api; -pub mod info_hash; -pub mod lengths; -pub mod magnet; -pub mod peer_binary_protocol; pub mod peer_connection; pub mod peer_handler; -pub mod peer_id; pub mod peer_info_reader; pub mod peer_state; -pub mod serde_bencode_de; -pub mod serde_bencode_ser; -pub mod sha1w; pub mod spawn_utils; -pub mod speed_estimator; pub mod torrent_manager; -pub mod torrent_metainfo; pub mod torrent_state; pub mod tracker_comms; pub mod type_aliases; + +pub use buffers::*; +pub use clone_to_owned::CloneToOwned; +pub use librqbit_core::info_hash::*; +pub use librqbit_core::magnet::*; +pub use librqbit_core::peer_id::*; +pub use librqbit_core::torrent_metainfo::*; diff --git a/crates/librqbit/src/peer_connection.rs b/crates/librqbit/src/peer_connection.rs index 6ad3f5d..d8d9181 100644 --- a/crates/librqbit/src/peer_connection.rs +++ b/crates/librqbit/src/peer_connection.rs @@ -1,20 +1,16 @@ use std::{net::SocketAddr, time::Duration}; use anyhow::Context; +use buffers::{ByteBuf, ByteString}; +use clone_to_owned::CloneToOwned; +use librqbit_core::{lengths::ChunkInfo, peer_id::try_decode_peer_id}; use log::{debug, trace}; -use tokio::time::timeout; - -use crate::{ - buffers::{ByteBuf, ByteString}, - clone_to_owned::CloneToOwned, - lengths::ChunkInfo, - peer_binary_protocol::{ - extended::{handshake::ExtendedHandshake, ExtendedMessage}, - serialize_piece_preamble, Handshake, Message, MessageBorrowed, MessageDeserializeError, - MessageOwned, PIECE_MESSAGE_DEFAULT_LEN, - }, - peer_id::try_decode_peer_id, +use peer_binary_protocol::{ + extended::{handshake::ExtendedHandshake, ExtendedMessage}, + serialize_piece_preamble, Handshake, Message, MessageBorrowed, MessageDeserializeError, + MessageOwned, PIECE_MESSAGE_DEFAULT_LEN, }; +use tokio::time::timeout; pub trait PeerConnectionHandler { fn get_have_bytes(&self) -> u64; diff --git a/crates/librqbit/src/peer_info_reader/mod.rs b/crates/librqbit/src/peer_info_reader/mod.rs index f378de2..0e9c267 100644 --- a/crates/librqbit/src/peer_info_reader/mod.rs +++ b/crates/librqbit/src/peer_info_reader/mod.rs @@ -1,23 +1,22 @@ use std::net::SocketAddr; -use crate::sha1w::ISha1; -use log::debug; -use parking_lot::{Mutex, RwLock}; -use tokio::sync::mpsc::UnboundedSender; - -use crate::{ - buffers::{ByteBuf, ByteString}, +use bencode::from_bytes; +use buffers::{ByteBuf, ByteString}; +use librqbit_core::{ constants::CHUNK_SIZE, lengths::{ceil_div_u64, last_element_size_u64, ChunkInfo}, - peer_binary_protocol::{ - extended::{handshake::ExtendedHandshake, ut_metadata::UtMetadata, ExtendedMessage}, - Handshake, Message, - }, - peer_connection::{PeerConnection, PeerConnectionHandler, WriterRequest}, - serde_bencode_de::from_bytes, torrent_metainfo::TorrentMetaV1Info, - type_aliases::Sha1, }; +use log::debug; +use parking_lot::{Mutex, RwLock}; +use peer_binary_protocol::{ + extended::{handshake::ExtendedHandshake, ut_metadata::UtMetadata, ExtendedMessage}, + Handshake, Message, +}; +use sha1w::{ISha1, Sha1}; +use tokio::sync::mpsc::UnboundedSender; + +use crate::peer_connection::{PeerConnection, PeerConnectionHandler, WriterRequest}; pub async fn read_metainfo_from_peer( addr: SocketAddr, @@ -217,7 +216,7 @@ impl PeerConnectionHandler for Handler { mod tests { use std::{net::SocketAddr, str::FromStr, sync::Once}; - use crate::peer_id::generate_peer_id; + use librqbit_core::peer_id::generate_peer_id; use super::read_metainfo_from_peer; diff --git a/crates/librqbit/src/peer_state.rs b/crates/librqbit/src/peer_state.rs index f6f979f..763f7f2 100644 --- a/crates/librqbit/src/peer_state.rs +++ b/crates/librqbit/src/peer_state.rs @@ -1,11 +1,9 @@ use std::{collections::HashSet, net::SocketAddr, sync::Arc}; +use librqbit_core::lengths::{ChunkInfo, ValidPieceIndex}; use tokio::sync::{Notify, Semaphore}; -use crate::{ - lengths::{ChunkInfo, ValidPieceIndex}, - type_aliases::BF, -}; +use crate::type_aliases::BF; #[derive(Debug, Hash, PartialEq, Eq)] pub struct InflightRequest { diff --git a/crates/librqbit/src/torrent_manager.rs b/crates/librqbit/src/torrent_manager.rs index f86336c..ef02cba 100644 --- a/crates/librqbit/src/torrent_manager.rs +++ b/crates/librqbit/src/torrent_manager.rs @@ -12,24 +12,25 @@ use std::{ }; use anyhow::Context; +use bencode::from_bytes; +use buffers::ByteString; +use librqbit_core::{ + lengths::Lengths, peer_id::generate_peer_id, speed_estimator::SpeedEstimator, + torrent_metainfo::TorrentMetaV1Info, +}; use log::{debug, info}; use parking_lot::{Mutex, RwLock}; use reqwest::Url; +use sha1w::Sha1; use size_format::SizeFormatterBinary as SF; use crate::{ - buffers::ByteString, chunk_tracker::ChunkTracker, file_ops::FileOps, http_api::make_and_run_http_api, - lengths::Lengths, - peer_id::generate_peer_id, spawn_utils::{spawn, BlockingSpawner}, - speed_estimator::SpeedEstimator, - torrent_metainfo::TorrentMetaV1Info, torrent_state::{AtomicStats, TorrentState, TorrentStateLocked}, tracker_comms::{TrackerError, TrackerRequest, TrackerRequestEvent, TrackerResponse}, - type_aliases::Sha1, }; pub struct TorrentManagerBuilder { info: TorrentMetaV1Info, @@ -307,13 +308,13 @@ impl TorrentManager { anyhow::bail!("tracker responded with {:?}", response.status()); } let bytes = response.bytes().await?; - if let Ok(error) = crate::serde_bencode_de::from_bytes::(&bytes) { + if let Ok(error) = from_bytes::(&bytes) { anyhow::bail!( "tracker returned failure. Failure reason: {}", error.failure_reason ) }; - let response = crate::serde_bencode_de::from_bytes::(&bytes)?; + let response = from_bytes::(&bytes)?; for peer in response.peers.iter_sockaddrs() { self.state.add_peer_if_not_seen(peer); diff --git a/crates/librqbit/src/torrent_state.rs b/crates/librqbit/src/torrent_state.rs index 45cc894..9192d76 100644 --- a/crates/librqbit/src/torrent_state.rs +++ b/crates/librqbit/src/torrent_state.rs @@ -10,25 +10,28 @@ use std::{ }; use anyhow::Context; +use buffers::{ByteBuf, ByteString}; +use clone_to_owned::CloneToOwned; use futures::{stream::FuturesUnordered, StreamExt}; +use librqbit_core::{ + lengths::{ChunkInfo, Lengths, ValidPieceIndex}, + torrent_metainfo::TorrentMetaV1Info, +}; use log::{debug, info, trace, warn}; use parking_lot::{Mutex, RwLock}; +use peer_binary_protocol::{ + extended::handshake::ExtendedHandshake, Handshake, Message, MessageOwned, Piece, Request, +}; +use sha1w::Sha1; use tokio::{sync::mpsc::UnboundedSender, time::timeout}; use crate::{ - buffers::{ByteBuf, ByteString}, chunk_tracker::{ChunkMarkingResult, ChunkTracker}, - clone_to_owned::CloneToOwned, file_ops::FileOps, - lengths::{Lengths, ValidPieceIndex}, - peer_binary_protocol::{ - extended::handshake::ExtendedHandshake, Handshake, Message, MessageOwned, Piece, Request, - }, peer_connection::{PeerConnection, PeerConnectionHandler, WriterRequest}, peer_state::{InflightRequest, LivePeerState, PeerState}, spawn_utils::{spawn, BlockingSpawner}, - torrent_metainfo::TorrentMetaV1Info, - type_aliases::{PeerHandle, Sha1, BF}, + type_aliases::{PeerHandle, BF}, }; pub struct InflightPiece { @@ -516,7 +519,7 @@ impl PeerConnectionHandler for PeerHandler { .fetch_add(bytes as u64, Ordering::Relaxed); } - fn read_chunk(&self, chunk: &crate::lengths::ChunkInfo, buf: &mut [u8]) -> anyhow::Result<()> { + fn read_chunk(&self, chunk: &ChunkInfo, buf: &mut [u8]) -> anyhow::Result<()> { self.state.file_ops().read_chunk(self.addr, chunk, buf) } @@ -764,7 +767,11 @@ impl PeerHandler { } fn on_received_piece(&self, handle: PeerHandle, piece: Piece) -> anyhow::Result<()> { - let chunk_info = match self.state.lengths.chunk_info_from_received_piece(&piece) { + let chunk_info = match self.state.lengths.chunk_info_from_received_piece( + piece.index, + piece.begin, + piece.block.len() as u32, + ) { Some(i) => i, None => { anyhow::bail!( diff --git a/crates/librqbit/src/tracker_comms.rs b/crates/librqbit/src/tracker_comms.rs index 2c4412b..9827ba6 100644 --- a/crates/librqbit/src/tracker_comms.rs +++ b/crates/librqbit/src/tracker_comms.rs @@ -1,3 +1,4 @@ +use buffers::ByteBuf; use byteorder::ByteOrder; use serde::{Deserialize, Deserializer}; use std::{ @@ -7,8 +8,6 @@ use std::{ str::FromStr, }; -use crate::buffers::ByteBuf; - #[derive(Clone, Copy)] pub enum TrackerRequestEvent { Started, diff --git a/crates/librqbit/src/type_aliases.rs b/crates/librqbit/src/type_aliases.rs index c01f88f..dc4de81 100644 --- a/crates/librqbit/src/type_aliases.rs +++ b/crates/librqbit/src/type_aliases.rs @@ -3,12 +3,3 @@ use std::net::SocketAddr; pub type BF = bitvec::vec::BitVec; pub type PeerHandle = SocketAddr; - -#[cfg(feature = "sha1-openssl")] -pub type Sha1 = crate::sha1w::Sha1Openssl; - -#[cfg(feature = "sha1-system")] -pub type Sha1 = crate::sha1w::Sha1System; - -#[cfg(feature = "sha1-rust")] -pub type Sha1 = crate::sha1w::Sha1Rust; diff --git a/crates/librqbit_core/Cargo.toml b/crates/librqbit_core/Cargo.toml new file mode 100644 index 0000000..e0c25b5 --- /dev/null +++ b/crates/librqbit_core/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "librqbit_core" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +hex = "0.4" +anyhow = "1" +url = "2" +uuid = {version = "0.8", features = ["v4"]} +log = "0.4" +parking_lot = "0.11" +serde = {version = "1", features=["derive"]} +buffers = {path="../buffers"} +bencode = {path="../bencode"} +clone_to_owned = {path="../clone_to_owned"} \ No newline at end of file diff --git a/crates/librqbit/src/constants.rs b/crates/librqbit_core/src/constants.rs similarity index 100% rename from crates/librqbit/src/constants.rs rename to crates/librqbit_core/src/constants.rs diff --git a/crates/librqbit/src/info_hash.rs b/crates/librqbit_core/src/info_hash.rs similarity index 100% rename from crates/librqbit/src/info_hash.rs rename to crates/librqbit_core/src/info_hash.rs diff --git a/crates/librqbit/src/lengths.rs b/crates/librqbit_core/src/lengths.rs similarity index 95% rename from crates/librqbit/src/lengths.rs rename to crates/librqbit_core/src/lengths.rs index 5ebd728..d65b53b 100644 --- a/crates/librqbit/src/lengths.rs +++ b/crates/librqbit_core/src/lengths.rs @@ -1,4 +1,4 @@ -use crate::{constants::CHUNK_SIZE, peer_binary_protocol::Piece}; +use crate::constants::CHUNK_SIZE; const fn is_power_of_two(x: u64) -> bool { (x != 0) && ((x & (x - 1)) == 0) @@ -189,18 +189,13 @@ impl Lengths { }) } - pub fn chunk_info_from_received_piece( + pub fn chunk_info_from_received_piece( &self, - piece: &Piece, - ) -> Option - where - ByteBuf: AsRef<[u8]>, - { - self.chunk_info_from_received_data( - self.validate_piece_index(piece.index)?, - piece.begin, - piece.block.as_ref().len() as u32, - ) + index: u32, + begin: u32, + block_len: u32, + ) -> Option { + self.chunk_info_from_received_data(self.validate_piece_index(index)?, begin, block_len) } pub const fn chunk_range(&self, index: ValidPieceIndex) -> std::ops::Range { let start = index.0 * self.chunks_per_piece; diff --git a/crates/librqbit_core/src/lib.rs b/crates/librqbit_core/src/lib.rs new file mode 100644 index 0000000..44c4b75 --- /dev/null +++ b/crates/librqbit_core/src/lib.rs @@ -0,0 +1,7 @@ +pub mod constants; +pub mod info_hash; +pub mod lengths; +pub mod magnet; +pub mod peer_id; +pub mod speed_estimator; +pub mod torrent_metainfo; diff --git a/crates/librqbit/src/magnet.rs b/crates/librqbit_core/src/magnet.rs similarity index 100% rename from crates/librqbit/src/magnet.rs rename to crates/librqbit_core/src/magnet.rs diff --git a/crates/librqbit/src/peer_id.rs b/crates/librqbit_core/src/peer_id.rs similarity index 100% rename from crates/librqbit/src/peer_id.rs rename to crates/librqbit_core/src/peer_id.rs diff --git a/crates/librqbit/src/speed_estimator.rs b/crates/librqbit_core/src/speed_estimator.rs similarity index 100% rename from crates/librqbit/src/speed_estimator.rs rename to crates/librqbit_core/src/speed_estimator.rs diff --git a/crates/librqbit/src/torrent_metainfo.rs b/crates/librqbit_core/src/torrent_metainfo.rs similarity index 98% rename from crates/librqbit/src/torrent_metainfo.rs rename to crates/librqbit_core/src/torrent_metainfo.rs index 4b6ef4d..f094ae2 100644 --- a/crates/librqbit/src/torrent_metainfo.rs +++ b/crates/librqbit_core/src/torrent_metainfo.rs @@ -1,13 +1,10 @@ use std::{fmt::Write, ops::Deref, path::PathBuf}; +use bencode::BencodeDeserializer; +use buffers::{ByteBuf, ByteString}; +use clone_to_owned::CloneToOwned; use serde::Deserialize; -use crate::{ - buffers::{ByteBuf, ByteString}, - clone_to_owned::CloneToOwned, - serde_bencode_de::BencodeDeserializer, -}; - pub type TorrentMetaV1Borrowed<'a> = TorrentMetaV1>; pub type TorrentMetaV1Owned = TorrentMetaV1; diff --git a/crates/peer_binary_protocol/Cargo.toml b/crates/peer_binary_protocol/Cargo.toml new file mode 100644 index 0000000..8094ca1 --- /dev/null +++ b/crates/peer_binary_protocol/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "peer_binary_protocol" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = {version = "1", features = ["derive"]} +bincode = "1" +byteorder = "1" +buffers = {path="../buffers"} +bencode = {path="../bencode"} +clone_to_owned = {path="../clone_to_owned"} +librqbit_core = {path="../librqbit_core"} +bitvec = "0.22" +anyhow = "1" \ No newline at end of file diff --git a/crates/librqbit/src/peer_binary_protocol/extended/handshake.rs b/crates/peer_binary_protocol/src/extended/handshake.rs similarity index 97% rename from crates/librqbit/src/peer_binary_protocol/extended/handshake.rs rename to crates/peer_binary_protocol/src/extended/handshake.rs index 54e8c72..5e9ef09 100644 --- a/crates/librqbit/src/peer_binary_protocol/extended/handshake.rs +++ b/crates/peer_binary_protocol/src/extended/handshake.rs @@ -3,13 +3,13 @@ use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, }; +use buffers::ByteBuf; use byteorder::ByteOrder; use byteorder::BE; +use clone_to_owned::CloneToOwned; use serde::{Deserialize, Deserializer, Serialize}; -use crate::{ - buffers::ByteBuf, clone_to_owned::CloneToOwned, peer_binary_protocol::MY_EXTENDED_UT_METADATA, -}; +use crate::MY_EXTENDED_UT_METADATA; #[derive(Deserialize, Serialize, Debug, Default)] pub struct ExtendedHandshake { diff --git a/crates/librqbit/src/peer_binary_protocol/extended/mod.rs b/crates/peer_binary_protocol/src/extended/mod.rs similarity index 90% rename from crates/librqbit/src/peer_binary_protocol/extended/mod.rs rename to crates/peer_binary_protocol/src/extended/mod.rs index 31adbcc..0a3569a 100644 --- a/crates/librqbit/src/peer_binary_protocol/extended/mod.rs +++ b/crates/peer_binary_protocol/src/extended/mod.rs @@ -1,7 +1,10 @@ +use bencode::bencode_serialize_to_writer; +use bencode::from_bytes; +use bencode::BencodeValue; +use buffers::ByteString; +use clone_to_owned::CloneToOwned; use serde::{Deserialize, Serialize}; -use crate::{bencode_value::BencodeValue, buffers::ByteString, clone_to_owned::CloneToOwned}; - use self::{handshake::ExtendedHandshake, ut_metadata::UtMetadata}; use super::MessageDeserializeError; @@ -46,11 +49,11 @@ impl<'a, ByteBuf: 'a + std::hash::Hash + Eq + Serialize> ExtendedMessage { out.push(*msg_id); - crate::serde_bencode_ser::bencode_serialize_to_writer(v, out)?; + bencode_serialize_to_writer(v, out)?; } ExtendedMessage::Handshake(h) => { out.push(0); - crate::serde_bencode_ser::bencode_serialize_to_writer(h, out)?; + bencode_serialize_to_writer(h, out)?; } ExtendedMessage::UtMetadata(u) => { let h = extended_handshake.ok_or_else(|| { @@ -70,8 +73,6 @@ impl<'a, ByteBuf: 'a + std::hash::Hash + Eq + Serialize> ExtendedMessage + From<&'a [u8]>, { - use crate::serde_bencode_de::from_bytes; - let emsg_id = buf.get(0).copied().ok_or_else(|| { MessageDeserializeError::Other(anyhow::anyhow!( "cannot deserialize extended message: can't read first byte" diff --git a/crates/librqbit/src/peer_binary_protocol/extended/ut_metadata.rs b/crates/peer_binary_protocol/src/extended/ut_metadata.rs similarity index 95% rename from crates/librqbit/src/peer_binary_protocol/extended/ut_metadata.rs rename to crates/peer_binary_protocol/src/extended/ut_metadata.rs index c80f0b7..d2f7be3 100644 --- a/crates/librqbit/src/peer_binary_protocol/extended/ut_metadata.rs +++ b/crates/peer_binary_protocol/src/extended/ut_metadata.rs @@ -1,12 +1,13 @@ use std::io::Write; -use crate::{ - clone_to_owned::CloneToOwned, peer_binary_protocol::MessageDeserializeError, - serde_bencode_de::BencodeDeserializer, serde_bencode_ser::bencode_serialize_to_writer, -}; +use bencode::bencode_serialize_to_writer; +use bencode::BencodeDeserializer; +use clone_to_owned::CloneToOwned; use serde::Deserialize; use serde::Serialize; +use crate::MessageDeserializeError; + #[derive(Debug)] pub enum UtMetadata { Request(u32), diff --git a/crates/librqbit/src/peer_binary_protocol/mod.rs b/crates/peer_binary_protocol/src/lib.rs similarity index 99% rename from crates/librqbit/src/peer_binary_protocol/mod.rs rename to crates/peer_binary_protocol/src/lib.rs index e197e08..9e93b1f 100644 --- a/crates/librqbit/src/peer_binary_protocol/mod.rs +++ b/crates/peer_binary_protocol/src/lib.rs @@ -1,16 +1,12 @@ pub mod extended; use bincode::Options; +use buffers::{ByteBuf, ByteString}; use byteorder::{ByteOrder, BE}; +use clone_to_owned::CloneToOwned; +use librqbit_core::{constants::CHUNK_SIZE, lengths::ChunkInfo}; use serde::{Deserialize, Serialize}; -use crate::{ - buffers::{ByteBuf, ByteString}, - clone_to_owned::CloneToOwned, - constants::CHUNK_SIZE, - lengths::ChunkInfo, -}; - use self::extended::{handshake::ExtendedHandshake, ExtendedMessage}; const INTEGER_LEN: usize = 4; diff --git a/crates/rqbit/Cargo.toml b/crates/rqbit/Cargo.toml index 6e4ac3a..6a7ef15 100644 --- a/crates/rqbit/Cargo.toml +++ b/crates/rqbit/Cargo.toml @@ -18,10 +18,4 @@ reqwest = "0.11" regex = "1" [dev-dependencies] -futures = {version = "0.3"} - -[profile.dev] -panic = "abort" - -[profile.release] -panic = "abort" \ No newline at end of file +futures = {version = "0.3"} \ No newline at end of file diff --git a/crates/rqbit/src/main.rs b/crates/rqbit/src/main.rs index 150fcb0..8b4be30 100644 --- a/crates/rqbit/src/main.rs +++ b/crates/rqbit/src/main.rs @@ -3,14 +3,12 @@ use std::{fs::File, io::Read, net::SocketAddr, time::Duration}; use anyhow::Context; use clap::Clap; use librqbit::{ - buffers::ByteString, dht::{inforead::read_metainfo_from_peer_receiver, jsdht::JsDht}, - info_hash::InfoHash, - magnet, - peer_id::generate_peer_id, + generate_peer_id, spawn_utils::{spawn, BlockingSpawner}, + torrent_from_bytes, torrent_manager::TorrentManagerBuilder, - torrent_metainfo::{torrent_from_bytes, TorrentMetaV1Info, TorrentMetaV1Owned}, + ByteString, InfoHash, Magnet, TorrentMetaV1Info, TorrentMetaV1Owned, }; use log::{info, warn}; use reqwest::Url; @@ -171,11 +169,10 @@ fn main() -> anyhow::Result<()> { async fn async_main(opts: Opts, spawner: BlockingSpawner) -> anyhow::Result<()> { let peer_id = generate_peer_id(); if opts.torrent_path.starts_with("magnet:") { - let magnet::Magnet { + let Magnet { info_hash, trackers, - } = magnet::Magnet::parse(&opts.torrent_path) - .context("provided path is not a valid magnet URL")?; + } = Magnet::parse(&opts.torrent_path).context("provided path is not a valid magnet URL")?; let dht_rx = JsDht::new(info_hash).start_peer_discovery()?; let (info, dht_rx, initial_peers) = match read_metainfo_from_peer_receiver(peer_id, info_hash, dht_rx).await { diff --git a/crates/sha1w/Cargo.toml b/crates/sha1w/Cargo.toml new file mode 100644 index 0000000..81fd3d0 --- /dev/null +++ b/crates/sha1w/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "sha1w" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["default-sha1-openssl"] +sha1-system = ["crypto-hash"] +sha1-openssl = ["openssl"] +sha1-rust = ["sha1"] +default-sha1-openssl = ["sha1-openssl"] +default-sha1-rust = ["sha1-rust"] +default-sha1-system = ["sha1-system"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +openssl = {version="0.10", optional=true} +crypto-hash = {version="0.3", optional=true} +sha1 = {version = "0.6", optional=true} \ No newline at end of file diff --git a/crates/librqbit/src/sha1w.rs b/crates/sha1w/src/lib.rs similarity index 89% rename from crates/librqbit/src/sha1w.rs rename to crates/sha1w/src/lib.rs index 0bc6db8..d761347 100644 --- a/crates/librqbit/src/sha1w.rs +++ b/crates/sha1w/src/lib.rs @@ -4,6 +4,15 @@ // leaving the pure-rust impl here too. Maybe someday make them // runtime swappable or enabled with a feature. +#[cfg(feature = "default-sha1-openssl")] +pub type Sha1 = Sha1Openssl; + +#[cfg(feature = "default-sha1-rust")] +pub type Sha1 = Sha1Rust; + +#[cfg(feature = "default-sha1-system")] +pub type Sha1 = Sha1System; + pub trait ISha1 { fn new() -> Self; fn update(&mut self, buf: &[u8]);