From f22814c77bd5336d137469dee38c431cd1d7c900 Mon Sep 17 00:00:00 2001 From: Igor Katson Date: Thu, 15 Aug 2024 13:31:56 +0100 Subject: [PATCH 1/2] Postgres example storage --- Cargo.lock | 595 +++++++++++++++++- crates/librqbit/Cargo.toml | 1 + .../librqbit/src/session_persistence/mod.rs | 2 + .../src/session_persistence/postgres.rs | 164 +++++ crates/librqbit/src/torrent_state/mod.rs | 4 + crates/librqbit_core/src/hash_id.rs | 6 +- 6 files changed, 748 insertions(+), 24 deletions(-) create mode 100644 crates/librqbit/src/session_persistence/postgres.rs diff --git a/Cargo.lock b/Cargo.lock index 67b00e0..c30a2f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -122,7 +123,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -144,7 +145,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -155,7 +156,16 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -308,6 +318,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bincode" version = "1.3.3" @@ -328,6 +344,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "bitvec" @@ -435,7 +454,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -505,6 +524,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.4" @@ -530,6 +555,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -548,6 +588,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -597,7 +646,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn", + "syn 2.0.72", ] [[package]] @@ -608,7 +657,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -631,6 +680,17 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -648,7 +708,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -672,11 +734,20 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" @@ -703,6 +774,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "fastrand" version = "2.1.0" @@ -725,6 +813,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -803,6 +902,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.30" @@ -817,7 +927,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -940,6 +1050,15 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "hdrhistogram" version = "7.5.4" @@ -958,6 +1077,9 @@ name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -977,6 +1099,33 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "0.2.12" @@ -1273,6 +1422,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "leaky-bucket" @@ -1291,6 +1443,12 @@ version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libredox" version = "0.1.3" @@ -1345,6 +1503,7 @@ dependencies = [ "serde_with", "sha1", "size_format", + "sqlx", "tempfile", "tokio", "tokio-socks", @@ -1497,6 +1656,17 @@ dependencies = [ "url", ] +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -1543,6 +1713,16 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -1674,6 +1854,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.2.4" @@ -1729,6 +1926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1769,7 +1967,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -1832,7 +2030,7 @@ dependencies = [ "cfg-if", "libc", "petgraph", - "redox_syscall", + "redox_syscall 0.5.3", "smallvec", "thread-id", "windows-targets 0.52.6", @@ -1849,6 +2047,21 @@ dependencies = [ "regex", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1882,7 +2095,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -1897,6 +2110,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.30" @@ -1947,7 +2181,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -2052,6 +2286,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.3" @@ -2212,6 +2455,26 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2357,7 +2620,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -2421,7 +2684,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -2435,6 +2698,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2444,6 +2718,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "size_format" version = "1.0.2" @@ -2484,6 +2768,234 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" +dependencies = [ + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex 0.4.3", + "indexmap 2.3.0", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex 0.4.3", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.6.0", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array 0.14.7", + "hex 0.4.3", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.6.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex 0.4.3", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] [[package]] name = "strsim" @@ -2503,6 +3015,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.72" @@ -2583,7 +3106,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -2687,7 +3210,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -2857,7 +3380,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] @@ -2954,6 +3477,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.9.0" @@ -3025,6 +3566,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -3046,7 +3593,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -3080,7 +3627,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3110,6 +3657,16 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3323,7 +3880,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.72", ] [[package]] diff --git a/crates/librqbit/Cargo.toml b/crates/librqbit/Cargo.toml index 955e8aa..dacb034 100644 --- a/crates/librqbit/Cargo.toml +++ b/crates/librqbit/Cargo.toml @@ -21,6 +21,7 @@ storage_middleware = ["lru"] storage_examples = [] [dependencies] +sqlx = { version = "0.7", features = ["runtime-tokio", "postgres"] } bencode = { path = "../bencode", default-features = false, package = "librqbit-bencode", version = "2.2.3" } tracker_comms = { path = "../tracker_comms", default-features = false, package = "librqbit-tracker-comms", version = "1.0.3" } buffers = { path = "../buffers", package = "librqbit-buffers", version = "3.0.1" } diff --git a/crates/librqbit/src/session_persistence/mod.rs b/crates/librqbit/src/session_persistence/mod.rs index c574d3c..f710d9a 100644 --- a/crates/librqbit/src/session_persistence/mod.rs +++ b/crates/librqbit/src/session_persistence/mod.rs @@ -1,4 +1,6 @@ pub mod json; +// #[cfg(feature = "postgres")] +pub mod postgres; use std::{collections::HashSet, path::PathBuf}; diff --git a/crates/librqbit/src/session_persistence/postgres.rs b/crates/librqbit/src/session_persistence/postgres.rs new file mode 100644 index 0000000..0485e8f --- /dev/null +++ b/crates/librqbit/src/session_persistence/postgres.rs @@ -0,0 +1,164 @@ +use std::{path::PathBuf, str::FromStr}; + +use crate::{session::TorrentId, torrent_state::ManagedTorrentHandle}; +use anyhow::Context; +use futures::{stream::BoxStream, StreamExt}; +use librqbit_core::Id20; +use sqlx::{Pool, Postgres}; + +use super::{SerializedTorrent, SessionPersistenceStore}; + +#[derive(Debug)] +pub struct PostgresSessionStorage { + pool: Pool, +} + +#[derive(sqlx::FromRow)] +struct TorrentsTableRecord { + id: i32, + info_hash: String, + torrent_bytes: Vec, + trackers: Vec, + output_folder: String, + only_files: Option>, + is_paused: bool, +} + +impl TorrentsTableRecord { + fn into_serialized_torrent(self) -> Option<(TorrentId, SerializedTorrent)> { + Some(( + self.id as TorrentId, + SerializedTorrent { + info_hash: Id20::from_str(&self.info_hash).ok()?, + torrent_bytes: self.torrent_bytes.into(), + trackers: self.trackers.into_iter().collect(), + output_folder: PathBuf::from(self.output_folder), + only_files: self + .only_files + .map(|v| v.into_iter().map(|v| v as usize).collect()), + is_paused: self.is_paused, + }, + )) + } +} + +impl PostgresSessionStorage { + pub async fn new(connection_string: &str) -> anyhow::Result { + use sqlx::postgres::PgPoolOptions; + + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(connection_string) + .await?; + + sqlx::query( + " + CREATE SEQUENCE IF NOT EXISTS torrents_id; + + CREATE TABLE IF NOT EXISTS torrents ( + id INTEGER PRIMARY KEY DEFAULT nextval('torrents_id'), + info_hash BYTEA NOT NULL, -- Assuming the info_hash is unique and fits in 64 characters + torrent_bytes BYTEA NOT NULL, + trackers TEXT[] NOT NULL, + output_folder TEXT NOT NULL, + only_files INTEGER[], -- Nullable array of integers (usize in Rust is equivalent to integer) + is_paused BOOLEAN NOT NULL -- Boolean value to indicate if the torrent is paused + ); + " + ).execute(&pool).await?; + + Ok(Self { pool }) + } +} + +#[async_trait::async_trait] +impl SessionPersistenceStore for PostgresSessionStorage { + async fn next_id(&self) -> anyhow::Result { + let (id,): (i32,) = sqlx::query_as("SELECT nextval('torrents_id')") + .fetch_one(&self.pool) + .await?; + Ok(id as usize) + } + + async fn store(&self, id: TorrentId, torrent: &ManagedTorrentHandle) -> anyhow::Result<()> { + let torrent_bytes: &[u8] = &torrent.info().torrent_bytes; + let q = " + INSERT INTO torrents (id, info_hash, torrent_bytes, trackers, output_folder, only_files, is_paused) + VALUES(?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO NOTHING + "; + sqlx::query(q) + .bind::(id.try_into()?) + .bind(torrent.info_hash().as_string()) + .bind(torrent_bytes) + .bind(torrent.info().trackers.iter().cloned().collect::>()) + .bind( + torrent + .info() + .options + .output_folder + .to_str() + .context("output_folder")? + .to_owned(), + ) + .bind(torrent.only_files().map(|o| { + o.into_iter() + .filter_map(|o| o.try_into().ok()) + .collect::>() + })) + .bind(torrent.is_paused()) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn delete(&self, id: TorrentId) -> anyhow::Result<()> { + sqlx::query("DELETE * FROM torrents WHERE id = ?") + .bind::(id.try_into()?) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn get(&self, id: TorrentId) -> anyhow::Result { + let row = sqlx::query_as::<_, TorrentsTableRecord>("SELECT * FROM torrents WHERE id = ?") + .bind::(id.try_into()?) + .fetch_one(&self.pool) + .await?; + row.into_serialized_torrent() + .context("bug") + .map(|(_, st)| st) + } + + async fn update_metadata( + &self, + id: TorrentId, + torrent: &ManagedTorrentHandle, + ) -> anyhow::Result<()> { + sqlx::query("UPDATE torrents SET only_files = ?, paused = ? WHERE id = ?") + .bind(torrent.only_files().map(|v| { + v.into_iter() + .filter_map(|f| f.try_into().ok()) + .collect::>() + })) + .bind(torrent.is_paused()) + .bind::(id.try_into()?) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn stream_all( + &self, + ) -> anyhow::Result>> { + let res = futures::stream::iter( + sqlx::query_as::<_, TorrentsTableRecord>("SELECT * FROM torrents") + .fetch_all(&self.pool) + .await? + .into_iter() + .filter_map(TorrentsTableRecord::into_serialized_torrent) + .map(Ok), + ); + Ok(res.boxed()) + } +} diff --git a/crates/librqbit/src/torrent_state/mod.rs b/crates/librqbit/src/torrent_state/mod.rs index 09e7389..8f75dff 100644 --- a/crates/librqbit/src/torrent_state/mod.rs +++ b/crates/librqbit/src/torrent_state/mod.rs @@ -350,6 +350,10 @@ impl ManagedTorrent { } } + pub fn is_paused(&self) -> bool { + self.with_state(|s| matches!(s, ManagedTorrentState::Paused(..))) + } + /// Pause the torrent if it's live. pub(crate) fn pause(&self) -> anyhow::Result<()> { let mut g = self.locked.write(); diff --git a/crates/librqbit_core/src/hash_id.rs b/crates/librqbit_core/src/hash_id.rs index e412a95..7d9c4ae 100644 --- a/crates/librqbit_core/src/hash_id.rs +++ b/crates/librqbit_core/src/hash_id.rs @@ -81,11 +81,7 @@ impl FromStr for Id { Ok(Id(out)) } Err(err) => { - anyhow::bail!( - "fail to decode base32 string {}: {}", - s, - err - ) + anyhow::bail!("fail to decode base32 string {}: {}", s, err) } } } else { From 2871c358e35192f438a690a89fcc64db81a295fb Mon Sep 17 00:00:00 2001 From: Igor Katson Date: Thu, 15 Aug 2024 14:18:55 +0100 Subject: [PATCH 2/2] postgres session storage backend --- Makefile | 7 ++ crates/librqbit/Cargo.toml | 6 +- crates/librqbit/src/session.rs | 8 ++ .../librqbit/src/session_persistence/mod.rs | 2 +- .../src/session_persistence/postgres.rs | 85 ++++++++++--------- crates/librqbit_core/src/hash_id.rs | 9 ++ crates/rqbit/Cargo.toml | 3 +- crates/rqbit/src/main.rs | 27 ++++-- 8 files changed, 99 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index 53bd9b1..800c7d9 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,13 @@ devserver: --log-file-rust-log=debug,librqbit=trace \ server start /tmp/scratch/ +@PHONY: devserver +devserver-postgres: + echo -n '' > /tmp/rqbit-log && cargo run -- \ + --log-file /tmp/rqbit-log \ + --log-file-rust-log=debug,librqbit=trace \ + server start --persistence-config postgres:///rqbit /tmp/scratch/ + @PHONY: clean clean: rm -rf target diff --git a/crates/librqbit/Cargo.toml b/crates/librqbit/Cargo.toml index dacb034..5d7fd62 100644 --- a/crates/librqbit/Cargo.toml +++ b/crates/librqbit/Cargo.toml @@ -19,9 +19,13 @@ default-tls = ["reqwest/default-tls"] rust-tls = ["reqwest/rustls-tls"] storage_middleware = ["lru"] storage_examples = [] +postgres = ["sqlx"] [dependencies] -sqlx = { version = "0.7", features = ["runtime-tokio", "postgres"] } +sqlx = { version = "0.7", features = [ + "runtime-tokio", + "postgres", +], optional = true } bencode = { path = "../bencode", default-features = false, package = "librqbit-bencode", version = "2.2.3" } tracker_comms = { path = "../tracker_comms", default-features = false, package = "librqbit-tracker-comms", version = "1.0.3" } buffers = { path = "../buffers", package = "librqbit-buffers", version = "3.0.1" } diff --git a/crates/librqbit/src/session.rs b/crates/librqbit/src/session.rs index 70afda0..848e4f4 100644 --- a/crates/librqbit/src/session.rs +++ b/crates/librqbit/src/session.rs @@ -332,6 +332,8 @@ impl<'a> AddTorrent<'a> { pub enum SessionPersistenceConfig { /// The filename for persistence. By default uses an OS-specific folder. Json { folder: Option }, + #[cfg(feature = "postgres")] + Postgres { connection_string: String }, } impl SessionPersistenceConfig { @@ -494,6 +496,12 @@ impl Session { .await .context("error initializing JsonSessionPersistenceStore")?, ))) + }, + #[cfg(feature = "postgres")] + Some(SessionPersistenceConfig::Postgres { connection_string }) => { + use crate::session_persistence::postgres::PostgresSessionStorage; + let p = PostgresSessionStorage::new(connection_string).await?; + Ok(Some(Box::new(p))) } None => Ok(None), } diff --git a/crates/librqbit/src/session_persistence/mod.rs b/crates/librqbit/src/session_persistence/mod.rs index f710d9a..3de287d 100644 --- a/crates/librqbit/src/session_persistence/mod.rs +++ b/crates/librqbit/src/session_persistence/mod.rs @@ -1,5 +1,5 @@ pub mod json; -// #[cfg(feature = "postgres")] +#[cfg(feature = "postgres")] pub mod postgres; use std::{collections::HashSet, path::PathBuf}; diff --git a/crates/librqbit/src/session_persistence/postgres.rs b/crates/librqbit/src/session_persistence/postgres.rs index 0485e8f..bd1d45b 100644 --- a/crates/librqbit/src/session_persistence/postgres.rs +++ b/crates/librqbit/src/session_persistence/postgres.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, str::FromStr}; +use std::path::PathBuf; use crate::{session::TorrentId, torrent_state::ManagedTorrentHandle}; use anyhow::Context; @@ -16,7 +16,7 @@ pub struct PostgresSessionStorage { #[derive(sqlx::FromRow)] struct TorrentsTableRecord { id: i32, - info_hash: String, + info_hash: Vec, torrent_bytes: Vec, trackers: Vec, output_folder: String, @@ -29,7 +29,7 @@ impl TorrentsTableRecord { Some(( self.id as TorrentId, SerializedTorrent { - info_hash: Id20::from_str(&self.info_hash).ok()?, + info_hash: Id20::from_bytes(&self.info_hash).ok()?, torrent_bytes: self.torrent_bytes.into(), trackers: self.trackers.into_iter().collect(), output_folder: PathBuf::from(self.output_folder), @@ -51,21 +51,24 @@ impl PostgresSessionStorage { .connect(connection_string) .await?; - sqlx::query( - " - CREATE SEQUENCE IF NOT EXISTS torrents_id; + sqlx::query("CREATE SEQUENCE IF NOT EXISTS torrents_id AS integer;") + .execute(&pool) + .await + .context("error executing CREATE SEQUENCE")?; - CREATE TABLE IF NOT EXISTS torrents ( - id INTEGER PRIMARY KEY DEFAULT nextval('torrents_id'), - info_hash BYTEA NOT NULL, -- Assuming the info_hash is unique and fits in 64 characters - torrent_bytes BYTEA NOT NULL, - trackers TEXT[] NOT NULL, - output_folder TEXT NOT NULL, - only_files INTEGER[], -- Nullable array of integers (usize in Rust is equivalent to integer) - is_paused BOOLEAN NOT NULL -- Boolean value to indicate if the torrent is paused - ); - " - ).execute(&pool).await?; + let create_q = "CREATE TABLE IF NOT EXISTS torrents ( + id INTEGER PRIMARY KEY DEFAULT nextval('torrents_id'), + info_hash BYTEA NOT NULL, + torrent_bytes BYTEA NOT NULL, + trackers TEXT[] NOT NULL, + output_folder TEXT NOT NULL, + only_files INTEGER[], + is_paused BOOLEAN NOT NULL + )"; + sqlx::query(create_q) + .execute(&pool) + .await + .context("error executing CREATE TABLE")?; Ok(Self { pool }) } @@ -74,22 +77,21 @@ impl PostgresSessionStorage { #[async_trait::async_trait] impl SessionPersistenceStore for PostgresSessionStorage { async fn next_id(&self) -> anyhow::Result { - let (id,): (i32,) = sqlx::query_as("SELECT nextval('torrents_id')") + let (id,): (i32,) = sqlx::query_as("SELECT nextval('torrents_id')::int") .fetch_one(&self.pool) - .await?; + .await + .context("error executing SELECT nextval")?; Ok(id as usize) } async fn store(&self, id: TorrentId, torrent: &ManagedTorrentHandle) -> anyhow::Result<()> { let torrent_bytes: &[u8] = &torrent.info().torrent_bytes; - let q = " - INSERT INTO torrents (id, info_hash, torrent_bytes, trackers, output_folder, only_files, is_paused) - VALUES(?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(id) DO NOTHING - "; + let q = "INSERT INTO torrents (id, info_hash, torrent_bytes, trackers, output_folder, only_files, is_paused) + VALUES($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT(id) DO NOTHING"; sqlx::query(q) .bind::(id.try_into()?) - .bind(torrent.info_hash().as_string()) + .bind(&torrent.info_hash().0[..]) .bind(torrent_bytes) .bind(torrent.info().trackers.iter().cloned().collect::>()) .bind( @@ -108,15 +110,17 @@ impl SessionPersistenceStore for PostgresSessionStorage { })) .bind(torrent.is_paused()) .execute(&self.pool) - .await?; + .await + .context("error executing INSERT INTO torrents")?; Ok(()) } async fn delete(&self, id: TorrentId) -> anyhow::Result<()> { - sqlx::query("DELETE * FROM torrents WHERE id = ?") + sqlx::query("DELETE FROM torrents WHERE id = $1") .bind::(id.try_into()?) .execute(&self.pool) - .await?; + .await + .context("error executing DELETE FROM torrents")?; Ok(()) } @@ -124,7 +128,8 @@ impl SessionPersistenceStore for PostgresSessionStorage { let row = sqlx::query_as::<_, TorrentsTableRecord>("SELECT * FROM torrents WHERE id = ?") .bind::(id.try_into()?) .fetch_one(&self.pool) - .await?; + .await + .context("error executing SELECT * FROM torrents")?; row.into_serialized_torrent() .context("bug") .map(|(_, st)| st) @@ -135,7 +140,7 @@ impl SessionPersistenceStore for PostgresSessionStorage { id: TorrentId, torrent: &ManagedTorrentHandle, ) -> anyhow::Result<()> { - sqlx::query("UPDATE torrents SET only_files = ?, paused = ? WHERE id = ?") + sqlx::query("UPDATE torrents SET only_files = $1, is_paused = $2 WHERE id = $3") .bind(torrent.only_files().map(|v| { v.into_iter() .filter_map(|f| f.try_into().ok()) @@ -144,21 +149,21 @@ impl SessionPersistenceStore for PostgresSessionStorage { .bind(torrent.is_paused()) .bind::(id.try_into()?) .execute(&self.pool) - .await?; + .await + .context("error executing UPDATE torrents")?; Ok(()) } async fn stream_all( &self, ) -> anyhow::Result>> { - let res = futures::stream::iter( - sqlx::query_as::<_, TorrentsTableRecord>("SELECT * FROM torrents") - .fetch_all(&self.pool) - .await? - .into_iter() - .filter_map(TorrentsTableRecord::into_serialized_torrent) - .map(Ok), - ); - Ok(res.boxed()) + let torrents = sqlx::query_as::<_, TorrentsTableRecord>("SELECT * FROM torrents") + .fetch_all(&self.pool) + .await + .context("error executing SELECT * FROM torrents")? + .into_iter() + .filter_map(TorrentsTableRecord::into_serialized_torrent) + .map(Ok); + Ok(futures::stream::iter(torrents).boxed()) } } diff --git a/crates/librqbit_core/src/hash_id.rs b/crates/librqbit_core/src/hash_id.rs index 7d9c4ae..08ae619 100644 --- a/crates/librqbit_core/src/hash_id.rs +++ b/crates/librqbit_core/src/hash_id.rs @@ -14,6 +14,15 @@ impl Id { hex::encode(self.0) } + pub fn from_bytes(b: &[u8]) -> anyhow::Result { + let mut v = [0u8; N]; + if b.len() != N { + anyhow::bail!("buffer length must be {}, but it's {}", N, b.len()); + } + v.copy_from_slice(b); + Ok(Id(v)) + } + pub fn distance(&self, other: &Id) -> Id { let mut xor = [0u8; N]; for (idx, (s, o)) in self diff --git a/crates/rqbit/Cargo.toml b/crates/rqbit/Cargo.toml index b880994..8aec1d7 100644 --- a/crates/rqbit/Cargo.toml +++ b/crates/rqbit/Cargo.toml @@ -12,7 +12,7 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["default-tls", "webui"] +default = ["default-tls", "webui", "postgres"] openssl-vendored = ["openssl/vendored"] tokio-console = ["console-subscriber", "tokio/tracing"] webui = ["librqbit/webui"] @@ -20,6 +20,7 @@ timed_existence = ["librqbit/timed_existence"] default-tls = ["librqbit/default-tls"] rust-tls = ["librqbit/rust-tls"] debug_slow_disk = ["librqbit/storage_middleware"] +postgres = ["librqbit/postgres"] [dependencies] librqbit = { path = "../librqbit", default-features = false, version = "6.0.0" } diff --git a/crates/rqbit/src/main.rs b/crates/rqbit/src/main.rs index e5799b3..19fedb7 100644 --- a/crates/rqbit/src/main.rs +++ b/crates/rqbit/src/main.rs @@ -137,8 +137,8 @@ struct ServerStartOptions { disable_persistence: bool, /// The folder to store session data in. By default uses OS specific folder. - #[arg(long = "persistence-folder")] - persistence_folder: Option, + #[arg(long = "persistence-config")] + persistence_config: Option, } #[derive(Parser)] @@ -393,9 +393,26 @@ async fn async_main(opts: Opts) -> anyhow::Result<()> { SubCommand::Server(server_opts) => match &server_opts.subcommand { ServerSubcommand::Start(start_opts) => { if !start_opts.disable_persistence { - sopts.persistence = Some(SessionPersistenceConfig::Json { - folder: start_opts.persistence_folder.clone().map(PathBuf::from), - }) + if let Some(p) = start_opts.persistence_config.as_ref() { + if p.starts_with("postgres://") { + #[cfg(feature = "postgres")] + { + sopts.persistence = Some(SessionPersistenceConfig::Postgres { + connection_string: p.clone(), + }) + } + #[cfg(not(feature = "postgres"))] + { + anyhow::bail!("rqbit was compiled without postgres support") + } + } else { + sopts.persistence = Some(SessionPersistenceConfig::Json { + folder: Some(p.into()), + }) + } + } else { + sopts.persistence = Some(SessionPersistenceConfig::Json { folder: None }) + } } let session =