leyoda/cosmic-files
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: use io_uring / IOCP when available for async file IO (#911)

Browse source 
Spawns a single thread for handling async file IO on the [compio runtime](https://github.com/compio-rs/compio).
It is a completion-based IO runtime that can dynamically select a polling mechanism at runtime. It defaults to
io_uring on Linux, IOCP on Windows, and the polling crate everywhere else. On Linux systems where io_uring is
unavailable or disabled, it will fall back to the polling crate.

This eliminates most of the threads that were needed previously. It significantly reduced the amount of memory
needed in the recursive Context to get a good transfer rate for each copy operation—from a 4 MB buffer to 128 KB.
Copies on a nvme drive are somewhat faster with the async IO changes, and use less CPU than before.

Although it uses a single thread for non-blocking tasks, it still manages to 100% max out my nvme drive's
activity for the whole duration of multiple long transfers. But it would be possible to enable compio's
dispatcher to spread operations across worker threads if necessary.

All but the extract and compress operations were updated to be async. I had to switch the `CondVar` in the
`Controller` to a `tokio::sync::Notify` to prevent the IO thread from being put to sleep when an operation is paused.
Fixed a deadlock in the `operation_copy` test function that was performing an operation without concurrently pulling
from the channel in the operation. Reduced the rate that `Message::None` is sent from a subscription to trigger a UI
redraw, and fixed it to not run when operations are paused.
This commit is contained in:
Michael Murphy 2025-04-09 23:15:07 +02:00 • committed by GitHub
parent 7a657c646b
commit 79aa8f887a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 924 additions and 400 deletions
Download patch file Download diff file Expand all files Collapse all files

408
Cargo.lock generated
View file

@ -1,6 +1,6 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
version = 4
[[package]]
name = "ab_glyph"
@ -152,6 +152,15 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
[[package]]
name = "aligned-array"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e05c92d086290f52938013f6242ac62bf7d401fab8ad36798a609faa65c3fd2c"
dependencies = [
"generic-array",
]
[[package]]
name = "aligned-vec"
version = "0.5.0"
@ -345,7 +354,7 @@ dependencies = [
"enumflags2",
"futures-channel",
"futures-util",
"rand",
"rand 0.8.5",
"serde",
"serde_repr",
"tokio",
@ -362,7 +371,7 @@ dependencies = [
"enumflags2",
"futures-channel",
"futures-util",
"rand",
"rand 0.8.5",
"serde",
"serde_repr",
"tokio",
@ -1140,6 +1149,215 @@ dependencies = [
"memchr",
]
[[package]]
name = "compio"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c52183b7eefcaa6441fe810885a34fcdec7378e2883673fc3d74ca6e9ff738b"
dependencies = [
"compio-buf",
"compio-dispatcher",
"compio-driver",
"compio-fs",
"compio-io",
"compio-log",
"compio-macros",
"compio-net",
"compio-process",
"compio-quic",
"compio-runtime",
"compio-signal",
]
[[package]]
name = "compio-buf"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d14413106aad7dd931df3c4724110dabd731c81d52ba18edb4f2d57e7beb611b"
dependencies = [
"arrayvec",
"bytes",
"libc",
]
[[package]]
name = "compio-dispatcher"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ae8fab55190537c8634232f395302011ce39c18facbd4b85363df41114677ac"
dependencies = [
"compio-driver",
"compio-runtime",
"flume",
"futures-channel",
]
[[package]]
name = "compio-driver"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9493b0c88b90d386bb3fd9b5618260d96ba2b09cab5c4a5ba50ec9b77f0711b"
dependencies = [
"aligned-array",
"cfg-if",
"cfg_aliases 0.2.1",
"compio-buf",
"compio-log",
"crossbeam-channel",
"crossbeam-queue",
"futures-util",
"io-uring",
"libc",
"once_cell",
"paste",
"polling 3.7.4",
"socket2 0.5.9",
"windows-sys 0.52.0",
]
[[package]]
name = "compio-fs"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dee2c5ba7c96f0caf3d62ed745278b26eebd4e9296817c4ef2ad6c359629f8ab"
dependencies = [
"cfg-if",
"cfg_aliases 0.2.1",
"compio-buf",
"compio-driver",
"compio-io",
"compio-runtime",
"libc",
"os_pipe",
"widestring",
"windows-sys 0.52.0",
]
[[package]]
name = "compio-io"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c18b1d7d4c058e3e92e9265d59f74981fda2693809b1e45f8ed7717d892c8ac"
dependencies = [
"compio-buf",
"futures-util",
"paste",
]
[[package]]
name = "compio-log"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc4e560213c1996b618da369b7c9109564b41af9033802ae534465c4ee4e132f"
dependencies = [
"tracing",
]
[[package]]
name = "compio-macros"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f05ed201484967dc70de77a8f7a02b29aaa8e6c81cbea2e75492ee0c8d97766b"
dependencies = [
"proc-macro-crate 3.3.0",
"proc-macro2",
"quote",
"syn 2.0.100",
]
[[package]]
name = "compio-net"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0882a85c535c7b5d6ea3b9b37cc7421ec3f8ae8b83a09eb53f4295fb87b54995"
dependencies = [
"cfg-if",
"compio-buf",
"compio-driver",
"compio-io",
"compio-runtime",
"either",
"libc",
"once_cell",
"socket2 0.5.9",
"widestring",
"windows-sys 0.52.0",
]
[[package]]
name = "compio-process"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dc299e4c0a2cdc4455bb4df86c554845d1abe611a1922e4b12a8af2a0fadc35"
dependencies = [
"cfg-if",
"compio-buf",
"compio-driver",
"compio-io",
"compio-runtime",
"futures-util",
"windows-sys 0.52.0",
]
[[package]]
name = "compio-quic"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8853537ade322b0d5dee3dca216465c463f480e530caeabc2b2df15b986068b"
dependencies = [
"cfg_aliases 0.2.1",
"compio-buf",
"compio-io",
"compio-log",
"compio-net",
"compio-runtime",
"flume",
"futures-util",
"libc",
"quinn-proto",
"rustc-hash 2.1.1",
"rustls",
"thiserror 2.0.12",
"windows-sys 0.52.0",
]
[[package]]
name = "compio-runtime"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a95ef126945a166879ef37d494015be13a1e4e452419bc4e5c4c5799f441756a"
dependencies = [
"async-task",
"cfg-if",
"compio-buf",
"compio-driver",
"compio-log",
"crossbeam-queue",
"futures-util",
"libc",
"once_cell",
"scoped-tls",
"slab",
"socket2 0.5.9",
"windows-sys 0.52.0",
]
[[package]]
name = "compio-signal"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd30ba3a28cd73fa49a6e4f1c31c1ad4742fb33802662aadf9ed188ae8a8f0e4"
dependencies = [
"compio-buf",
"compio-driver",
"compio-runtime",
"libc",
"once_cell",
"os_pipe",
"slab",
"windows-sys 0.52.0",
]
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@ -1244,6 +1462,7 @@ version = "0.1.0"
dependencies = [
"bzip2",
"chrono",
"compio",
"cosmic-mime-apps",
"dirs 6.0.0",
"env_logger",
@ -1251,6 +1470,7 @@ dependencies = [
"flate2",
"fork",
"freedesktop_entry_parser",
"futures",
"gio",
"glib",
"glob",
@ -1448,6 +1668,15 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
@ -2060,6 +2289,18 @@ dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "flume"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
dependencies = [
"futures-core",
"futures-sink",
"nanorand",
"spin",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -2342,8 +2583,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
]
[[package]]
@ -3365,6 +3608,17 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "io-uring"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab01638bb6a279897b7691f87f3f3c232451711fd419a69ced980ce61074fa46"
dependencies = [
"bitflags 2.9.0",
"cfg-if",
"libc",
]
[[package]]
name = "is-docker"
version = "0.2.0"
@ -3999,6 +4253,15 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "nanorand"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
dependencies = [
"getrandom 0.2.15",
]
[[package]]
name = "ndk"
version = "0.9.0"
@ -4547,6 +4810,16 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "os_pipe"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982"
dependencies = [
"libc",
"windows-sys 0.59.0",
]
[[package]]
name = "ouroboros"
version = "0.18.5"
@ -4710,7 +4983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
"rand",
"rand 0.8.5",
]
[[package]]
@ -5007,6 +5280,25 @@ dependencies = [
"memchr",
]
[[package]]
name = "quinn-proto"
version = "0.11.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc"
dependencies = [
"bytes",
"getrandom 0.3.2",
"rand 0.9.0",
"ring",
"rustc-hash 2.1.1",
"rustls-pki-types",
"slab",
"thiserror 2.0.12",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quote"
version = "1.0.40"
@ -5029,8 +5321,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
"rand_chacha 0.3.1",
"rand_core 0.6.4",
]
[[package]]
name = "rand"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.3",
"zerocopy 0.8.24",
]
[[package]]
@ -5040,7 +5343,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
"rand_core 0.6.4",
]
[[package]]
name = "rand_chacha"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
"rand_core 0.9.3",
]
[[package]]
@ -5052,6 +5365,15 @@ dependencies = [
"getrandom 0.2.15",
]
[[package]]
name = "rand_core"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
dependencies = [
"getrandom 0.3.2",
]
[[package]]
name = "range-alloc"
version = "0.1.4"
@ -5090,8 +5412,8 @@ dependencies = [
"once_cell",
"paste",
"profiling",
"rand",
"rand_chacha",
"rand 0.8.5",
"rand_chacha 0.3.1",
"simd_helpers",
"system-deps 6.2.2",
"thiserror 1.0.69",
@ -5313,6 +5635,20 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "ring"
version = "0.17.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
"getrandom 0.2.15",
"libc",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
name = "ron"
version = "0.9.0"
@ -5424,6 +5760,39 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "rustls"
version = "0.23.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c"
dependencies = [
"once_cell",
"rustls-pki-types",
"rustls-webpki",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-pki-types"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
dependencies = [
"web-time",
]
[[package]]
name = "rustls-webpki"
version = "0.103.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
[[package]]
name = "rustversion"
version = "1.0.20"
@ -5763,6 +6132,15 @@ dependencies = [
"x11rb",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
dependencies = [
"lock_api",
]
[[package]]
name = "spirv"
version = "0.3.0+sdk-1.3.268.0"
@ -6478,6 +6856,12 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7497808a85e03f612f13e9c5061e4c81cdee86e6c00adfa1096690990ccd08e9"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.4"
@ -7525,7 +7909,7 @@ dependencies = [
[[package]]
name = "xdg-mime"
version = "0.4.0"
source = "git+https://github.com/ellieplayswow/xdg-mime-rs?branch=feature/get-same-as#4f8d07ceedabbe58368a8e7f5547232490860790"
source = "git+https://github.com/ellieplayswow/xdg-mime-rs?branch=feature%2Fget-same-as#4f8d07ceedabbe58368a8e7f5547232490860790"
dependencies = [
"dirs-next",
"glob",
@ -7645,7 +8029,7 @@ dependencies = [
"nix 0.26.4",
"once_cell",
"ordered-stream",
"rand",
"rand 0.8.5",
"serde",
"serde_repr",
"sha1",
@ -7684,7 +8068,7 @@ dependencies = [
"hex",
"nix 0.29.0",
"ordered-stream",
"rand",
"rand 0.8.5",
"serde",
"serde_repr",
"sha1",