feat: use io_uring / IOCP when available for async file IO (#911)
Spawns a single thread for handling async file IO on the [compio runtime](https://github.com/compio-rs/compio). It is a completion-based IO runtime that can dynamically select a polling mechanism at runtime. It defaults to io_uring on Linux, IOCP on Windows, and the polling crate everywhere else. On Linux systems where io_uring is unavailable or disabled, it will fall back to the polling crate. This eliminates most of the threads that were needed previously. It significantly reduced the amount of memory needed in the recursive Context to get a good transfer rate for each copy operation—from a 4 MB buffer to 128 KB. Copies on a nvme drive are somewhat faster with the async IO changes, and use less CPU than before. Although it uses a single thread for non-blocking tasks, it still manages to 100% max out my nvme drive's activity for the whole duration of multiple long transfers. But it would be possible to enable compio's dispatcher to spread operations across worker threads if necessary. All but the extract and compress operations were updated to be async. I had to switch the `CondVar` in the `Controller` to a `tokio::sync::Notify` to prevent the IO thread from being put to sleep when an operation is paused. Fixed a deadlock in the `operation_copy` test function that was performing an operation without concurrently pulling from the channel in the operation. Reduced the rate that `Message::None` is sent from a subscription to trigger a UI redraw, and fixed it to not run when operations are paused.
This commit is contained in:
Michael Murphy
GitHub
parent
7a657c646b
commit
79aa8f887a
9 changed files with 924 additions and 400 deletions
167
src/app.rs
167
src/app.rs
|
|
@ -51,12 +51,15 @@ use slotmap::Key as SlotMapKey;
|
|||
use std::{
|
||||
any::TypeId,
|
||||
collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque},
|
||||
env, fmt, fs, io,
|
||||
env, fmt, fs,
|
||||
future::Future,
|
||||
io,
|
||||
num::NonZeroU16,
|
||||
path::{Path, PathBuf},
|
||||
pin::Pin,
|
||||
process,
|
||||
sync::{Arc, Mutex},
|
||||
time::{self, Instant},
|
||||
time::{self, Duration, Instant},
|
||||
};
|
||||
use tokio::sync::mpsc;
|
||||
use trash::TrashItem;
|
||||
|
|
@ -539,6 +542,7 @@ pub struct App {
|
|||
config: Config,
|
||||
mode: Mode,
|
||||
app_themes: Vec<String>,
|
||||
compio_tx: mpsc::Sender<Pin<Box<dyn Future<Output = ()> + Send>>>,
|
||||
context_page: ContextPage,
|
||||
dialog_pages: VecDeque<DialogPage>,
|
||||
dialog_text_input: widget::Id,
|
||||
|
|
@ -837,6 +841,7 @@ impl App {
|
|||
self.margin = overlaps;
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
fn open_tab_entity(
|
||||
&mut self,
|
||||
location: Location,
|
||||
|
|
@ -883,14 +888,46 @@ impl App {
|
|||
self.open_tab_entity(location, activate, selection_paths).1
|
||||
}
|
||||
|
||||
fn operation(&mut self, operation: Operation) {
|
||||
#[must_use]
|
||||
fn operation(&mut self, operation: Operation) -> Task<Message> {
|
||||
let id = self.pending_operation_id;
|
||||
let controller = Controller::default();
|
||||
let compio_tx = self.compio_tx.clone();
|
||||
|
||||
self.pending_operation_id += 1;
|
||||
if operation.show_progress_notification() {
|
||||
self.progress_operations.insert(id);
|
||||
}
|
||||
self.pending_operations
|
||||
.insert(id, (operation, Controller::default()));
|
||||
.insert(id, (operation.clone(), controller.clone()));
|
||||
|
||||
// Use a task to send operations to the compio runtime thread.
|
||||
cosmic::Task::stream(cosmic::iced_futures::stream::channel(
|
||||
4,
|
||||
move |msg_tx| async move {
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let msg_tx = Arc::new(tokio::sync::Mutex::new(msg_tx));
|
||||
|
||||
let msg_tx_clone = msg_tx.clone();
|
||||
|
||||
_ = compio_tx
|
||||
.send(Box::pin(async move {
|
||||
let msg = match operation.perform(&msg_tx_clone, controller).await {
|
||||
Ok(result_paths) => Message::PendingComplete(id, result_paths),
|
||||
Err(err) => Message::PendingError(id, err),
|
||||
};
|
||||
|
||||
_ = tx.send(msg);
|
||||
}))
|
||||
.await;
|
||||
|
||||
if let Ok(msg) = rx.await {
|
||||
let _ = msg_tx.lock().await.send(msg).await;
|
||||
}
|
||||
},
|
||||
))
|
||||
.map(cosmic::Action::App)
|
||||
}
|
||||
|
||||
fn remove_window(&mut self, id: &window::Id) {
|
||||
|
|
@ -900,6 +937,7 @@ impl App {
|
|||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
fn rescan_operation_selection(&mut self, op_sel: OperationSelection) -> Task<Message> {
|
||||
log::info!("rescan_operation_selection {:?}", op_sel);
|
||||
let entity = self.tab_model.active();
|
||||
|
|
@ -1789,6 +1827,22 @@ impl Application for App {
|
|||
|
||||
let window_id_opt = core.main_window_id();
|
||||
|
||||
// Create a dedicated thread for the compio runtime to handle operations on.
|
||||
// Supports io_uring on Linux, IOPC on Windows, and polling everywhere else.
|
||||
let (compio_tx, mut compio_rx) = mpsc::channel(1);
|
||||
let tokio_handle = tokio::runtime::Handle::current();
|
||||
std::thread::spawn(move || {
|
||||
let _tokio = tokio_handle.enter();
|
||||
compio::runtime::RuntimeBuilder::new()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async move {
|
||||
while let Some(task) = compio_rx.recv().await {
|
||||
_ = compio::runtime::spawn(task).detach();
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
let mut app = App {
|
||||
core,
|
||||
nav_bar_context_id: segmented_button::Entity::null(),
|
||||
|
|
@ -1798,6 +1852,7 @@ impl Application for App {
|
|||
config: flags.config,
|
||||
mode: flags.mode,
|
||||
app_themes,
|
||||
compio_tx,
|
||||
context_page: ContextPage::Preview(None, PreviewKind::Selected),
|
||||
dialog_pages: VecDeque::new(),
|
||||
dialog_text_input: widget::Id::unique(),
|
||||
|
|
@ -2209,14 +2264,15 @@ impl Application for App {
|
|||
}
|
||||
}
|
||||
if !trash_items.is_empty() {
|
||||
self.operation(Operation::DeleteTrash { items: trash_items });
|
||||
return self
|
||||
.operation(Operation::DeleteTrash { items: trash_items });
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let paths = dbg!(self.selected_paths(entity_opt));
|
||||
if !paths.is_empty() {
|
||||
self.operation(Operation::Delete { paths });
|
||||
return self.operation(Operation::Delete { paths });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2265,15 +2321,15 @@ impl Application for App {
|
|||
let extension = archive_type.extension();
|
||||
let name = format!("{}{}", name, extension);
|
||||
let to = to.join(name);
|
||||
self.operation(Operation::Compress {
|
||||
return self.operation(Operation::Compress {
|
||||
paths,
|
||||
to,
|
||||
archive_type,
|
||||
password,
|
||||
})
|
||||
});
|
||||
}
|
||||
DialogPage::EmptyTrash => {
|
||||
self.operation(Operation::EmptyTrash);
|
||||
return self.operation(Operation::EmptyTrash);
|
||||
}
|
||||
DialogPage::FailedOperation(id) => {
|
||||
log::warn!("TODO: retry operation {}", id);
|
||||
|
|
@ -2288,7 +2344,7 @@ impl Application for App {
|
|||
},
|
||||
_ => unreachable!(),
|
||||
};
|
||||
self.operation(new_op);
|
||||
return self.operation(new_op);
|
||||
}
|
||||
DialogPage::MountError {
|
||||
mounter_key,
|
||||
|
|
@ -2326,7 +2382,7 @@ impl Application for App {
|
|||
}
|
||||
DialogPage::NewItem { parent, name, dir } => {
|
||||
let path = parent.join(name);
|
||||
self.operation(if dir {
|
||||
return self.operation(if dir {
|
||||
Operation::NewFolder { path }
|
||||
} else {
|
||||
Operation::NewFile { path }
|
||||
|
|
@ -2375,13 +2431,13 @@ impl Application for App {
|
|||
from, parent, name, ..
|
||||
} => {
|
||||
let to = parent.join(name);
|
||||
self.operation(Operation::Rename { from, to });
|
||||
return self.operation(Operation::Rename { from, to });
|
||||
}
|
||||
DialogPage::Replace { .. } => {
|
||||
log::warn!("replace dialog should be completed with replace result");
|
||||
}
|
||||
DialogPage::SetExecutableAndLaunch { path } => {
|
||||
self.operation(Operation::SetExecutableAndLaunch { path });
|
||||
return self.operation(Operation::SetExecutableAndLaunch { path });
|
||||
}
|
||||
DialogPage::FavoritePathError { entity, .. } => {
|
||||
if let Some(FavoriteIndex(favorite_i)) =
|
||||
|
|
@ -2417,7 +2473,7 @@ impl Application for App {
|
|||
.and_then(|first| first.parent())
|
||||
.map(|parent| parent.to_path_buf())
|
||||
{
|
||||
self.operation(Operation::Extract {
|
||||
return self.operation(Operation::Extract {
|
||||
paths,
|
||||
to: destination,
|
||||
password: None,
|
||||
|
|
@ -2458,7 +2514,8 @@ impl Application for App {
|
|||
}
|
||||
if let Some(archive_paths) = archive_paths {
|
||||
if !selected_paths.is_empty() {
|
||||
self.operation(Operation::Extract {
|
||||
self.file_dialog_opt = None;
|
||||
return self.operation(Operation::Extract {
|
||||
paths: archive_paths,
|
||||
to: selected_paths[0].clone(),
|
||||
password: None,
|
||||
|
|
@ -2912,20 +2969,16 @@ impl Application for App {
|
|||
Message::PasteContents(to, mut contents) => {
|
||||
contents.paths.retain(|p| p != &to);
|
||||
if !contents.paths.is_empty() {
|
||||
match contents.kind {
|
||||
ClipboardKind::Copy => {
|
||||
self.operation(Operation::Copy {
|
||||
paths: contents.paths,
|
||||
to,
|
||||
});
|
||||
}
|
||||
ClipboardKind::Cut => {
|
||||
self.operation(Operation::Move {
|
||||
paths: contents.paths,
|
||||
to,
|
||||
});
|
||||
}
|
||||
}
|
||||
return match contents.kind {
|
||||
ClipboardKind::Copy => self.operation(Operation::Copy {
|
||||
paths: contents.paths,
|
||||
to,
|
||||
}),
|
||||
ClipboardKind::Cut => self.operation(Operation::Move {
|
||||
paths: contents.paths,
|
||||
to,
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
Message::PendingCancel(id) => {
|
||||
|
|
@ -3182,7 +3235,7 @@ impl Application for App {
|
|||
}
|
||||
}
|
||||
if !trash_items.is_empty() {
|
||||
self.operation(Operation::Restore { items: trash_items });
|
||||
return self.operation(Operation::Restore { items: trash_items });
|
||||
}
|
||||
}
|
||||
Message::ScrollTab(scroll_speed) => {
|
||||
|
|
@ -3353,7 +3406,7 @@ impl Application for App {
|
|||
]));
|
||||
}
|
||||
tab::Command::Delete(paths) => {
|
||||
self.operation(Operation::Delete { paths });
|
||||
commands.push(self.operation(Operation::Delete { paths }))
|
||||
}
|
||||
tab::Command::DropFiles(to, from) => {
|
||||
commands.push(self.update(Message::PasteContents(to, from)));
|
||||
|
|
@ -3497,7 +3550,7 @@ impl Application for App {
|
|||
});
|
||||
}
|
||||
Message::UndoTrashStart(items) => {
|
||||
self.operation(Operation::Restore { items });
|
||||
return self.operation(Operation::Restore { items });
|
||||
}
|
||||
Message::WindowClose => {
|
||||
if let Some(window_id) = self.window_id_opt.take() {
|
||||
|
|
@ -3617,8 +3670,7 @@ impl Application for App {
|
|||
},
|
||||
)),
|
||||
Location::Trash if matches!(action, DndAction::Move) => {
|
||||
self.operation(Operation::Delete { paths: data.paths });
|
||||
Task::none()
|
||||
self.operation(Operation::Delete { paths: data.paths })
|
||||
}
|
||||
_ => {
|
||||
log::warn!("Copy to trash is not supported.");
|
||||
|
|
@ -3678,8 +3730,7 @@ impl Application for App {
|
|||
},
|
||||
)),
|
||||
Location::Trash if matches!(action, DndAction::Move) => {
|
||||
self.operation(Operation::Delete { paths: data.paths });
|
||||
Task::none()
|
||||
self.operation(Operation::Delete { paths: data.paths })
|
||||
}
|
||||
_ => {
|
||||
log::warn!("Copy to trash is not supported.");
|
||||
|
|
@ -5250,8 +5301,17 @@ impl Application for App {
|
|||
//TODO: inhibit suspend/shutdown?
|
||||
|
||||
if self.window_id_opt.is_some() {
|
||||
// Refresh progress when window is open and operations are in progress
|
||||
subscriptions.push(window::frames().map(|_| Message::None));
|
||||
// Force refresh the UI every 100ms while an operation is active.
|
||||
if self
|
||||
.pending_operations
|
||||
.values()
|
||||
.any(|(_, controller)| !controller.is_paused())
|
||||
{
|
||||
subscriptions.push(
|
||||
cosmic::iced::time::every(Duration::from_millis(100))
|
||||
.map(|_| Message::None),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
// Handle notification when window is closed and operations are in progress
|
||||
#[cfg(feature = "notify")]
|
||||
|
|
@ -5293,37 +5353,6 @@ impl Application for App {
|
|||
}
|
||||
}
|
||||
|
||||
for (id, (pending_operation, controller)) in self.pending_operations.iter() {
|
||||
//TODO: use recipe?
|
||||
let id = *id;
|
||||
let pending_operation = pending_operation.clone();
|
||||
let controller = controller.clone();
|
||||
subscriptions.push(Subscription::run_with_id(
|
||||
id,
|
||||
stream::channel(16, move |msg_tx| async move {
|
||||
let msg_tx = Arc::new(tokio::sync::Mutex::new(msg_tx));
|
||||
match pending_operation.perform(&msg_tx, controller).await {
|
||||
Ok(result_paths) => {
|
||||
let _ = msg_tx
|
||||
.lock()
|
||||
.await
|
||||
.send(Message::PendingComplete(id, result_paths))
|
||||
.await;
|
||||
}
|
||||
Err(err) => {
|
||||
let _ = msg_tx
|
||||
.lock()
|
||||
.await
|
||||
.send(Message::PendingError(id, err))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
std::future::pending().await
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
let mut selected_preview = None;
|
||||
if self.core.window.show_context {
|
||||
if let ContextPage::Preview(entity_opt, PreviewKind::Selected) = self.context_page {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue