drastically speed up gvfs mount directory listings (#1003)

* feat(gvfs): use gio::enumerate_children to iterate gvfs dirs and split some logic to avoid native fs:: methods

* feat(gvfs): allow thumbnails for GvfsPath + create ItemMetadata::file_size method

* feat(gvfs): fix up details panel + cargo fmt

* fixing panic with really large directories

* cargo fmt
This commit is contained in:
ellieplayswow 2025-06-19 03:39:42 +01:00 • committed by GitHub
parent b0e89cd5e3
commit 1ad5ec0398
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -61,6 +61,7 @@ use std::{
time::{Duration, Instant, SystemTime}, time::{Duration, Instant, SystemTime},
}; };
use tokio::sync::mpsc; use tokio::sync::mpsc;
use trash::{TrashItemMetadata, TrashItemSize};
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::{ use crate::{
@ -505,7 +506,7 @@ fn hidden_attribute(metadata: &Metadata) -> bool {
metadata.file_attributes() & FILE_ATTRIBUTE_HIDDEN == FILE_ATTRIBUTE_HIDDEN metadata.file_attributes() & FILE_ATTRIBUTE_HIDDEN == FILE_ATTRIBUTE_HIDDEN
} }
#[derive(Clone, Copy, Debug)] #[derive(Clone, Copy, Debug, PartialEq)]
pub enum FsKind { pub enum FsKind {
Local, Local,
Remote, Remote,
@ -585,6 +586,115 @@ pub fn parse_desktop_file(path: &Path) -> (Option<String>, Option<String>) {
) )
} }
#[cfg(feature = "gvfs")]
pub fn item_from_gvfs_info(path: PathBuf, file_info: gio::FileInfo, sizes: IconSizes) -> Item {
let file_name = file_info
.attribute_as_string(gio::FILE_ATTRIBUTE_STANDARD_NAME)
.unwrap_or_default();
let mtime = file_info.attribute_uint64(gio::FILE_ATTRIBUTE_TIME_MODIFIED);
let mut display_name = Item::display_name(&file_info.display_name());
let remote = file_info.boolean(gio::FILE_ATTRIBUTE_FILESYSTEM_REMOTE);
let is_dir = match file_info.file_type() {
gio::FileType::Directory => true,
_ => false,
};
let size_opt = match is_dir {
true => None,
false => Some(file_info.size() as u64),
};
let (mime, icon_handle_grid, icon_handle_list, icon_handle_list_condensed) = if is_dir {
(
//TODO: make this a static
"inode/directory".parse().unwrap(),
folder_icon(&path, sizes.grid()),
folder_icon(&path, sizes.list()),
folder_icon(&path, sizes.list_condensed()),
)
} else {
// ALWAYS assume we're remote for mime guessing here, since gvfs reading can be expensive
// @todo - expose this as a config option?
let mime = mime_for_path(&path, None, true);
//TODO: clean this up, implement for trash
let icon_name_opt = if mime == "application/x-desktop" {
let (desktop_name_opt, icon_name_opt) = parse_desktop_file(&path);
if let Some(desktop_name) = desktop_name_opt {
display_name = Item::display_name(&desktop_name);
}
icon_name_opt
} else {
None
};
if let Some(icon_name) = icon_name_opt {
(
mime.clone(),
widget::icon::from_name(&*icon_name)
.size(sizes.grid())
.handle(),
widget::icon::from_name(&*icon_name)
.size(sizes.list())
.handle(),
widget::icon::from_name(&*icon_name)
.size(sizes.list_condensed())
.handle(),
)
} else {
(
mime.clone(),
mime_icon(mime.clone(), sizes.grid()),
mime_icon(mime.clone(), sizes.list()),
mime_icon(mime, sizes.list_condensed()),
)
}
};
let mut children_opt = None;
let mut dir_size = DirSize::NotDirectory;
if is_dir && !remote {
dir_size = DirSize::Calculating(Controller::default());
//TODO: calculate children in the background (and make it cancellable?)
match fs::read_dir(&path) {
Ok(entries) => {
children_opt = Some(entries.count());
}
Err(err) => {
log::warn!("failed to read directory {:?}: {}", path, err);
}
}
}
Item {
name: file_name.clone().to_string(),
display_name,
metadata: ItemMetadata::GvfsPath {
mtime,
size_opt,
children_opt,
},
hidden: file_name.starts_with("."),
location_opt: Some(Location::Path(path)),
mime,
icon_handle_grid,
icon_handle_list,
icon_handle_list_condensed,
thumbnail_opt: if remote {
Some(ItemThumbnail::NotImage)
} else {
None
},
button_id: widget::Id::unique(),
pos_opt: Cell::new(None),
rect_opt: Cell::new(None),
selected: false,
highlighted: false,
overlaps_drag_rect: false,
dir_size,
cut: false,
}
}
pub fn item_from_entry( pub fn item_from_entry(
path: PathBuf, path: PathBuf,
name: String, name: String,
@ -747,48 +857,96 @@ pub fn item_from_path<P: Into<PathBuf>>(path: P, sizes: IconSizes) -> Result<Ite
pub fn scan_path(tab_path: &PathBuf, sizes: IconSizes) -> Vec<Item> { pub fn scan_path(tab_path: &PathBuf, sizes: IconSizes) -> Vec<Item> {
let mut items = Vec::new(); let mut items = Vec::new();
let mut hidden_files = Vec::new(); let mut hidden_files = Vec::new();
match fs::read_dir(tab_path) { let mut remote_scannable = false;
Ok(entries) => {
for entry_res in entries {
let entry = match entry_res {
Ok(ok) => ok,
Err(err) => {
log::warn!("failed to read entry in {:?}: {}", tab_path, err);
continue;
}
};
let path = entry.path(); #[cfg(feature = "gvfs")]
{
if let Ok(path_meta) = fs::metadata(tab_path) {
if fs_kind(&path_meta) == FsKind::Gvfs {
let file = gio::File::for_path(&tab_path);
let name = match entry.file_name().into_string() { // gio crate expects a comma delimited string
Ok(ok) => ok, let mut attr_string = String::new();
Err(name_os) => { for attr in vec![
log::warn!( gio::FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME,
"failed to parse entry at {:?}: {:?} is not valid UTF-8", gio::FILE_ATTRIBUTE_FILESYSTEM_REMOTE,
path, gio::FILE_ATTRIBUTE_TIME_MODIFIED,
name_os, gio::FILE_ATTRIBUTE_STANDARD_SIZE,
); gio::FILE_ATTRIBUTE_STANDARD_TYPE,
continue; gio::FILE_ATTRIBUTE_STANDARD_NAME,
} ] {
}; attr_string.push_str(attr);
attr_string.push(',');
if name == ".hidden" && path.is_file() {
hidden_files = parse_hidden_file(&path);
} }
attr_string.pop();
let metadata = match fs::metadata(&path) { match gio::prelude::FileExt::enumerate_children(
Ok(ok) => ok, &file,
Err(err) => { attr_string.as_str(),
log::warn!("failed to read metadata for entry at {:?}: {}", path, err); gio::FileQueryInfoFlags::NONE,
continue; gio::Cancellable::NONE,
) {
Ok(res) => {
remote_scannable = true;
for file in res {
if let Ok(file) = file {
let full_path = Path::new(tab_path).join(file.name());
items.push(item_from_gvfs_info(full_path, file, sizes));
}
}
} }
}; Err(err) => {
log::warn!("could not enumerate {:?} via gio: {}", tab_path, err);
items.push(item_from_entry(path, name, metadata, sizes)); }
}
} }
} }
Err(err) => { }
log::warn!("failed to read directory {:?}: {}", tab_path, err);
if !remote_scannable {
match fs::read_dir(tab_path) {
Ok(entries) => {
for entry_res in entries {
let entry = match entry_res {
Ok(ok) => ok,
Err(err) => {
log::warn!("failed to read entry in {:?}: {}", tab_path, err);
continue;
}
};
let path = entry.path();
let name = match entry.file_name().into_string() {
Ok(ok) => ok,
Err(name_os) => {
log::warn!(
"failed to parse entry at {:?}: {:?} is not valid UTF-8",
path,
name_os,
);
continue;
}
};
if name == ".hidden" && path.is_file() {
hidden_files = parse_hidden_file(&path);
}
let metadata = match fs::metadata(&path) {
Ok(ok) => ok,
Err(err) => {
log::warn!("failed to read metadata for entry at {:?}: {}", path, err);
continue;
}
};
items.push(item_from_entry(path, name, metadata, sizes));
}
}
Err(err) => {
log::warn!("failed to read directory {:?}: {}", tab_path, err);
}
} }
} }
items.sort_by(|a, b| match (a.metadata.is_dir(), b.metadata.is_dir()) { items.sort_by(|a, b| match (a.metadata.is_dir(), b.metadata.is_dir()) {
@ -1456,6 +1614,12 @@ pub enum ItemMetadata {
SimpleFile { SimpleFile {
size: u64, size: u64,
}, },
#[cfg(feature = "gvfs")]
GvfsPath {
mtime: u64,
size_opt: Option<u64>,
children_opt: Option<usize>,
},
} }
impl ItemMetadata { impl ItemMetadata {
@ -1468,12 +1632,34 @@ impl ItemMetadata {
}, },
Self::SimpleDir { .. } => true, Self::SimpleDir { .. } => true,
Self::SimpleFile { .. } => false, Self::SimpleFile { .. } => false,
#[cfg(feature = "gvfs")]
Self::GvfsPath { children_opt, .. } => children_opt.is_some(),
} }
} }
pub fn modified(&self) -> Option<SystemTime> { pub fn modified(&self) -> Option<SystemTime> {
match self { match self {
Self::Path { metadata, .. } => metadata.modified().ok(), Self::Path { metadata, .. } => metadata.modified().ok(),
#[cfg(feature = "gvfs")]
Self::GvfsPath { mtime, .. } => {
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(*mtime))
}
_ => None,
}
}
pub fn file_size(&self) -> Option<u64> {
match self {
Self::Path { metadata, .. } => match metadata.is_dir() {
true => None,
false => Some(metadata.len()),
},
Self::Trash { metadata, .. } => match metadata.size {
TrashItemSize::Bytes(size) => Some(size),
TrashItemSize::Entries(_) => None,
},
#[cfg(feature = "gvfs")]
Self::GvfsPath { size_opt, .. } => *size_opt,
_ => None, _ => None,
} }
} }
@ -1502,8 +1688,8 @@ impl Clone for ItemThumbnail {
} }
impl ItemThumbnail { impl ItemThumbnail {
pub fn new(path: &Path, metadata: fs::Metadata, mime: mime::Mime, thumbnail_size: u32) -> Self { pub fn new(path: &Path, metadata: ItemMetadata, mime: mime::Mime, thumbnail_size: u32) -> Self {
let size = metadata.len(); let size = metadata.file_size().unwrap_or_default();
let check_size = |thumbnailer: &str, max_size| { let check_size = |thumbnailer: &str, max_size| {
if size <= max_size { if size <= max_size {
true true
@ -1772,133 +1958,145 @@ impl Item {
); );
} }
} }
let mut file_metadata = None;
let mut dir_children_count = None;
match &self.metadata { match &self.metadata {
ItemMetadata::Path { ItemMetadata::Path {
metadata, metadata,
children_opt, children_opt,
} => { } => {
if metadata.is_dir() { file_metadata = Some(metadata.clone());
if let Some(children) = children_opt { dir_children_count = *children_opt;
details = details.push(widget::text::body(fl!("items", items = children))); }
} #[cfg(feature = "gvfs")]
let size = match &self.dir_size { ItemMetadata::GvfsPath { children_opt, .. } => {
DirSize::Calculating(_) => fl!("calculating"), // grab the fs::metadata object for gvfs paths since this is run on-demand
DirSize::Directory(size) => format_size(*size), if let Some(path) = &self.path_opt() {
DirSize::NotDirectory => String::new(), file_metadata = fs::metadata(*path).ok();
DirSize::Error(err) => err.clone(),
};
if !size.is_empty() {
details = details.push(widget::text::body(fl!("item-size", size = size)));
}
} else {
details = details.push(widget::text::body(fl!(
"item-size",
size = format_size(metadata.len())
)));
} }
let date_time_formatter = date_time_formatter(military_time); dir_children_count = *children_opt;
let time_formatter = time_formatter(military_time);
if let Ok(time) = metadata.created() {
details = details.push(widget::text::body(fl!(
"item-created",
created =
format_time(time, &date_time_formatter, &time_formatter).to_string()
)));
}
if let Ok(time) = metadata.modified() {
details = details.push(widget::text::body(fl!(
"item-modified",
modified =
format_time(time, &date_time_formatter, &time_formatter).to_string()
)));
}
if let Ok(time) = metadata.accessed() {
details = details.push(widget::text::body(fl!(
"item-accessed",
accessed =
format_time(time, &date_time_formatter, &time_formatter).to_string()
)));
}
#[cfg(unix)]
if let Some(path) = self.path_opt() {
use std::os::unix::fs::MetadataExt;
let mode = metadata.mode();
let user_name = get_user_by_uid(metadata.uid())
.and_then(|user| user.name().to_str().map(ToOwned::to_owned))
.unwrap_or_default();
let user_path = path.clone();
settings.push(
widget::settings::item::builder(user_name)
.description(fl!("owner"))
.control(widget::dropdown(
Cow::Borrowed(MODE_NAMES.as_slice()),
Some(get_mode_part(mode, MODE_SHIFT_USER).try_into().unwrap()),
move |selected| {
Message::SetPermissions(
user_path.clone(),
set_mode_part(
mode,
MODE_SHIFT_USER,
selected.try_into().unwrap(),
),
)
},
)),
);
let group_name = get_group_by_gid(metadata.gid())
.and_then(|group| group.name().to_str().map(ToOwned::to_owned))
.unwrap_or_default();
let group_path = path.clone();
settings.push(
widget::settings::item::builder(group_name)
.description(fl!("group"))
.control(widget::dropdown(
Cow::Borrowed(MODE_NAMES.as_slice()),
Some(get_mode_part(mode, MODE_SHIFT_GROUP).try_into().unwrap()),
move |selected| {
Message::SetPermissions(
group_path.clone(),
set_mode_part(
mode,
MODE_SHIFT_GROUP,
selected.try_into().unwrap(),
),
)
},
)),
);
let other_path = path.clone();
settings.push(widget::settings::item::builder(fl!("other")).control(
widget::dropdown(
Cow::Borrowed(MODE_NAMES.as_slice()),
Some(get_mode_part(mode, MODE_SHIFT_OTHER).try_into().unwrap()),
move |selected| {
Message::SetPermissions(
other_path.clone(),
set_mode_part(
mode,
MODE_SHIFT_OTHER,
selected.try_into().unwrap(),
),
)
},
),
));
}
} }
_ => { _ => {
//TODO: other metadata types //TODO: other metadata types
} }
} }
if let Some(metadata) = file_metadata {
if metadata.is_dir() {
if let Some(children) = dir_children_count {
details = details.push(widget::text::body(fl!("items", items = children)));
}
let size = match &self.dir_size {
DirSize::Calculating(_) => fl!("calculating"),
DirSize::Directory(size) => format_size(*size),
DirSize::NotDirectory => String::new(),
DirSize::Error(err) => err.clone(),
};
if !size.is_empty() {
details = details.push(widget::text::body(fl!("item-size", size = size)));
}
} else {
details = details.push(widget::text::body(fl!(
"item-size",
size = format_size(metadata.len())
)));
}
let date_time_formatter = date_time_formatter(military_time);
let time_formatter = time_formatter(military_time);
if let Ok(time) = metadata.created() {
details = details.push(widget::text::body(fl!(
"item-created",
created = format_time(time, &date_time_formatter, &time_formatter).to_string()
)));
}
if let Ok(time) = metadata.modified() {
details = details.push(widget::text::body(fl!(
"item-modified",
modified = format_time(time, &date_time_formatter, &time_formatter).to_string()
)));
}
if let Ok(time) = metadata.accessed() {
details = details.push(widget::text::body(fl!(
"item-accessed",
accessed = format_time(time, &date_time_formatter, &time_formatter).to_string()
)));
}
#[cfg(unix)]
if let Some(path) = self.path_opt() {
use std::os::unix::fs::MetadataExt;
let mode = metadata.mode();
let user_name = get_user_by_uid(metadata.uid())
.and_then(|user| user.name().to_str().map(ToOwned::to_owned))
.unwrap_or_default();
let user_path = path.clone();
settings.push(
widget::settings::item::builder(user_name)
.description(fl!("owner"))
.control(widget::dropdown(
Cow::Borrowed(MODE_NAMES.as_slice()),
Some(get_mode_part(mode, MODE_SHIFT_USER).try_into().unwrap()),
move |selected| {
Message::SetPermissions(
user_path.clone(),
set_mode_part(
mode,
MODE_SHIFT_USER,
selected.try_into().unwrap(),
),
)
},
)),
);
let group_name = get_group_by_gid(metadata.gid())
.and_then(|group| group.name().to_str().map(ToOwned::to_owned))
.unwrap_or_default();
let group_path = path.clone();
settings.push(
widget::settings::item::builder(group_name)
.description(fl!("group"))
.control(widget::dropdown(
Cow::Borrowed(MODE_NAMES.as_slice()),
Some(get_mode_part(mode, MODE_SHIFT_GROUP).try_into().unwrap()),
move |selected| {
Message::SetPermissions(
group_path.clone(),
set_mode_part(
mode,
MODE_SHIFT_GROUP,
selected.try_into().unwrap(),
),
)
},
)),
);
let other_path = path.clone();
settings.push(widget::settings::item::builder(fl!("other")).control(
widget::dropdown(
Cow::Borrowed(MODE_NAMES.as_slice()),
Some(get_mode_part(mode, MODE_SHIFT_OTHER).try_into().unwrap()),
move |selected| {
Message::SetPermissions(
other_path.clone(),
set_mode_part(mode, MODE_SHIFT_OTHER, selected.try_into().unwrap()),
)
},
),
));
}
}
if let ItemThumbnail::Image(_, Some((width, height))) = self if let ItemThumbnail::Image(_, Some((width, height))) = self
.thumbnail_opt .thumbnail_opt
.as_ref() .as_ref()
@ -3706,6 +3904,15 @@ impl Tab {
}, },
ItemMetadata::SimpleDir { entries } => (true, *entries), ItemMetadata::SimpleDir { entries } => (true, *entries),
ItemMetadata::SimpleFile { size } => (false, *size), ItemMetadata::SimpleFile { size } => (false, *size),
#[cfg(feature = "gvfs")]
ItemMetadata::GvfsPath {
size_opt,
children_opt,
..
} => match children_opt {
Some(child_count) => (true, *child_count as u64),
None => (false, size_opt.unwrap_or_default()),
},
}; };
let (a_is_entry, a_size) = get_size(a.1); let (a_is_entry, a_size) = get_size(a.1);
let (b_is_entry, b_size) = get_size(b.1); let (b_is_entry, b_size) = get_size(b.1);
@ -4664,7 +4871,7 @@ impl Tab {
let row_height = icon_size + 2 * space_xxs; let row_height = icon_size + 2 * space_xxs;
let mut children: Vec<Element<_>> = Vec::new(); let mut children: Vec<Element<_>> = Vec::new();
let mut y = 0; let mut y: f32 = 0.0;
let rule_padding = theme::active().cosmic().corner_radii.radius_xs[0] as u16; let rule_padding = theme::active().cosmic().corner_radii.radius_xs[0] as u16;
@ -4682,7 +4889,7 @@ impl Tab {
} }
item.pos_opt.set(Some((count, 0))); item.pos_opt.set(Some((count, 0)));
item.rect_opt.set(Some(Rectangle::new( item.rect_opt.set(Some(Rectangle::new(
Point::new(space_m as f32, y as f32), Point::new(space_m as f32, y),
Size::new(size.width - (2 * space_m) as f32, row_height as f32), Size::new(size.width - (2 * space_m) as f32, row_height as f32),
))); )));
@ -4692,7 +4899,7 @@ impl Tab {
.padding([0, rule_padding]) .padding([0, rule_padding])
.into(), .into(),
); );
y += 1; y += 1.0;
} }
let modified_text = match &item.metadata { let modified_text = match &item.metadata {
@ -4707,6 +4914,11 @@ impl Tab {
) )
.map(|t| t.to_string()) .map(|t| t.to_string())
.unwrap_or_default(), .unwrap_or_default(),
#[cfg(feature = "gvfs")]
ItemMetadata::GvfsPath { .. } => match item.metadata.modified() {
Some(mtime) => self.format_time(mtime).to_string(),
None => String::new(),
},
_ => String::new(), _ => String::new(),
}; };
@ -4750,6 +4962,21 @@ impl Tab {
} }
} }
ItemMetadata::SimpleFile { size } => format_size(*size), ItemMetadata::SimpleFile { size } => format_size(*size),
#[cfg(feature = "gvfs")]
ItemMetadata::GvfsPath {
size_opt,
children_opt,
..
} => match children_opt {
Some(child_count) => {
if *child_count == 1 {
format!("{} item", child_count)
} else {
format!("{} items", child_count)
}
}
None => format_size(size_opt.unwrap_or_default()),
},
}; };
let row = if condensed { let row = if condensed {
@ -4933,7 +5160,7 @@ impl Tab {
} }
count += 1; count += 1;
y += row_height; y += row_height as f32;
children.push(button_row); children.push(button_row);
} }
@ -5192,37 +5419,53 @@ impl Tab {
let Some(path) = item.path_opt().map(|path| path.to_path_buf()) else { let Some(path) = item.path_opt().map(|path| path.to_path_buf()) else {
continue; continue;
}; };
let ItemMetadata::Path { metadata, .. } = item.metadata.clone() else {
continue;
};
let mime = item.mime.clone();
subscriptions.push(Subscription::run_with_id( let metadata = item.metadata.clone();
("thumbnail", path.clone()), match metadata {
stream::channel(1, |mut output| async move { ItemMetadata::Path { .. } | ItemMetadata::GvfsPath { .. } => {
let message = { let mime = item.mime.clone();
let path = path.clone();
tokio::task::spawn_blocking(move || {
let start = Instant::now();
let thumbnail =
ItemThumbnail::new(&path, metadata, mime, THUMBNAIL_SIZE);
log::debug!("thumbnailed {:?} in {:?}", path, start.elapsed());
Message::Thumbnail(path.clone(), thumbnail)
})
.await
.unwrap()
};
match output.send(message).await { subscriptions.push(Subscription::run_with_id(
Ok(()) => {} ("thumbnail", path.clone()),
Err(err) => { stream::channel(1, |mut output| async move {
log::warn!("failed to send thumbnail for {:?}: {}", &path, err); let message = {
} let path = path.clone();
} tokio::task::spawn_blocking(move || {
let start = Instant::now();
let thumbnail = ItemThumbnail::new(
&path,
metadata,
mime,
THUMBNAIL_SIZE,
);
log::debug!(
"thumbnailed {:?} in {:?}",
path,
start.elapsed()
);
Message::Thumbnail(path.clone(), thumbnail)
})
.await
.unwrap()
};
std::future::pending().await match output.send(message).await {
}), Ok(()) => {}
)); Err(err) => {
log::warn!(
"failed to send thumbnail for {:?}: {}",
&path,
err
);
}
}
std::future::pending().await
}),
));
}
_ => {}
}
if subscriptions.len() >= jobs { if subscriptions.len() >= jobs {
break; break;