Skip to content

Commit e9ffdbf

Browse files
authored
fix(filesystem): read all files from large directories (#1745)
1 parent a2f0245 commit e9ffdbf

File tree

1 file changed

+40
-16
lines changed

1 file changed

+40
-16
lines changed

lua/neo-tree/sources/filesystem/lib/fs_scan.lua

+40-16
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ local async = require("plenary.async")
1414

1515
local M = {}
1616

17+
--- how many entries to load per readdir
18+
local ENTRIES_BATCH_SIZE = 1000
19+
1720
local on_directory_loaded = function(context, dir_path)
1821
local state = context.state
1922
local scanned_folder = context.folders[dir_path]
@@ -198,23 +201,28 @@ end
198201

199202
local function get_children_sync(path)
200203
local children = {}
201-
local dir, err = uv.fs_opendir(path, nil, 1000)
202-
if err then
204+
local dir, err = uv.fs_opendir(path, nil, ENTRIES_BATCH_SIZE)
205+
if not dir then
206+
---@cast err -nil
203207
if is_permission_error(err) then
204208
log.debug(err)
205209
else
206210
log.error(err)
207211
end
208212
return children
209213
end
210-
---@cast dir uv.luv_dir_t
211-
local stats = uv.fs_readdir(dir)
212-
if stats then
213-
for _, stat in ipairs(stats) do
214+
repeat
215+
local stats = uv.fs_readdir(dir)
216+
if not stats then
217+
break
218+
end
219+
local more = false
220+
for i, stat in ipairs(stats) do
221+
more = i == ENTRIES_BATCH_SIZE
214222
local child_path = utils.path_join(path, stat.name)
215223
table.insert(children, { path = child_path, type = stat.type })
216224
end
217-
end
225+
until not more
218226
uv.fs_closedir(dir)
219227
return children
220228
end
@@ -231,17 +239,27 @@ local function get_children_async(path, callback)
231239
callback(children)
232240
return
233241
end
234-
uv.fs_readdir(dir, function(_, stats)
242+
local readdir_batch
243+
---@param _ string?
244+
---@param stats uv.fs_readdir.entry[]
245+
readdir_batch = function(_, stats)
235246
if stats then
236-
for _, stat in ipairs(stats) do
247+
local more = false
248+
for i, stat in ipairs(stats) do
249+
more = i == ENTRIES_BATCH_SIZE
237250
local child_path = utils.path_join(path, stat.name)
238251
table.insert(children, { path = child_path, type = stat.type })
239252
end
253+
if more then
254+
return uv.fs_readdir(dir, readdir_batch)
255+
end
240256
end
241257
uv.fs_closedir(dir)
242258
callback(children)
243-
end)
244-
end, 1000)
259+
end
260+
261+
uv.fs_readdir(dir, readdir_batch)
262+
end, ENTRIES_BATCH_SIZE)
245263
end
246264

247265
local function scan_dir_sync(context, path)
@@ -399,13 +417,19 @@ local function sync_scan(context, path_to_scan)
399417
end
400418
job_complete(context)
401419
else -- scan_mode == "shallow"
402-
local dir, err = uv.fs_opendir(path_to_scan, nil, 1000)
420+
local dir, err = uv.fs_opendir(path_to_scan, nil, ENTRIES_BATCH_SIZE)
403421
if dir then
404422
local stats = uv.fs_readdir(dir)
405-
if stats then
406-
for _, stat in ipairs(stats) do
423+
repeat
424+
if not stats then
425+
break
426+
end
427+
428+
local more = false
429+
for i, stat in ipairs(stats) do
430+
more = i == ENTRIES_BATCH_SIZE
407431
local path = utils.path_join(path_to_scan, stat.name)
408-
local success, item = pcall(file_items.create_item, context, path, stat.type)
432+
local success, _ = pcall(file_items.create_item, context, path, stat.type)
409433
if success then
410434
if context.recursive and stat.type == "directory" then
411435
table.insert(context.paths_to_load, path)
@@ -414,7 +438,7 @@ local function sync_scan(context, path_to_scan)
414438
log.error("error creating item for ", path)
415439
end
416440
end
417-
end
441+
until not more
418442
uv.fs_closedir(dir)
419443
else
420444
log.error("Error opening dir:", err)

0 commit comments

Comments
 (0)