From 49e3cfba61b524bfeb7745a99a660755bb53445c Mon Sep 17 00:00:00 2001 From: barneygale Date: Tue, 28 May 2024 00:55:47 +0100 Subject: [PATCH 1/8] GH-89727: Fix `os.fwalk()` recursion error on deep trees Implement `os.fwalk()` using a list as a stack to avoid emitting recursion errors on deeply nested trees. --- Lib/os.py | 31 +++++++++++-------- Lib/test/test_os.py | 2 -- ...4-05-28-00-56-59.gh-issue-89727._bxoL3.rst | 3 ++ 3 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst diff --git a/Lib/os.py b/Lib/os.py index 7661ce68ca3be2..ceb460aa35df84 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -479,16 +479,25 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= try: if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) + stack = [(top, topfd)] + isbytes = isinstance(top, bytes) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) finally: close(topfd) - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + toppath, topfd = stack.pop() + if toppath is None: + if isinstance(topfd, int): + close(topfd) + else: + yield topfd + return scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -514,8 +523,10 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd + else: + stack.append((None, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): + for name in reversed(dirs) if entries is None else zip(dirs, entries): try: if not follow_symlinks: if topdown: @@ -528,17 +539,11 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): except OSError as err: if onerror is not None: onerror(err) - continue - try: + else: + stack.append((None, dirfd)) if follow_symlinks or path.samestat(orig_st, stat(dirfd)): dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + stack.append((dirpath, dirfd)) __all__.append("fwalk") diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 941fa2b2c5c87f..7dc5784820e847 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1687,8 +1687,6 @@ def test_fd_leak(self): # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): diff --git a/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst b/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst new file mode 100644 index 00000000000000..92222bc673350f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-28-00-56-59.gh-issue-89727._bxoL3.rst @@ -0,0 +1,3 @@ +Fix issue with :func:`os.fwalk` where a :exc:`RecursionError` was raised on +deep directory trees by adjusting the implementation to be iterative instead +of recursive. From 409f5ea747ab2074c699e560d4b17309fb5f017d Mon Sep 17 00:00:00 2001 From: barneygale Date: Tue, 28 May 2024 02:35:09 +0100 Subject: [PATCH 2/8] Defer open()ing --- Lib/os.py | 71 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index ceb460aa35df84..420339e122cdc5 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -436,6 +436,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False): if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd: + _fwalk_close = 0 + _fwalk_yield = 1 + _fwalk_walk = 2 + def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None): """Directory tree generator. @@ -473,31 +477,49 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= top = fspath(top) # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. - if not follow_symlinks: + if follow_symlinks: + orig_st = None + else: orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY | O_NONBLOCK, dir_fd=dir_fd) - try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - stack = [(top, topfd)] - isbytes = isinstance(top, bytes) - while stack: - yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) - finally: - close(topfd) + stack = [(_fwalk_walk, (top, dir_fd, None, orig_st))] + isbytes = isinstance(top, bytes) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. - toppath, topfd = stack.pop() - if toppath is None: - if isinstance(topfd, int): - close(topfd) - else: - yield topfd + action, value = stack.pop() + if action == _fwalk_close: + close(value) + return + elif action == _fwalk_yield: + yield value return + assert action == _fwalk_walk + toppath, dirfd, topname, orig_st = value + if topname is None: + # Root directory of the walk: propagate any error from open(). + topfd = open(toppath, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + else: + # Inner directory: call onerror() to handle error from open(). + try: + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if onerror is not None: + onerror(err) + return + stack.append((_fwalk_close, topfd)) + if not follow_symlinks: + if topname is None and not st.S_ISDIR(orig_st.st_mode): + # Root of walk: ignore non-directory. + return + if not path.samestat(orig_st, stat(topfd)): + # Ignore symlink. + return + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -524,26 +546,25 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd else: - stack.append((None, (toppath, dirs, nondirs, topfd))) + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) for name in reversed(dirs) if entries is None else zip(dirs, entries): try: - if not follow_symlinks: + if follow_symlinks: + orig_st = None + else: if topdown: orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) else: assert entries is not None name, entry = name orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY | O_NONBLOCK, dir_fd=topfd) except OSError as err: if onerror is not None: onerror(err) - else: - stack.append((None, dirfd)) - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - stack.append((dirpath, dirfd)) + continue + stack.append((_fwalk_walk, + (path.join(toppath, name), topfd, name, orig_st))) __all__.append("fwalk") From 82606a4fad7d0d0432ea47e45bf5d1aa6b9f8c5c Mon Sep 17 00:00:00 2001 From: barneygale Date: Thu, 30 May 2024 00:37:56 +0100 Subject: [PATCH 3/8] De-duplicate stat()ing --- Lib/os.py | 66 ++++++++++++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 420339e122cdc5..53a184d0429fff 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -475,13 +475,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - if follow_symlinks: - orig_st = None - else: - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - stack = [(_fwalk_walk, (top, dir_fd, None, orig_st))] + stack = [(_fwalk_walk, (True, top, dir_fd, top, None))] isbytes = isinstance(top, bytes) while stack: yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) @@ -499,21 +493,22 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): yield value return assert action == _fwalk_walk - toppath, dirfd, topname, orig_st = value - if topname is None: - # Root directory of the walk: propagate any error from open(). - topfd = open(toppath, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) - else: - # Inner directory: call onerror() to handle error from open(). - try: - topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) - except OSError as err: - if onerror is not None: - onerror(err) - return + isroot, toppath, dirfd, topname, orig_st = value + try: + if orig_st is None and not follow_symlinks: + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if isroot: + raise + if onerror is not None: + onerror(err) + return stack.append((_fwalk_close, topfd)) if not follow_symlinks: - if topname is None and not st.S_ISDIR(orig_st.st_mode): + if isroot and not st.S_ISDIR(orig_st.st_mode): # Root of walk: ignore non-directory. return if not path.samestat(orig_st, stat(topfd)): @@ -548,23 +543,20 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): else: stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - for name in reversed(dirs) if entries is None else zip(dirs, entries): - try: - if follow_symlinks: - orig_st = None - else: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - except OSError as err: - if onerror is not None: - onerror(err) - continue - stack.append((_fwalk_walk, - (path.join(toppath, name), topfd, name, orig_st))) + if entries is None: + for name in reversed(dirs): + args = (False, path.join(toppath, name), topfd, name, None) + stack.append((_fwalk_walk, args)) + else: + for name, entry in zip(dirs, entries): + try: + orig_st = entry.stat(follow_symlinks=False) + except OSError as err: + if onerror is not None: + onerror(err) + continue + args = (False, path.join(toppath, name), topfd, name, orig_st) + stack.append((_fwalk_walk, args)) __all__.append("fwalk") From c9d0134656f6cbe9899b866ec2222a1cab5f3284 Mon Sep 17 00:00:00 2001 From: barneygale Date: Thu, 30 May 2024 00:53:02 +0100 Subject: [PATCH 4/8] Defer `DirEntry.stat()` --- Lib/os.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 53a184d0429fff..1c13a4ee5a0a9a 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -493,12 +493,15 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): yield value return assert action == _fwalk_walk - isroot, toppath, dirfd, topname, orig_st = value + isroot, toppath, dirfd, topname, entry = value try: - if orig_st is None and not follow_symlinks: + if not follow_symlinks: # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. - orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + if entry is None: + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + else: + orig_st = entry.stat(follow_symlinks=False) topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) except OSError as err: if isroot: @@ -544,19 +547,13 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) if entries is None: - for name in reversed(dirs): - args = (False, path.join(toppath, name), topfd, name, None) - stack.append((_fwalk_walk, args)) + stack.extend( + (_fwalk_walk, (False, path.join(toppath, name), topfd, name, None)) + for name in reversed(dirs)) else: - for name, entry in zip(dirs, entries): - try: - orig_st = entry.stat(follow_symlinks=False) - except OSError as err: - if onerror is not None: - onerror(err) - continue - args = (False, path.join(toppath, name), topfd, name, orig_st) - stack.append((_fwalk_walk, args)) + stack.extend( + (_fwalk_walk, (False, path.join(toppath, name), topfd, name, entry)) + for name, entry in zip(dirs, entries)) __all__.append("fwalk") From 5d908c4b611813da4bc01b64f9cc463a76006ba8 Mon Sep 17 00:00:00 2001 From: barneygale Date: Thu, 30 May 2024 00:57:33 +0100 Subject: [PATCH 5/8] Fix tests, line length. --- Lib/os.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 1c13a4ee5a0a9a..20044333fa1f75 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -546,13 +546,17 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): else: stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) + dirs = dirs[::-1] if entries is None: stack.extend( - (_fwalk_walk, (False, path.join(toppath, name), topfd, name, None)) - for name in reversed(dirs)) + (_fwalk_walk, + (False, path.join(toppath, name), topfd, name, None)) + for name in dirs) else: + entries = entries[::-1] stack.extend( - (_fwalk_walk, (False, path.join(toppath, name), topfd, name, entry)) + (_fwalk_walk, + (False, path.join(toppath, name), topfd, name, entry)) for name, entry in zip(dirs, entries)) __all__.append("fwalk") From df60bc7ed0f21179359bf33d9898e8357aac7afb Mon Sep 17 00:00:00 2001 From: barneygale Date: Thu, 30 May 2024 01:06:35 +0100 Subject: [PATCH 6/8] Simplify handling of root dir slightly. --- Lib/os.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 20044333fa1f75..2017a48dcaa4a9 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -439,6 +439,7 @@ def walk(top, topdown=True, onerror=None, followlinks=False): _fwalk_close = 0 _fwalk_yield = 1 _fwalk_walk = 2 + _fwalk_walk_root = 3 def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None): """Directory tree generator. @@ -475,7 +476,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - stack = [(_fwalk_walk, (True, top, dir_fd, top, None))] + stack = [(_fwalk_walk_root, top, dir_fd, top, None)] isbytes = isinstance(top, bytes) while stack: yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) @@ -485,15 +486,14 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. - action, value = stack.pop() + action, *value = stack.pop() if action == _fwalk_close: - close(value) + close(value[0]) return elif action == _fwalk_yield: yield value return - assert action == _fwalk_walk - isroot, toppath, dirfd, topname, entry = value + toppath, dirfd, topname, entry = value try: if not follow_symlinks: # Note: To guard against symlink races, we use the standard @@ -504,14 +504,15 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): orig_st = entry.stat(follow_symlinks=False) topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) except OSError as err: - if isroot: + if action == _fwalk_walk_root: + # Root of walk: propagate error from stat() or open(). raise if onerror is not None: onerror(err) return stack.append((_fwalk_close, topfd)) if not follow_symlinks: - if isroot and not st.S_ISDIR(orig_st.st_mode): + if action == _fwalk_walk_root and not st.S_ISDIR(orig_st.st_mode): # Root of walk: ignore non-directory. return if not path.samestat(orig_st, stat(topfd)): @@ -544,19 +545,17 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd else: - stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) + stack.append((_fwalk_yield, toppath, dirs, nondirs, topfd)) dirs = dirs[::-1] if entries is None: stack.extend( - (_fwalk_walk, - (False, path.join(toppath, name), topfd, name, None)) + (_fwalk_walk, path.join(toppath, name), topfd, name, None) for name in dirs) else: entries = entries[::-1] stack.extend( - (_fwalk_walk, - (False, path.join(toppath, name), topfd, name, entry)) + (_fwalk_walk, path.join(toppath, name), topfd, name, entry) for name, entry in zip(dirs, entries)) __all__.append("fwalk") From e600988926d04ce39a65708799efc24fea2420a2 Mon Sep 17 00:00:00 2001 From: barneygale Date: Thu, 30 May 2024 01:29:45 +0100 Subject: [PATCH 7/8] Simplify some naming, tuple unpacking. --- Lib/os.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 2017a48dcaa4a9..2e4da7b6c04cfa 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -436,10 +436,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False): if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd: - _fwalk_close = 0 - _fwalk_yield = 1 - _fwalk_walk = 2 - _fwalk_walk_root = 3 + _fwalk_root = 0 + _fwalk_inner = 1 + _fwalk_yield = 2 + _fwalk_close = 3 def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None): """Directory tree generator. @@ -476,7 +476,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - stack = [(_fwalk_walk_root, top, dir_fd, top, None)] + stack = [(_fwalk_root, (top, dir_fd, top, None))] isbytes = isinstance(top, bytes) while stack: yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) @@ -486,9 +486,9 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. - action, *value = stack.pop() + action, value = stack.pop() if action == _fwalk_close: - close(value[0]) + close(value) return elif action == _fwalk_yield: yield value @@ -504,7 +504,7 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): orig_st = entry.stat(follow_symlinks=False) topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) except OSError as err: - if action == _fwalk_walk_root: + if action == _fwalk_root: # Root of walk: propagate error from stat() or open(). raise if onerror is not None: @@ -512,7 +512,7 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): return stack.append((_fwalk_close, topfd)) if not follow_symlinks: - if action == _fwalk_walk_root and not st.S_ISDIR(orig_st.st_mode): + if action == _fwalk_root and not st.S_ISDIR(orig_st.st_mode): # Root of walk: ignore non-directory. return if not path.samestat(orig_st, stat(topfd)): @@ -545,17 +545,17 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd else: - stack.append((_fwalk_yield, toppath, dirs, nondirs, topfd)) + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) dirs = dirs[::-1] if entries is None: stack.extend( - (_fwalk_walk, path.join(toppath, name), topfd, name, None) + (_fwalk_inner, (path.join(toppath, name), topfd, name, None)) for name in dirs) else: entries = entries[::-1] stack.extend( - (_fwalk_walk, path.join(toppath, name), topfd, name, entry) + (_fwalk_inner, (path.join(toppath, name), topfd, name, entry)) for name, entry in zip(dirs, entries)) __all__.append("fwalk") From 46a557f2f417b0bc57c58af1b741e87d2f7f11ed Mon Sep 17 00:00:00 2001 From: barneygale Date: Thu, 30 May 2024 02:25:56 +0100 Subject: [PATCH 8/8] Merge similar walk actions. --- Lib/os.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 2e4da7b6c04cfa..f2680860213603 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -436,11 +436,6 @@ def walk(top, topdown=True, onerror=None, followlinks=False): if {open, stat} <= supports_dir_fd and {scandir, stat} <= supports_fd: - _fwalk_root = 0 - _fwalk_inner = 1 - _fwalk_yield = 2 - _fwalk_close = 3 - def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None): """Directory tree generator. @@ -476,11 +471,16 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - stack = [(_fwalk_root, (top, dir_fd, top, None))] + stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] isbytes = isinstance(top, bytes) while stack: yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) + # Each item in the _fwalk() stack is a pair (action, args). + _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) + _fwalk_yield = 1 # args: (toppath, dirnames, filenames, topfd) + _fwalk_close = 2 # args: dirfd + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue @@ -493,7 +493,8 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): elif action == _fwalk_yield: yield value return - toppath, dirfd, topname, entry = value + assert action == _fwalk_walk + isroot, dirfd, toppath, topname, entry = value try: if not follow_symlinks: # Note: To guard against symlink races, we use the standard @@ -504,19 +505,16 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): orig_st = entry.stat(follow_symlinks=False) topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) except OSError as err: - if action == _fwalk_root: - # Root of walk: propagate error from stat() or open(). + if isroot: raise if onerror is not None: onerror(err) return stack.append((_fwalk_close, topfd)) if not follow_symlinks: - if action == _fwalk_root and not st.S_ISDIR(orig_st.st_mode): - # Root of walk: ignore non-directory. + if isroot and not st.S_ISDIR(orig_st.st_mode): return if not path.samestat(orig_st, stat(topfd)): - # Ignore symlink. return scandir_it = scandir(topfd) @@ -547,16 +545,15 @@ def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): else: stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - dirs = dirs[::-1] + toppath = path.join(toppath, toppath[:0]) # Add trailing slash. if entries is None: stack.extend( - (_fwalk_inner, (path.join(toppath, name), topfd, name, None)) - for name in dirs) + (_fwalk_walk, (False, topfd, toppath + name, name, None)) + for name in dirs[::-1]) else: - entries = entries[::-1] stack.extend( - (_fwalk_inner, (path.join(toppath, name), topfd, name, entry)) - for name, entry in zip(dirs, entries)) + (_fwalk_walk, (False, topfd, toppath + name, name, entry)) + for name, entry in zip(dirs[::-1], entries[::-1])) __all__.append("fwalk")