Add parallel removal of items

NOTICE.txt

@@ -95,3 +95,12 @@ This product contains a derivation of the mocking infrastructure from Swift Syst
     * https://www.apache.org/licenses/LICENSE-2.0
   * HOMEPAGE:
     * https://github.com/apple/swift-system
+
+---
+
+This product contains a derivation of "TokenBucket.swift" from Swift Package Manager.
+
+  * LICENSE (Apache License 2.0):
+    * https://www.apache.org/licenses/LICENSE-2.0
+  * HOMEPAGE:
+    * https://github.com/swiftlang/swift-package-manager

Sources/NIOFileSystem/FileSystem.swift

@@ -366,6 +366,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         }
     }
 
+    /// See ``FileSystemProtocol/removeItem(at:strategy:recursively:)``
+    ///
     /// Deletes the file or directory (and its contents) at `path`.
     ///
     /// Only regular files, symbolic links and directories may be removed. If the file at `path` is
@@ -387,11 +389,14 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     ///
     /// - Parameters:
     ///   - path: The path to delete.
+    ///   - removalStrategy: Whether to delete files sequentially (one-by-one), or perform a
+    ///       concurrent scan of the tree at `path` and delete files when they are found.
     ///   - removeItemRecursively: Whether or not to remove items recursively.
     /// - Returns: The number of deleted items which may be zero if `path` did not exist.
     @discardableResult
     public func removeItem(
         at path: FilePath,
+        strategy removalStrategy: RemovalStrategy,
         recursively removeItemRecursively: Bool
     ) async throws -> Int {
         // Try to remove the item: we might just get lucky.
@@ -421,39 +426,60 @@ public struct FileSystem: Sendable, FileSystemProtocol {
                 )
             }
 
-            var (subdirectories, filesRemoved) = try await self.withDirectoryHandle(
-                atPath: path
-            ) { directory in
-                var subdirectories = [FilePath]()
-                var filesRemoved = 0
+            switch removalStrategy.wrapped {
+            case .sequential:
+                return try await self.removeItemSequentially(at: path)
+            case let .parallel(maxDescriptors):
+                return try await self.removeConcurrently(at: path, maxDescriptors)
+            }
 
-                for try await batch in directory.listContents().batched() {
-                    for entry in batch {
-                        switch entry.type {
-                        case .directory:
-                            subdirectories.append(entry.path)
+        case let .failure(errno):
+            throw FileSystemError.remove(errno: errno, path: path, location: .here())
+        }
+    }
 
-                        default:
-                            filesRemoved += try await self.removeOneItem(at: entry.path)
-                        }
+    @discardableResult
+    private func removeItemSequentially(
+        at path: FilePath
+    ) async throws -> Int {
+        var (subdirectories, filesRemoved) = try await self.withDirectoryHandle(
+            atPath: path
+        ) { directory in
+            var subdirectories = [FilePath]()
+            var filesRemoved = 0
+
+            for try await batch in directory.listContents().batched() {
+                for entry in batch {
+                    switch entry.type {
+                    case .directory:
+                        subdirectories.append(entry.path)
+
+                    default:
+                        filesRemoved += try await self.removeOneItem(at: entry.path)
                     }
                 }
-
-                return (subdirectories, filesRemoved)
             }
 
-            for subdirectory in subdirectories {
-                filesRemoved += try await self.removeItem(at: subdirectory)
-            }
+            return (subdirectories, filesRemoved)
+        }
 
-            // The directory should be empty now. Remove ourself.
-            filesRemoved += try await self.removeOneItem(at: path)
+        for subdirectory in subdirectories {
+            filesRemoved += try await self.removeItemSequentially(at: subdirectory)
+        }
 
-            return filesRemoved
+        // The directory should be empty now. Remove ourself.
+        filesRemoved += try await self.removeOneItem(at: path)
 
-        case let .failure(errno):
-            throw FileSystemError.remove(errno: errno, path: path, location: .here())
-        }
+        return filesRemoved
+
+    }
+
+    private func removeConcurrently(
+        at path: FilePath,
+        _ maxDescriptors: Int
+    ) async throws -> Int {
+        let bucket: TokenBucket = .init(tokens: maxDescriptors)
+        return try await self.discoverAndRemoveItemsInTree(at: path, bucket)
     }
 
     /// Moves the named file or directory to a new location.
@@ -490,7 +516,7 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         case .differentLogicalDevices:
             // Fall back to copy and remove.
             try await self.copyItem(at: sourcePath, to: destinationPath)
-            try await self.removeItem(at: sourcePath)
+            try await self.removeItem(at: sourcePath, strategy: .platformDefault)
         }
     }
 
@@ -518,9 +544,9 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         withItemAt existingPath: FilePath
     ) async throws {
         do {
-            try await self.removeItem(at: destinationPath)
+            try await self.removeItem(at: destinationPath, strategy: .platformDefault)
             try await self.moveItem(at: existingPath, to: destinationPath)
-            try await self.removeItem(at: existingPath)
+            try await self.removeItem(at: existingPath, strategy: .platformDefault)
         } catch let error as FileSystemError {
             throw FileSystemError(
                 message: "Can't replace '\(destinationPath)' with '\(existingPath)'.",

Sources/NIOFileSystem/FileSystemProtocol.swift

@@ -249,19 +249,23 @@ public protocol FileSystemProtocol: Sendable {
     /// at the given path then this function returns zero.
     ///
     /// If the item at the `path` is a directory and `removeItemRecursively` is `true` then the
-    /// contents of all of its subdirectories will be removed recursively before the directory
-    /// at `path`. Symbolic links are removed (but their targets are not deleted).
+    /// contents of all of its subdirectories will be removed recursively before the directory at
+    /// `path`. Symbolic links are removed (but their targets are not deleted).
     ///
     /// - Parameters:
     ///   - path: The path to delete.
-    ///   - removeItemRecursively: If the item being removed is a directory, remove it by
-    ///       recursively removing its children. Setting this to `true` is synonymous with
-    ///       calling `rm -r`, setting this false is synonymous to calling `rmdir`. Ignored if
+    ///   - removalStrategy: Whether to delete files sequentially (one-by-one), or perform a
+    ///       concurrent scan of the tree at `path` and delete files when they are found. Ignored if
     ///       the item being removed isn't a directory.
+    ///   - removeItemRecursively: If the item being removed is a directory, remove it by
+    ///       recursively removing its children. Setting this to `true` is synonymous with calling
+    ///       `rm -r`, setting this false is synonymous to calling `rmdir`. Ignored if the item
+    ///       being removed isn't a directory.
     /// - Returns: The number of deleted items which may be zero if `path` did not exist.
     @discardableResult
     func removeItem(
         at path: FilePath,
+        strategy removalStrategy: RemovalStrategy,
         recursively removeItemRecursively: Bool
     ) async throws -> Int
 
@@ -588,9 +592,12 @@ extension FileSystemProtocol {
     /// The item to be removed must be a regular file, symbolic link or directory. If no file exists
     /// at the given path then this function returns zero.
     ///
-    /// If the item at the `path` is a directory then the contents of all of its subdirectories
-    /// will be removed recursively before the directory at `path`. Symbolic links are removed (but
-    /// their targets are not deleted).
+    /// If the item at the `path` is a directory then the contents of all of its subdirectories will
+    /// be removed recursively before the directory at `path`. Symbolic links are removed (but their
+    /// targets are not deleted).
+    ///
+    /// The strategy for deletion will be determined automatically depending on the discovered
+    /// platform.
     ///
     /// - Parameters:
     ///   - path: The path to delete.
@@ -599,7 +606,56 @@ extension FileSystemProtocol {
     public func removeItem(
         at path: FilePath
     ) async throws -> Int {
-        try await self.removeItem(at: path, recursively: true)
+        try await self.removeItem(at: path, strategy: .platformDefault, recursively: true)
+    }
+
+    /// Deletes the file or directory (and its contents) at `path`.
+    ///
+    /// The item to be removed must be a regular file, symbolic link or directory. If no file exists
+    /// at the given path then this function returns zero.
+    ///
+    /// If the item at the `path` is a directory then the contents of all of its subdirectories will
+    /// be removed recursively before the directory at `path`. Symbolic links are removed (but their
+    /// targets are not deleted).
+    ///
+    /// The strategy for deletion will be determined automatically depending on the discovered
+    /// platform.
+    ///
+    /// - Parameters:
+    ///   - path: The path to delete.
+    ///   - removeItemRecursively: If the item being removed is a directory, remove it by
+    ///       recursively removing its children. Setting this to `true` is synonymous with calling
+    ///       `rm -r`, setting this false is synonymous to calling `rmdir`. Ignored if the item
+    ///       being removed isn't a directory.
+    /// - Returns: The number of deleted items which may be zero if `path` did not exist.
+    @discardableResult
+    public func removeItem(
+        at path: FilePath,
+        recursively removeItemRecursively: Bool
+    ) async throws -> Int {
+        try await self.removeItem(at: path, strategy: .platformDefault, recursively: removeItemRecursively)
+    }
+
+    /// Deletes the file or directory (and its contents) at `path`.
+    ///
+    /// The item to be removed must be a regular file, symbolic link or directory. If no file exists
+    /// at the given path then this function returns zero.
+    ///
+    /// If the item at the `path` is a directory then the contents of all of its subdirectories will
+    /// be removed recursively before the directory at `path`. Symbolic links are removed (but their
+    /// targets are not deleted).
+    ///
+    /// - Parameters:
+    ///   - path: The path to delete.
+    ///   - removalStrategy: Whether to delete files sequentially (one-by-one), or perform a
+    ///       concurrent scan of the tree at `path` and delete files when they are found.
+    /// - Returns: The number of deleted items which may be zero if `path` did not exist.
+    @discardableResult
+    public func removeItem(
+        at path: FilePath,
+        strategy removalStrategy: RemovalStrategy
+    ) async throws -> Int {
+        try await self.removeItem(at: path, strategy: removalStrategy, recursively: true)
     }
 
     /// Create a directory at the given path.
@@ -659,7 +715,7 @@ extension FileSystemProtocol {
                 try await execute(handle, directory)
             }
         } tearDown: { _ in
-            try await self.removeItem(at: directory, recursively: true)
+            try await self.removeItem(at: directory, strategy: .platformDefault, recursively: true)
         }
     }
 }