Retention Policy

README.md

@@ -261,8 +261,7 @@ Note that in the example above, even though the author is persisted first, if an
 
 As this project matures towards release, the project will focus on the functionality and work listed below:
 - Force migration methods
-- Composite indexes (via macros?)
-- Cleaning up old resources on disk
+- Composite indexes
 - Ranged deletes
 - Controls for the edit history
 - Helper types to use with SwiftUI/Observability/Combine that can make data available on the main actor and filter and stay up to date
@@ -271,7 +270,7 @@ As this project matures towards release, the project will focus on the functiona
 - An example app
 - A memory persistence useful for testing apps with
 - A pre-configured data store tuned to storing pure Data, useful for types like Images
-- Cleaning up memory leaks
+- Cleaning up memory and file descriptor leaks
 
 The above list will be kept up to date during development and will likely see additions during that process.
 

Sources/CodableDatastore/Helpers/FileManager+Helpers.swift

@@ -0,0 +1,33 @@
+//
+//  FileManager+Helpers.swift
+//  https://github.com/mochidev/CodableDatastore
+//
+//  Created by Dimitri Bouniol on 2024-09-08.
+//  Copyright © 2023-26 Mochi Development, Inc. All rights reserved.
+//  mochidev-codable-datastore: 8A3D87799CB24B2BA7A7661369B88325
+//
+
+import Foundation
+
+enum DirectoryRemovalError: Error {
+    case missingEnumerator
+}
+
+extension FileManager {
+    @discardableResult
+    func removeDirectoryIfEmpty(url: URL, recursivelyRemoveParents: Bool) throws -> Bool {
+        guard let enumerator = self.enumerator(at: url, includingPropertiesForKeys: [], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants, .skipsPackageDescendants, .includesDirectoriesPostOrder])
+        else { throw DirectoryRemovalError.missingEnumerator }
+
+        for case _ as URL in enumerator {
+            /// If this is called a single time, then we don't have an empty directory, and can stop
+            return false
+        }
+
+        try self.removeItem(at: url)
+
+        guard recursivelyRemoveParents else { return true }
+        try self.removeDirectoryIfEmpty(url: url.deletingLastPathComponent(), recursivelyRemoveParents: recursivelyRemoveParents)
+        return true
+    }
+}

Sources/CodableDatastore/Helpers/Swift6.1+Compatibility.swift

@@ -8,10 +8,45 @@
 //
 
 #if compiler(<6.2)
-extension Task where Failure == Never {
+extension Task {
     @discardableResult
-    init(name: String?, priority: TaskPriority? = nil, operation: sending @escaping @isolated(any) () async -> Success) {
+    init(
+        name: String?,
+        priority: TaskPriority? = nil,
+        @_inheritActorContext @_implicitSelfCapture operation: sending @escaping @isolated(any) () async -> Success
+    ) where Failure == Never {
         self.init(priority: priority, operation: operation)
     }
+
+    @discardableResult
+    init(
+        name: String?,
+        priority: TaskPriority? = nil,
+        @_inheritActorContext @_implicitSelfCapture operation: sending @escaping @isolated(any) () async throws -> Success
+    ) where Failure == any Error {
+        self.init(priority: priority, operation: operation)
+    }
+
+    @discardableResult
+    public static func detached(
+        name: String?,
+        priority: TaskPriority? = nil,
+        operation: sending @escaping @isolated(any) () async -> Success
+    ) -> Task<Success, Failure> where Failure == Never {
+        Task.detached(priority: priority, operation: operation)
+    }
+
+    @discardableResult
+    public static func detached(
+        name: String?,
+        priority: TaskPriority? = nil,
+        operation: sending @escaping @isolated(any) () async throws -> Success
+    ) -> Task<Success, Failure> where Failure == any Error {
+        Task.detached(priority: priority, operation: operation)
+    }
+}
+
+public func extendLifetime<T>(_ x: borrowing T) where T : ~Copyable {
+    withExtendedLifetime(x) {}
 }
 #endif

Sources/CodableDatastore/Helpers/URLCollector.swift

@@ -0,0 +1,30 @@
+//
+//  URLCollector.swift
+//  https://github.com/mochidev/CodableDatastore
+//
+//  Created by Dimitri Bouniol on 2026-09-06.
+//  Copyright © 2023-26 Mochi Development, Inc. All rights reserved.
+//  mochidev-codable-datastore: 8A3D87799CB24B2BA7A7661369B88325
+//
+
+import Foundation
+import QuestionableConcurrency
+
+class URLCollector: @unchecked Sendable {
+    private var gate = UnfairLock()
+    private var urls: Set<URL> = []
+
+    init() {}
+
+    func insertURL(_ url: URL) {
+        gate.withLock {
+            _ = urls.insert(url)
+        }
+    }
+
+    /// Return all URLs, sorted by longest.
+    func removeAllURLs() -> [URL] {
+        let urls = gate.withLock { self.urls }
+        return urls.sorted { $0.absoluteString > $1.absoluteString }
+    }
+}

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreIndexManifest.swift

@@ -91,6 +91,15 @@ extension DatastoreIndexManifest {
     }
 }
 
+extension DatastoreIndexManifest {
+    func pagesToPrune(for mode: SnapshotPruneMode) -> Set<DatastorePageIdentifier> {
+        switch mode {
+        case .pruneRemoved: Set(removedPageIDs)
+        case .pruneAdded:   Set(addedPageIDs)
+        }
+    }
+}
+
 // MARK: - Decoding
 
 extension DatastoreIndexManifest {

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/DatastoreRootManifest.swift

@@ -110,3 +110,25 @@ extension DatastoreRootManifest {
         }
     }
 }
+
+extension DatastoreRootManifest {
+    func indexesToPrune(for mode: SnapshotPruneMode) -> Set<IndexID> {
+        switch mode {
+        case .pruneRemoved: removedIndexes
+        case .pruneAdded:   addedIndexes
+        }
+    }
+
+    func indexManifestsToPrune(
+        for mode: SnapshotPruneMode,
+        options: SnapshotPruneOptions
+    ) -> Set<IndexManifestID> {
+        switch (mode, options) {
+        case (.pruneRemoved, .pruneAndDelete):  removedIndexManifests
+        case (.pruneAdded, .pruneAndDelete):    addedIndexManifests
+        /// Flip the results when we aren't deleting, but only when removing from the bottom end.
+        case (.pruneRemoved, .pruneOnly):       addedIndexManifests
+        case (.pruneAdded, .pruneOnly):         []
+        }
+    }
+}

Sources/CodableDatastore/Persistence/Disk Persistence/Datastore/PersistenceDatastore.swift

@@ -159,6 +159,109 @@ extension DiskPersistence.Datastore {
         }
     }
 
+    func pruneRootObject(with identifier: RootObject.ID, mode: SnapshotPruneMode, shouldDelete: Bool) async throws {
+        let fileManager = FileManager()
+        let rootObject = try loadRootObject(for: identifier, shouldCache: false)
+
+        /// Collect the indexes and related manifests we'll be deleting.
+        /// - For indexes, only collect the ones we'll be deleting since the ones we are keeping won't be making references to other deletable assets.
+        /// - For the manifests, we'll be deleting the entries that are being removed (relative to the direction we are removing from, so the removed ones from the oldest edge, and the added ones from the newest edge, as determined by the caller), while we'll be checking for pages to remove from entries that have just been added, but only when removing from the oldest edge. We only do this for the oldest edge because pages that have been "removed" from the newest edge are actually being _restored_ and not replaced, which maintains symmetry in a non-obvious way.
+        let indexesToPruneAndDelete = rootObject.indexesToPrune(for: mode)
+        let indexManifestsToPruneAndDelete = rootObject.indexManifestsToPrune(for: mode, options: .pruneAndDelete)
+        let indexManifestsToPrune = rootObject.indexManifestsToPrune(for: mode, options: .pruneOnly)
+
+        /// Delete the index manifests and pages we know to be removed.
+        for indexManifestID in indexManifestsToPruneAndDelete {
+            let indexID = Index.ID(indexManifestID)
+            defer {
+                trackedIndexes.removeValue(forKey: indexID)
+                loadedIndexes.remove(indexID)
+            }
+            /// Skip any manifests for indexes being deleted, since we'll just unlink the whole directory in that case.
+            guard !indexesToPruneAndDelete.contains(indexID.indexID) else { continue }
+
+            let manifestURL = manifestURL(for: indexID)
+            let manifest: DatastoreIndexManifest?
+            do {
+                manifest = try await DatastoreIndexManifest(contentsOf: manifestURL, id: indexID.manifestID)
+            } catch FileNotFoundError() {
+                manifest = nil
+            } catch {
+                print("Uncaught Manifest Error: \(error)")
+                throw error
+            }
+
+            guard let manifest else { continue }
+
+            /// Only delete the pages we know to be removed
+            let pagesToPruneAndDelete = manifest.pagesToPrune(for: mode)
+            for pageID in pagesToPruneAndDelete {
+                let indexedPageID = Page.ID(index: indexID, page: pageID)
+                defer {
+                    trackedPages.removeValue(forKey: indexedPageID.withoutManifest)
+                    loadedPages.remove(indexedPageID.withoutManifest)
+                }
+
+                let pageURL = pageURL(for: indexedPageID)
+
+                try? fileManager.removeItem(at: pageURL)
+                snapshot.persistence.directoriesToRemove.insertURL(pageURL.deletingLastPathComponent())
+            }
+
+            try? fileManager.removeItem(at: manifestURL)
+        }
+
+        /// Prune the index manifests that were just added, as they themselves refer to other deleted pages.
+        for indexManifestID in indexManifestsToPrune {
+            let indexID = Index.ID(indexManifestID)
+            /// Skip any manifests for indexes being deleted, since we'll just unlink the whole directory in that case.
+            guard !indexesToPruneAndDelete.contains(indexID.indexID) else { continue }
+
+            let manifestURL = manifestURL(for: indexID)
+            let manifest: DatastoreIndexManifest?
+            do {
+                manifest = try await DatastoreIndexManifest(contentsOf: manifestURL, id: indexID.manifestID)
+            } catch FileNotFoundError() {
+                manifest = nil
+            } catch {
+                print("Uncaught Manifest Error: \(error)")
+                throw error
+            }
+
+            guard let manifest else { continue }
+
+            /// Only delete the pages we know to be removed
+            let pagesToPruneAndDelete = manifest.pagesToPrune(for: mode)
+            for pageID in pagesToPruneAndDelete {
+                let indexedPageID = Page.ID(index: indexID, page: pageID)
+                defer {
+                    trackedPages.removeValue(forKey: indexedPageID.withoutManifest)
+                    loadedPages.remove(indexedPageID.withoutManifest)
+                }
+
+                let pageURL = pageURL(for: indexedPageID)
+
+                try? fileManager.removeItem(at: pageURL)
+                snapshot.persistence.directoriesToRemove.insertURL(pageURL.deletingLastPathComponent())
+            }
+        }
+
+        /// Delete any indexes in their entirety.
+        for indexID in indexesToPruneAndDelete {
+            try? fileManager.removeItem(at: indexURL(for: indexID))
+        }
+
+        /// If we are deleting the root object itself, do so at the very end as everything else would have been cleaned up.
+        if shouldDelete {
+            trackedRootObjects.removeValue(forKey: identifier)
+            loadedRootObjects.remove(identifier)
+
+            let rootURL = rootURL(for: rootObject.id)
+            try? fileManager.removeItem(at: rootURL)
+            snapshot.persistence.directoriesToRemove.insertURL(rootURL.deletingLastPathComponent())
+        }
+    }
+
     func index(for identifier: Index.ID) -> Index {
         if let index = trackedIndexes[identifier]?.value {
             return index
@@ -220,14 +323,16 @@ extension DiskPersistence.Datastore {
 
 extension DiskPersistence.Datastore {
     /// Load the root object from disk for the given identifier.
-    func loadRootObject(for rootIdentifier: DatastoreRootIdentifier) throws -> DatastoreRootManifest {
+    func loadRootObject(for rootIdentifier: DatastoreRootIdentifier, shouldCache: Bool = true) throws -> DatastoreRootManifest {
         let rootObjectURL = rootURL(for: rootIdentifier)
 
         let data = try Data(contentsOf: rootObjectURL)
 
         let root = try JSONDecoder.shared.decode(DatastoreRootManifest.self, from: data)
 
-        cachedRootObject = root
+        if shouldCache {
+            cachedRootObject = root
+        }
         return root
     }