Custom CGFonts never deallocated when scrolling TableView

Question

I am having an issue with CGFont not being deallocated when used within a TableView, minimal example of what I'm trying to do is at follow:

extension FontPickerPageViewController: UICollectionViewDataSource {

    func collectionView(_ collectionView: UICollectionView, numberOfItemsInSection section: Int) -> Int {
        fontItems.count
    }

    func collectionView(
        _ collectionView: UICollectionView,
        cellForItemAt indexPath: IndexPath
    ) -> UICollectionViewCell {
        let cell = collectionView.dequeueReusableCell(
            withReuseIdentifier: FontPickerCollectionViewCell.reuseIdentifier,
            for: IndexPath(item: indexPath.item, section: 0)
        ) as! FontPickerCollectionViewCell

        let fontItem = fontItems[indexPath.item]
        let name = fontItem.family

        cell.label.text = name

        if let font = try? fontManager.getFont(forName: name) {
            cell.label.font = font
        } else if let fontFile = fontItem.files.first(where: { $0.variant == .regular }),
                  let url = URL(string: fontFile.urlString)
        {
            // Download font
            Task {
                do {
                    let data = try await NetworkManager.downloadData(URLRequest(url: url))
                    try await fontManager.registerFont(for: data, name: name)
                } catch {
                    print(error)
                }
            }
        }

        return cell
    }

}

And the font manager:

final class FontManager {

    enum FontManager2Error: LocalizedError {
        case registerProviderError
        case postScriptNameError
        case fontNotFound

        var errorDescription: String? {
            switch self {
            case .registerProviderError:
                return "Failed to register font with provider"
            case .postScriptNameError:
                return "Failed to get the post script name"
            case .fontNotFound:
                return "Failed to find font"
            }
        }
    }

    // MARK: - Proprties

    static let shared = FontManager()


    private var storage = [String: String]() //fontName : other name
    private let lockQueue = DispatchQueue(label: "lock", attributes: .concurrent)

    // MARK: - Initializers

    private init() {}

    // MARK: - Font Management

    private func registerFont(for data: Data, name: String, completion: @escaping (Result<(), Error>) -> Void) {
        guard let provider = CGDataProvider(data: data as CFData),
              let cgFont = CGFont(provider)
        else {
            return completion(.failure(FontManager2Error.registerProviderError))
        }

        guard let postScriptName = cgFont.postScriptName as? String else {
            return completion(.failure(FontManager2Error.postScriptNameError))
        }

        CTFontManagerRegisterGraphicsFont(cgFont, nil)
        lockQueue.async(flags: .barrier) { [weak self] in
            self?.storage[name] = postScriptName
            completion(.success(()))
        }
    }

    func registerFont(for data: Data, name: String) async throws {
        try await withCheckedThrowingContinuation { continuation in
            registerFont(for: data, name: name) { result in
                switch result {
                case .success:
                    continuation.resume()
                case .failure(let error):
                    continuation.resume(throwing: error)
                }
            }
        }
    }

    func isFontRegistered(name: String) -> Bool {
        lockQueue.sync {
            storage[name] != nil
        }
    }

    func getFont(forName name: String) throws -> UIFont {
        try lockQueue.sync {
            guard let postScriptName = storage[name],
                  let uiFont = UIFont(name: postScriptName, size: Config.defaultFontSize)
            else {
                throw FontManager2Error.fontNotFound
            }

            return uiFont
        }
    }

}

I attempt to retrieve the UIFont from the FontManager, if it doesn't exist then I download the font data and register it in the FontManager. Note that in this minimal example font won't update until the table has been scrolled several times.

The new fonts are set correctly however when I inspect Debug Memory graph I can see there are around 400 CGFont objects in the memory, this corresponds to the total number of fonts I have loaded.

Why are all the fonts in the memory and not just the ones for the displayed table cells? If I use UIFont.systemFont(ofSize: 16.0) for the fonts instead of a custom font then there will only be around 33 CGFont objects at a time in the memory which corresponds to the number of cells displayed on the screen. What am I missing here?