Using a CATextLayer to Mask Out From Another CALayer

Question

For whatever reason, I can't get this to work with a CATextLayer. I suspect it's totally obvious, and I can't see the forest for the trees, but what I need to do, is use a CATextLayer to mask a "hole" into a CAGradientLayer (so the effect is a gradient, with text "cut out" of it).

I have this working fine, the other way, but I am coming up snake eyes, trying to mask the text from the gradient.

Here's the code that I'm using (It's a UIButton class, and this is the layoutSubviews() override):

override func layoutSubviews() {
    super.layoutSubviews()
    layer.borderColor = UIColor.clear.cgColor
    if let text = titleLabel?.text,
       var dynFont = titleLabel?.font {
        let minimumFontSizeInPoints = (dynFont.pointSize * 0.5)
        let scalingStep = 0.025
        while dynFont.pointSize >= minimumFontSizeInPoints {
            let calcString = NSAttributedString(string: text, attributes: [.font: dynFont])
            let cropRect = calcString.boundingRect(with: CGSize.init(width: CGFloat.greatestFiniteMagnitude, height: CGFloat.greatestFiniteMagnitude), options: [.usesLineFragmentOrigin, .usesFontLeading], context: nil)
            if bounds.size.width >= cropRect.size.width {
                break
            }
            guard let tempDynFont = UIFont(name: dynFont.fontName, size: dynFont.pointSize - (dynFont.pointSize * scalingStep)) else { break }
            dynFont = tempDynFont
        }
        
        titleLabel?.font = dynFont
    }
    
    if let titleLabel = titleLabel,
       let font = titleLabel.font,
       let text = titleLabel.text {
        let textLayer = CATextLayer()
        textLayer.frame = titleLabel.frame
        textLayer.rasterizationScale = UIScreen.main.scale
        textLayer.contentsScale = UIScreen.main.scale
        textLayer.alignmentMode = .left
        textLayer.fontSize = font.pointSize
        textLayer.font = font
        textLayer.isWrapped = true
        textLayer.truncationMode = .none
        textLayer.string = text
        self.textLayer = textLayer
        titleLabel.textColor = .clear

        let gradient = CAGradientLayer()
        gradient.colors = [gradientStartColor.cgColor, gradientEndColor.cgColor]
        gradient.startPoint = CGPoint(x: 0.5, y: 0)
        gradient.endPoint = CGPoint(x: 0.5, y: 1.0)
        var layerFrame = textLayer.frame

        if !reversed {
            if 0 < layer.borderWidth {
                let outlineLayer = CAShapeLayer()
                outlineLayer.frame = bounds
                outlineLayer.path = UIBezierPath(roundedRect: bounds, cornerRadius: layer.cornerRadius).cgPath
                outlineLayer.lineWidth = layer.borderWidth
                outlineLayer.strokeColor = UIColor.white.cgColor
                outlineLayer.fillColor = UIColor.clear.cgColor
                layerFrame = bounds
                textLayer.masksToBounds = false
                if let compositingFilter = CIFilter(name: "CIAdditionCompositing") {
                    textLayer.compositingFilter = compositingFilter
                    outlineLayer.addSublayer(textLayer)
                }
                layer.mask = outlineLayer
            } else {
                layer.mask = textLayer
            }
        } else {
            let outlineLayer = CAShapeLayer()
            outlineLayer.frame = bounds
            textLayer.foregroundColor = UIColor.white.cgColor
            outlineLayer.backgroundColor = UIColor.white.cgColor
            layerFrame = bounds
            textLayer.masksToBounds = false
            if let compositingFilter = CIFilter(name: "CISourceOutCompositing") {
                outlineLayer.compositingFilter = compositingFilter
                outlineLayer.addSublayer(textLayer)
            }
            layer.mask = outlineLayer
        }
        
        gradient.frame = layerFrame
        layer.addSublayer(gradient)
    }
}

The problem is in this part of the code:

            let outlineLayer = CAShapeLayer()
            outlineLayer.frame = bounds
            textLayer.foregroundColor = UIColor.white.cgColor
            outlineLayer.backgroundColor = UIColor.white.cgColor
            layerFrame = bounds
            textLayer.masksToBounds = false
            if let compositingFilter = CIFilter(name: "CISourceOutCompositing") {
                outlineLayer.compositingFilter = compositingFilter
                outlineLayer.addSublayer(textLayer)
            }
            layer.mask = outlineLayer

The !reversed part works fine. I get a gradient masked through text, and, possibly, an outline.

What I need, is to get the gradient to fill the button, with the text "cut out," so the background shows through.

Like I said, this seems deeply obvious, and I seem to have a block.

Are there any suggestions as to what I might be screwing up?

I could probably break this into a playground, but maybe this is enough.

Thanks!

UPDATE:

Here it is as a playground:

//: A UIKit based Playground for presenting user interface
  
import UIKit
import PlaygroundSupport

@IBDesignable
class Rcvrr_GradientTextMaskButton: UIButton {
    /* ################################################################## */
    /**
     This contains our text
     */
    var textLayer: CALayer?
    
    /* ################################################################## */
    /**
     The starting color for the gradient.
     */
    @IBInspectable var gradientStartColor: UIColor = .white

    /* ################################################################## */
    /**
     The ending color.
     */
    @IBInspectable var gradientEndColor: UIColor = .black

    /* ################################################################## */
    /**
     The angle of the gradient. 0 (default) is top-to-bottom.
     */
    @IBInspectable var gradientAngleInDegrees: CGFloat = 0

    /* ################################################################## */
    /**
     If true, then the label is reversed, so the background is "cut out" of the foreground.
     */
    @IBInspectable var reversed: Bool = false
}

/* ###################################################################################################################################### */
// MARK: Base Class Overrides
/* ###################################################################################################################################### */
extension Rcvrr_GradientTextMaskButton {
    /* ################################################################## */
    /**
     If the button is "standard" (the text is filled with the gradient), then this method takes care of that.
     */
    override func layoutSubviews() {
        super.layoutSubviews()
        layer.borderColor = UIColor.clear.cgColor
        if let text = titleLabel?.text,
           var dynFont = titleLabel?.font {
            let minimumFontSizeInPoints = (dynFont.pointSize * 0.5)
            let scalingStep = 0.025
            while dynFont.pointSize >= minimumFontSizeInPoints {
                let calcString = NSAttributedString(string: text, attributes: [.font: dynFont])
                let cropRect = calcString.boundingRect(with: CGSize.init(width: CGFloat.greatestFiniteMagnitude, height: CGFloat.greatestFiniteMagnitude), options: [.usesLineFragmentOrigin, .usesFontLeading], context: nil)
                if bounds.size.width >= cropRect.size.width {
                    break
                }
                guard let tempDynFont = UIFont(name: dynFont.fontName, size: dynFont.pointSize - (dynFont.pointSize * scalingStep)) else { break }
                dynFont = tempDynFont
            }
            
            titleLabel?.font = dynFont
        }
        
        if let titleLabel = titleLabel,
           let font = titleLabel.font,
           let text = titleLabel.text {
            let textLayer = CATextLayer()
            textLayer.frame = titleLabel.frame
            textLayer.rasterizationScale = UIScreen.main.scale
            textLayer.contentsScale = UIScreen.main.scale
            textLayer.alignmentMode = .left
            textLayer.fontSize = font.pointSize
            textLayer.font = font
            textLayer.isWrapped = true
            textLayer.truncationMode = .none
            textLayer.string = text
            self.textLayer = textLayer
            titleLabel.textColor = .clear

            let gradient = CAGradientLayer()
            gradient.colors = [gradientStartColor.cgColor, gradientEndColor.cgColor]
            gradient.startPoint = CGPoint(x: 0.5, y: 0)
            gradient.endPoint = CGPoint(x: 0.5, y: 1.0)
            var layerFrame = textLayer.frame

            if !reversed {
                if 0 < layer.borderWidth {
                    let outlineLayer = CAShapeLayer()
                    outlineLayer.frame = bounds
                    outlineLayer.path = UIBezierPath(roundedRect: bounds, cornerRadius: layer.cornerRadius).cgPath
                    outlineLayer.lineWidth = layer.borderWidth
                    outlineLayer.strokeColor = UIColor.white.cgColor
                    outlineLayer.fillColor = UIColor.clear.cgColor
                    layerFrame = bounds
                    textLayer.masksToBounds = false
                    if let compositingFilter = CIFilter(name: "CIAdditionCompositing") {
                        textLayer.compositingFilter = compositingFilter
                        outlineLayer.addSublayer(textLayer)
                    }
                    layer.mask = outlineLayer
                } else {
                    layer.mask = textLayer
                }
            } else {
                let outlineLayer = CAShapeLayer()
                outlineLayer.frame = bounds
                textLayer.foregroundColor = UIColor.white.cgColor
                outlineLayer.backgroundColor = UIColor.white.cgColor
                layerFrame = bounds
                textLayer.masksToBounds = false
                if let compositingFilter = CIFilter(name: "CISourceOutCompositing") {
                    outlineLayer.compositingFilter = compositingFilter
                    outlineLayer.addSublayer(textLayer)
                }
                layer.mask = outlineLayer
            }
            
            gradient.frame = layerFrame
            layer.addSublayer(gradient)
        }
    }
}

class MyViewController : UIViewController {
    override func loadView() {
        let view = UIView()
        view.backgroundColor = .yellow

        let button = Rcvrr_GradientTextMaskButton()
        button.frame = CGRect(x: 10, y: 200, width: 300, height: 50)
        button.setTitle("HI", for: .normal)
        button.gradientStartColor = .green
        button.gradientEndColor = .blue
        button.reversed = true
        
        view.addSubview(button)
        self.view = view
    }
}
// Present the view controller in the Live View window
PlaygroundPage.current.liveView = MyViewController()

Answer 1

The immediate problem is that you are putting the gradient layer in front of the layer that is being masked. Your masking is working, but you are covering it up! It is not self.layer you want to mask if you want to see something happening here; it's gradient. Change layer.mask = to gradient.mask = everywhere, and you will see an actual visible result.

You will then probably realize that your mask itself is faulty, but at least you won't just be looking at the unadulterated gradient wondering where the mask went!

Answer 2

I'll probably be doing a bunch more work on it, but here's what's working now (and answers the question).

Thanks @matt!

UPDATE: This project has now been integrated into a published SPM module

//: A UIKit based Playground for presenting user interface
  
import UIKit
import PlaygroundSupport

/* ###################################################################################################################################### */
// MARK: - A Special Button Class That Can Be Filled With A Gradient -
/* ###################################################################################################################################### */
/**
 This class can be displayed with either the text filled with a gradient, or the background filled, and the text "cut out" of it.
 All behavior is the same as any other UIButton.
 
 This allows you to specify a border, which will be included in the gradient fill.
 If the borderWidth value is anything greater than 0, there will be a border, with corners specified by cornerRadius.
 The border will be filled with the gradient, as well as the text.
 
 This is a very, very simple control. I'll probably gussy it up, down the line, but it fills a need, right now.
 */
@IBDesignable
class Rcvrr_GradientTextMaskButton: UIButton {
    /* ################################################################## */
    /**
     This caches the gradient layer.
     */
    private var _gradientLayer: CAGradientLayer?
    
    /* ################################################################## */
    /**
     This caches the mask layer.
     */
    private var _outlineLayer: CAShapeLayer?
    
    /* ################################################################## */
    /**
     The starting color for the gradient.
     */
    @IBInspectable var gradientStartColor: UIColor = .white

    /* ################################################################## */
    /**
     The ending color.
     */
    @IBInspectable var gradientEndColor: UIColor = .black

    /* ################################################################## */
    /**
     The angle of the gradient. 0 (default) is top-to-bottom.
     */
    @IBInspectable var gradientAngleInDegrees: CGFloat = 0

    /* ################################################################## */
    /**
     If true, then the label is reversed, so the background is "cut out" of the foreground.
     */
    @IBInspectable var reversed: Bool = false { didSet { setNeedsLayout() }}
}

/* ###################################################################################################################################### */
// MARK: Computed Properties
/* ###################################################################################################################################### */
extension Rcvrr_GradientTextMaskButton {
    /* ################################################################## */
    /**
     This returns the background gradient layer, rendering it, if necessary.
     */
    var gradientLayer: CALayer? { makeGradientLayer() }
    
    /* ################################################################## */
    /**
     This returns the mask layer, rendering it, if necessary.
     */
    var outlineLayer: CALayer? { makeOutlineLayer() }
}

/* ###################################################################################################################################### */
// MARK: Instance Methods
/* ###################################################################################################################################### */
extension Rcvrr_GradientTextMaskButton {
    /* ################################################################## */
    /**
     This creates the gradient layer, using our specified start and stop colors.
     */
    func makeGradientLayer() -> CALayer? {
        guard nil == _gradientLayer else { return _gradientLayer }
        
        _gradientLayer = CAGradientLayer()
        _gradientLayer?.frame = bounds
        _gradientLayer?.colors = [gradientStartColor.cgColor, gradientEndColor.cgColor]
        _gradientLayer?.startPoint = CGPoint(x: 0.5, y: 0) // .rotated(around: CGPoint(x: 0.5, y: 0.5), byDegrees: gradientAngleInDegrees)
        _gradientLayer?.endPoint = CGPoint(x: 0.5, y: 1.0) // .rotated(around: CGPoint(x: 0.5, y: 0.5), byDegrees: gradientAngleInDegrees)
        
        return _gradientLayer
    }
    
    /* ################################################################## */
    /**
     This uses our text to generate a mask layer.
     */
    func makeOutlineLayer() -> CALayer? {
        guard nil == _outlineLayer else { return _outlineLayer }
        
        if let text = titleLabel?.text,
           var dynFont = titleLabel?.font {
            let minimumFontSizeInPoints = (dynFont.pointSize * 0.5)
            let scalingStep = 0.025
            while dynFont.pointSize >= minimumFontSizeInPoints {
                let calcString = NSAttributedString(string: text, attributes: [.font: dynFont])
                let cropRect = calcString.boundingRect(with: CGSize.init(width: CGFloat.greatestFiniteMagnitude, height: CGFloat.greatestFiniteMagnitude), options: [.usesLineFragmentOrigin, .usesFontLeading], context: nil)
                if bounds.size.width >= cropRect.size.width {
                    break
                }
                guard let tempDynFont = UIFont(name: dynFont.fontName, size: dynFont.pointSize - (dynFont.pointSize * scalingStep)) else { break }
                dynFont = tempDynFont
            }
            
            titleLabel?.font = dynFont
        }
        
        let foreColor = reversed ? UIColor.black.cgColor : UIColor.white.cgColor
        let backColor = reversed ? UIColor.white.cgColor : UIColor.black.cgColor
        
        if let titleLabel = titleLabel,
           let font = titleLabel.font,
           let text = titleLabel.text {
            let textLayer = CATextLayer()
            textLayer.frame = titleLabel.frame
            textLayer.rasterizationScale = UIScreen.main.scale
            textLayer.contentsScale = UIScreen.main.scale
            textLayer.alignmentMode = .left
            textLayer.fontSize = font.pointSize
            textLayer.font = font
            textLayer.isWrapped = true
            textLayer.truncationMode = .none
            textLayer.string = text
            textLayer.foregroundColor = foreColor

            let outlineLayer = CAShapeLayer()
            outlineLayer.frame = bounds
            outlineLayer.strokeColor = foreColor
            outlineLayer.fillColor = backColor

            outlineLayer.path = UIBezierPath(roundedRect: bounds, cornerRadius: layer.cornerRadius).cgPath
            outlineLayer.lineWidth = layer.borderWidth
            
            if let compositingFilter = CIFilter(name: "CIAdditionCompositing") {
                textLayer.compositingFilter = compositingFilter
                outlineLayer.addSublayer(textLayer)
                
                self._outlineLayer = outlineLayer
            }
        }
        
        return _outlineLayer
    }
}

/* ###################################################################################################################################### */
// MARK: Base Class Overrides
/* ###################################################################################################################################### */
extension Rcvrr_GradientTextMaskButton {
    /* ################################################################## */
    /**
     We call this, when it's time to layout the control.
     We subvert the standard rendering, and replace it with our own rendering.
     Some of this comes from [this SO answer](https://stackoverflow.com/questions/42238603/reverse-a-calayer-mask/42238699#42238699)
     */
    override func layoutSubviews() {
        super.layoutSubviews()
        // This sets up the baseline.
        _outlineLayer = nil
        backgroundColor = .clear
        layer.borderColor = UIColor.clear.cgColor
        tintColor = .clear
        titleLabel?.textColor = .clear
        _gradientLayer?.removeFromSuperlayer()
        layer.mask = nil
        
        // Create a mask, and apply that to our background gradient.
        if let gradientLayer = gradientLayer,
           let outlineLayer = outlineLayer,
           let filter = CIFilter(name: "CIMaskToAlpha") {
            layer.addSublayer(gradientLayer)
            let renderedCoreImage = CIImage(image: UIGraphicsImageRenderer(size: bounds.size).image { context in return outlineLayer.render(in: context.cgContext) })
            filter.setValue(renderedCoreImage, forKey: "inputImage")
            if let outputImage = filter.outputImage {
                let coreGraphicsImage = CIContext().createCGImage(outputImage, from: outputImage.extent)
                let maskLayer = CALayer()
                maskLayer.frame = bounds
                maskLayer.contents = coreGraphicsImage
                layer.mask = maskLayer
            }
        }
    }
}

class MyViewController : UIViewController {
    var button1: Rcvrr_GradientTextMaskButton!
    var button2: Rcvrr_GradientTextMaskButton!

    override func loadView() {
        let view = UIView()
        view.backgroundColor = .yellow

        button1 = Rcvrr_GradientTextMaskButton()
        button1.frame = CGRect(x: 10, y: 100, width: 300, height: 50)
        button1.setTitle("HI", for: .normal)
        button1.gradientStartColor = .green
        button1.gradientEndColor = .blue
        button1.reversed = true
        button1.addTarget(self, action: #selector(buttonHit), for: .primaryActionTriggered)

        view.addSubview(button1)

        button2 = Rcvrr_GradientTextMaskButton()
        button2.frame = CGRect(x: 10, y: 200, width: 300, height: 50)
        button2.setTitle("BYE", for: .normal)
        button2.gradientStartColor = .green
        button2.gradientEndColor = .blue
        button2.reversed = false
        button2.addTarget(self, action: #selector(buttonHit), for: .primaryActionTriggered)

        view.addSubview(button2)

        self.view = view
    }
    
    @objc func buttonHit(_ inButton: Rcvrr_GradientTextMaskButton) {
        print("Button is\(inButton.reversed ? "" : " not") reversed.")
        if button1 == inButton {
            print("HI!")
            button2.reversed = !inButton.reversed
        } else {
            print("Bye!")
            button1.reversed = !inButton.reversed
        }
    }
}

// Present the view controller in the Live View window
PlaygroundPage.current.liveView = MyViewController()