UIButton deinit not being called

Question

I have a custom UIButton and want to remove it after I've pressed it. But somehow deinit isn't called when I try so. I've tried this in an empty project:

class CustomButton: UIButton {
    deinit { print("deinit") }
}

class ViewController: UIViewController {

    var button: CustomButton!

    override func viewDidLoad() {
        super.viewDidLoad()

        button = CustomButton(frame: CGRectZero)
        button.translatesAutoresizingMaskIntoConstraints = false

        let views = ["button": button]

        view.addSubview(button)

        view.addConstraints(NSLayoutConstraint.constraintsWithVisualFormat("H:|-50-[button]", options: [], metrics: nil, views: views))
        view.addConstraints(NSLayoutConstraint.constraintsWithVisualFormat("V:[button]-50-|", options: [], metrics: nil, views: views))

        button.backgroundColor = .redColor()

        button.addTarget(self, action: "pressed:", forControlEvents: .TouchUpInside)

//      button.removeFromSuperview()
//      button = nil
    }

    func pressed(sender: UIButton) {
        button.removeTarget(nil, action: nil, forControlEvents: .AllEvents)
        button.removeFromSuperview()
        button = nil
    }
}

If I remove it directly in viewDidLoad(), deinit will be called, but whenever I use the code as shown above, nothing will show. How can I make it call deinit? Does it mean it's not being deallocated or is deinit simply just not being called?