RecyclerView - Custom DefaultItemAnimator problem

Question

I have custom animator for recyclerview items, that change alpha channel from 1 to 0 and 0 to 1

Code:

class CustomAnimator: DefaultItemAnimator() {
lateinit var animator: ObjectAnimator
companion object {
    const val PAYLOAD = "PAYLOAD"
    private val ANIMATE = ItemHolderInfo()
}

private val animators = ConcurrentHashMap<RecyclerView.ViewHolder, Animator>()

override fun canReuseUpdatedViewHolder(viewHolder: RecyclerView.ViewHolder, payloads: MutableList<Any>): Boolean {
    return true
}

override fun recordPreLayoutInformation(state: RecyclerView.State, viewHolder: RecyclerView.ViewHolder, changeFlags: Int, payloads: MutableList<Any>): ItemHolderInfo {
    if (PAYLOAD in payloads)
        return ANIMATE
    return super.recordPreLayoutInformation(state, viewHolder, changeFlags, payloads)
}

override fun animateChange(oldHolder: RecyclerView.ViewHolder, newHolder: RecyclerView.ViewHolder, preInfo: ItemHolderInfo, postInfo: ItemHolderInfo): Boolean {
    if (preInfo === ANIMATE) {
        animator = ObjectAnimator
                .ofFloat(newHolder.itemView, "alpha", 1f, 0f, 1f)
                .setDuration(5000L)
        animator.repeatCount = Animation.INFINITE
        animator.doOnEnd {
            newHolder.itemView.alpha = 1f
            animators.remove(newHolder)
            dispatchAnimationFinished(newHolder)
            if (oldHolder != newHolder) dispatchAnimationFinished(oldHolder)
        }
        animators[newHolder] = animator
        animator.start()
        return false
    }
    animator.cancel()
    return super.animateChange(oldHolder, newHolder, preInfo, postInfo)
}

override fun endAnimation(item: RecyclerView.ViewHolder) {
    super.endAnimation(item)
}

override fun endAnimations() {
    super.endAnimations()
}

}

I accept this animation for added item like this:

 recyclerview.postDelayed({ 
     adapter.notifyItemChanged(index, CustomAnimator.PAYLOAD)
 }, 200).let { }

Problem: When i removed all items from adapter's list and then try to add new item, i got this effect, last animations of all removed items not removing, what i'm doing wrong?

Upd: this happens if the animation hasn't finished yet but the item has been removed