JuMP: Issues of expecting float64, typeError: in typeassert, expected Float64, got ForwardDiff.Dual with autodiff = true and problems with exp()

Question

So I tried to make a minimum example to ask questions based on a more complicated piece of code I have written:

1. A HUGE common error I'm getting is expecting float64 and instead got ForwardDiff.Dual - can someone give me a tip how in general I always make sure I avoid this bug. I feel like every time I do a new optimization problem I have to reinvent the wheel to try to make this go away
2. Apparently you cannot autodiff the julia exp() function? Does anyone know how to make it work?
3. A workaround is I did a finite sum to approximate it via the taylor series. In my one function if I had 20 terms the autodiff worked, but it wasn't accurate enough - so I went to 40 terms but then julia told me to do factorial(big(k)) and then when I try to do that with autodiff it doesn't work now - anyone have a fix for this?

Any advice would be greatly appreciated!

using Cubature
    
    using Juniper
    using Ipopt
    using JuMP
    using LinearAlgebra 
    using Base.Threads
    using Cbc
    using DifferentialEquations
    using Trapz
    function mat_exp(x::AbstractVector{T},dim,num_terms,A) where T
    
        sum = zeros(Complex{T},(dim,dim))
        A[1,1] = A[1,1]*x[1]
        A[2,2] = A[2,2]*x[2]
    
       return exp(A)-1
    end
    
    function exp_approx_no_big(x::AbstractVector{T},dim,num_terms,A) where T
    
        sum = zeros(Complex{T},(dim,dim))
        A[1,1] = A[1,1]*x[1]
        A[2,2] = A[2,2]*x[2]
    
        for k=0:num_terms-1
        
        sum  = sum + (1.0/factorial(k))*A^k
        end
    
        return norm(sum)-1
    end
    function exp_approx_big(x::AbstractVector{T},dim,num_terms,A) where  T
    
        sum = zeros(Complex{T},(dim,dim))
        A[1,1] = A[1,1]*x[1]
        A[2,2] = A[2,2]*x[2]
    
        for k=0:num_terms-1
        
        sum  = sum + (1.0/factorial(big(k)))*A^k
        end
    
        return norm(sum)-1
    
    
    end
    
    
    
    
    optimizer = Juniper.Optimizer
    nl_solver= optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0)
    mip_solver = optimizer_with_attributes(Cbc.Optimizer, "logLevel" => 0, "threads"=>nthreads())
    m = Model(optimizer_with_attributes(optimizer, "nl_solver"=>nl_solver, "mip_solver"=>mip_solver))
    
    @variable(m, 0.0<=x[1:2]<=1.0)
    dim=5
    A=zeros(Complex,(dim,dim))
    for k=1:dim
    A[k,k]=1.0
    end
    println(A)
    
    
    
    f(x...) = exp_approx_no_big(collect(x),dim,20,A)
    g(x...) = exp_approx_big(collect(x),dim,40,A)
    h(x...) = mat_exp(collect(x),dim,20,A)
    register(m, :f, 2, f; autodiff = true)
    @NLobjective(m, Min, f(x...))
    
    
    optimize!(m)
    
    
    println(JuMP.value.(x))
    println(JuMP.objective_value(m))
    println(JuMP.termination_status(m))
    
       

Answer 1

There are quite a few problems with your mat_exp function:

• It modifies A in-place, so repeated calls will not do what you think
• It returns exp(x) - 1, which is a matrix. JuMP only supports scalar calls
• You probably meant norm(exp(x)) - 1
• But ForwardDiff doesn't support differentiating through exp

julia> using ForwardDiff

julia> function mat_exp(x::AbstractVector{T}) where {T}
           A = zeros(Complex{T}, (dim, dim))
           for k = 1:dim
               A[k, k] = one(T)
           end
           A[1, 1] = A[1, 1] * x[1]
           A[2, 2] = A[2, 2] * x[2]
           return norm(exp(A)) - one(T)
       end
mat_exp (generic function with 3 methods)

julia> ForwardDiff.gradient(mat_exp, [0.5, 0.5])
ERROR: MethodError: no method matching exp(::Matrix{Complex{ForwardDiff.Dual{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2}}})
Closest candidates are:
  exp(::StridedMatrix{var"#s832"} where var"#s832"<:Union{Float32, Float64, ComplexF32, ComplexF64}) at /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/LinearAlgebra/src/dense.jl:557
  exp(::StridedMatrix{var"#s832"} where var"#s832"<:Union{Integer, Complex{var"#s831"} where var"#s831"<:Integer}) at /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/LinearAlgebra/src/dense.jl:558
  exp(::Diagonal) at /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/LinearAlgebra/src/diagonal.jl:603
  ...
Stacktrace:
 [1] mat_exp(x::Vector{ForwardDiff.Dual{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2}})
   @ Main ./REPL[34]:8
 [2] vector_mode_dual_eval!(f::typeof(mat_exp), cfg::ForwardDiff.GradientConfig{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2, Vector{ForwardDiff.Dual{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2}}}, x::Vector{Float64})
   @ ForwardDiff ~/.julia/packages/ForwardDiff/jJIvy/src/apiutils.jl:37
 [3] vector_mode_gradient(f::typeof(mat_exp), x::Vector{Float64}, cfg::ForwardDiff.GradientConfig{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2, Vector{ForwardDiff.Dual{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2}}})
   @ ForwardDiff ~/.julia/packages/ForwardDiff/jJIvy/src/gradient.jl:106
 [4] gradient(f::Function, x::Vector{Float64}, cfg::ForwardDiff.GradientConfig{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2, Vector{ForwardDiff.Dual{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2}}}, ::Val{true})
   @ ForwardDiff ~/.julia/packages/ForwardDiff/jJIvy/src/gradient.jl:19
 [5] gradient(f::Function, x::Vector{Float64}, cfg::ForwardDiff.GradientConfig{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2, Vector{ForwardDiff.Dual{ForwardDiff.Tag{typeof(mat_exp), Float64}, Float64, 2}}}) (repeats 2 times)
   @ ForwardDiff ~/.julia/packages/ForwardDiff/jJIvy/src/gradient.jl:17
 [6] top-level scope
   @ REPL[35]:1

I also don't know why you're using Juniper, or that you have a bunch of other packages installed.

If you want to have a discussion on this, come join the community forum: https://discourse.julialang.org/c/domain/opt/13. (It's much better for back-and-forth than stackoverflow.) Someone might have suggestions, but I don't know of an AD tool in Julia that can differentiate through a matrix exponential.