Julia: Various CuArray errors when trying to run examples from model zoo on GPU

Question

I apologize at first if it turns out that I simply overlooked something. I am very new to Julia programming and am now stuck since a week on making the model zoo examples run on GPU.

With inspiration from https://github.com/FluxML/model-zoo/blob/master/vision/mnist/conv.jl I changed https://github.com/FluxML/model-zoo/blob/master/text/lang-detection/model.jl to this:

using Flux: onehot, onehotbatch, logitcrossentropy, reset!, throttle
using Statistics: mean
using Random
using Unicode
using Parameters: @with_kw
using CUDAapi
if has_cuda()
    @info "CUDA is on"
    import CUDA
    #CUDA.allowscalar(false)
end

@with_kw mutable struct Args
    lr::Float64 = 1e-3     # learning rate
    N::Int = 300            # Number of perceptrons in hidden layer
    test_len::Int = 100    # length of test data
    langs_len::Int = 0     # Number of different languages in Corpora
    alphabet_len::Int = 0  # Total number of characters possible, in corpora
    throttle::Int = 5     # throttle timeout
end

function get_processed_data(args)
    corpora = Dict()

    for file in readdir("corpus")
        lang = Symbol(match(r"(.*)\.txt", file).captures[1])
        corpus = split(String(read("corpus/$file")), ".")
        corpus = strip.(Unicode.normalize.(corpus, casefold=true, stripmark=true))
        corpus = filter(!isempty, corpus)
        corpora[lang] = corpus
    end

    langs = collect(keys(corpora))
    args.langs_len = length(langs)
    alphabet = ['a':'z'; '0':'9'; ' '; '\n'; '_']
    args.alphabet_len = length(alphabet)

    # See which chars will be represented as "unknown"
    unique(filter(x -> x ∉ alphabet, join(vcat(values(corpora)...))))

    dataset = [(onehotbatch(s, alphabet, '_'), onehot(l, langs)) for l in langs for s in corpora[l]] |> shuffle

    train, test = dataset[1:end-args.test_len], dataset[end-args.test_len+1:end]
    return train, test
end

function build_model(args)
    scanner = Chain(Dense(args.alphabet_len, args.N, σ), LSTM(args.N, args.N))
    encoder = Dense(args.N, args.langs_len)
    return scanner, encoder
end

function model(x, scanner, encoder)
    state = scanner.(x.data)[end]
    reset!(scanner)
    encoder(state)
end

function train(; kws...)
    # Initialize Hyperparameters
    args = Args(; kws...)
    # Load Data
    train_data, test_data = get_processed_data(args)

    @info("Constructing Model...")
    scanner, encoder = build_model(args)

    # Load model and datasets onto GPU, if enabled
    train_data = gpu.(train_data)
    test_data = gpu.(test_data)
    scanner = gpu(scanner)
    encoder = gpu(encoder)

    loss(x, y) = logitcrossentropy(model(x, scanner, encoder), y)
    testloss() = mean(loss(t...) for t in test_data)

    opt = ADAM(args.lr)
    ps = params(scanner, encoder)
    evalcb = () -> @show testloss()
    @info("Training...")
    Flux.train!(loss, ps, train_data, opt, cb = throttle(evalcb, args.throttle))
end

cd(@__DIR__)
train()

The result is:

[ Info: CUDA is on
[ Info: Constructing Model...
[ Info: Training...
ERROR: LoadError: CuArray only supports bits types
Stacktrace:
 [1] error(::String) at .\error.jl:33
 [2] CUDA.CuArray{CUDA.CuArray{Float32,1},1}(::UndefInitializer, ::Tuple{Int64}) at C:\Users\Fenmore\.julia\packages\CUDA\dZvbp\src\array.jl:115
 [3] CUDA.CuArray{CUDA.CuArray{Float32,1},N} where N(::UndefInitializer, ::Tuple{Int64}) at C:\Users\Fenmore\.julia\packages\CUDA\dZvbp\src\array.jl:124
 [4] similar(::Type{CUDA.CuArray{CUDA.CuArray{Float32,1},N} where N}, ::Tuple{Int64}) at .\abstractarray.jl:675
 [5] similar(::Type{CUDA.CuArray{CUDA.CuArray{Float32,1},N} where N}, ::Tuple{Base.OneTo{Int64}}) at .\abstractarray.jl:674
 [6] similar(::Base.Broadcast.Broadcasted{CUDA.CuArrayStyle{1},Tuple{Base.OneTo{Int64}},Zygote.var"#1177#1180"{Chain{Tuple{Dense{typeof(σ),CUDA.CuArray{Float32,2},CUDA.CuArray{Float32,1}},Flux.Recur{Flux.LSTMCell{CUDA.CuArray{Float32,2},CUDA.CuArray{Float32,1}}}}}},Tuple{Base.Broadcast.Extruded{CUDA.CuArray{Flux.OneHotVector,1},Tuple{Bool},Tuple{Int64}}}}, ::Type{CUDA.CuArray{Float32,1}}) at C:\Users\Fenmore\.julia\packages\CUDA\dZvbp\src\broadcast.jl:11
 [7] copy at .\broadcast.jl:877 [inlined]
 [8] materialize at .\broadcast.jl:837 [inlined]
 [9] broadcast_forward at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\broadcast.jl:190 [inlined]
 [10] adjoint at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\broadcast.jl:202 [inlined]
 [11] _pullback at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:47 [inlined]
 [12] adjoint at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\lib.jl:175 [inlined]
 [13] _pullback at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:47 [inlined]
 [14] broadcasted at .\broadcast.jl:1257 [inlined]
 [15] model at D:\Users\Fenmore\git\Term-Project\module.jl:55 [inlined]
 [16] _pullback(::Zygote.Context, ::typeof(model), ::Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}}, ::Chain{Tuple{Dense{typeof(σ),CUDA.CuArray{Float32,2},CUDA.CuArray{Float32,1}},Flux.Recur{Flux.LSTMCell{CUDA.CuArray{Float32,2},CUDA.CuArray{Float32,1}}}}}, ::Dense{typeof(identity),CUDA.CuArray{Float32,2},CUDA.CuArray{Float32,1}}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [17] loss at D:\Users\Fenmore\git\Term-Project\module.jl:75 [inlined]
 [18] _pullback(::Zygote.Context, ::var"#loss#35", ::Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}}, ::Flux.OneHotVector) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [19] adjoint at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\lib.jl:175 [inlined]
 [20] _pullback at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:47 [inlined]
 [21] #14 at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\optimise\train.jl:83 [inlined]
 [22] _pullback(::Zygote.Context, ::Flux.Optimise.var"#14#20"{var"#loss#35",Tuple{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},Flux.OneHotVector}}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [23] pullback(::Function, ::Zygote.Params) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface.jl:172
 [24] gradient(::Function, ::Zygote.Params) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface.jl:53
 [25] macro expansion at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\optimise\train.jl:82 [inlined]
 [26] macro expansion at C:\Users\Fenmore\.julia\packages\Juno\hEPx8\src\progress.jl:119 [inlined]
 [27] train!(::Function, ::Zygote.Params, ::Array{Tuple{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},Flux.OneHotVector},1}, ::ADAM; cb::Flux.var"#throttled#42"{Flux.var"#throttled#38#43"{Bool,Bool,var"#34#38"{var"#testloss#36"{var"#loss#35"}},Int64}}) at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\optimise\train.jl:80
 [28] train(; kws::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at D:\Users\Fenmore\git\Term-Project\module.jl:82
 [29] train() at D:\Users\Fenmore\git\Term-Project\module.jl:62
 [30] top-level scope at D:\Users\Fenmore\git\Term-Project\module.jl:86
 [31] include_string(::Function, ::Module, ::String, ::String) at .\loading.jl:1088
in expression starting at D:\Users\Fenmore\git\Term-Project\module.jl:86 

Another model I tried is https://github.com/FluxML/model-zoo/blob/master/text/char-rnn/char-rnn.jl which I changed to:

using Flux
using Flux: onehot, chunk, batchseq, throttle, logitcrossentropy
using StatsBase: wsample
using Base.Iterators: partition
using Parameters: @with_kw

# Hyperparameter arguments
@with_kw mutable struct Args
    lr::Float64 = 1e-2  # Learning rate
    seqlen::Int = 50    # Length of batchseqences
    nbatch::Int = 50    # number of batches text is divided into
    throttle::Int = 30  # Throttle timeout
end

function getdata(args)
    # Download the data if not downloaded as 'input.txt'
    isfile("input.txt") ||
        download("https://cs.stanford.edu/people/karpathy/char-rnn/shakespeare_input.txt","input.txt")

    text = collect(String(read("input.txt")))

    # an array of all unique characters
    alphabet = [unique(text)..., '_']

    text = map(ch -> onehot(ch, alphabet), text)
    stop = onehot('_', alphabet)

    N = length(alphabet)

    # Partitioning the data as sequence of batches, which are then collected as array of batches
    Xs = collect(partition(batchseq(chunk(text, args.nbatch), stop), args.seqlen))
    Ys = collect(partition(batchseq(chunk(text[2:end], args.nbatch), stop), args.seqlen))

    return Xs, Ys, N, alphabet
end

# Function to construct model
function build_model(N)
    return Chain(
            LSTM(N, 128),
            LSTM(128, 128),
            Dense(128, N))
end

function train(; kws...)
    # Initialize the parameters
    args = Args(; kws...)

    # Get Data
    Xs, Ys, N, alphabet = getdata(args)

    # Constructing Model
    m = build_model(N)

    Xs = gpu.(Xs)
    Ys = gpu.(Ys)
    m = gpu(m)

    function loss(xs, ys)
      l = sum(logitcrossentropy.(m.(xs), ys))
      return l
    end

    ## Training
    opt = ADAM(args.lr)
    tx, ty = (Xs[5], Ys[5])
    evalcb = () -> @show loss(tx, ty)

    Flux.train!(loss, params(m), zip(Xs, Ys), opt, cb = throttle(evalcb, args.throttle))
    return m, alphabet
end

# Sampling
function sample(m, alphabet, len; seed="")
    m = cpu(m)
    Flux.reset!(m)
    buf = IOBuffer()
    if seed == ""
        seed = string(rand(alphabet))
    end
    write(buf, seed)
    c = wsample(alphabet, softmax(m.(map(c -> onehot(c, alphabet), collect(seed)))[end]))
    for i = 1:len
        write(buf, c)
        c = wsample(alphabet, softmax(m(onehot(c, alphabet))))
    end
    return String(take!(buf))
end

cd(@__DIR__)
m, alphabet = train()
sample(m, alphabet, 1000) |> println

With this result:

ERROR: LoadError: Mutating arrays is not supported
Stacktrace:
 [1] error(::String) at .\error.jl:33
 [2] (::Zygote.var"#455#456")(::Nothing) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\array.jl:68
 [3] (::Zygote.var"#2384#back#457"{Zygote.var"#455#456"})(::Nothing) at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:49
 [4] CuArray at C:\Users\Fenmore\.julia\packages\CUDA\dZvbp\src\array.jl:206 [inlined]
 [5] CuArray at C:\Users\Fenmore\.julia\packages\CUDA\dZvbp\src\array.jl:211 [inlined]
 [6] (::typeof(∂(CUDA.CuArray{Float32,N} where N)))(::CUDA.CuArray{Float32,2}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [7] LSTMCell at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\cuda\curnn.jl:45 [inlined]
 [8] (::typeof(∂(λ)))(::Tuple{Nothing,CUDA.CuArray{Float32,2}}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [9] (::Zygote.var"#177#178"{typeof(∂(λ)),Tuple{Tuple{Nothing},Tuple{Nothing}}})(::Tuple{Nothing,CUDA.CuArray{Float32,2}}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\lib.jl:178
 [10] (::Zygote.var"#1730#back#179"{Zygote.var"#177#178"{typeof(∂(λ)),Tuple{Tuple{Nothing},Tuple{Nothing}}}})(::Tuple{Nothing,CUDA.CuArray{Float32,2}}) at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:49
 [11] Recur at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\layers\recurrent.jl:36 [inlined]
 [12] (::typeof(∂(λ)))(::CUDA.CuArray{Float32,2}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [13] applychain at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\layers\basic.jl:36 [inlined]
 [14] (::typeof(∂(applychain)))(::CUDA.CuArray{Float32,2}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [15] Chain at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\layers\basic.jl:38 [inlined]
 [16] (::typeof(∂(λ)))(::CUDA.CuArray{Float32,2}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [17] #1157 at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\broadcast.jl:142 [inlined]
 [18] (::Base.var"#3#4"{Zygote.var"#1157#1164"})(::Tuple{typeof(∂(λ)),CUDA.CuArray{Float32,2}}) at .\generator.jl:36
 [19] iterate at .\generator.jl:47 [inlined]
 [20] collect(::Base.Generator{Base.Iterators.Zip{Tuple{Array{typeof(∂(λ)),1},Array{CUDA.CuArray{Float32,2},1}}},Base.var"#3#4"{Zygote.var"#1157#1164"}}) at .\array.jl:686
 [21] map at .\abstractarray.jl:2248 [inlined]
 [22] (::Zygote.var"#1156#1163"{Tuple{Array{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},1}},Val{2},Array{typeof(∂(λ)),1}})(::Array{CUDA.CuArray{Float32,2},1}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\broadcast.jl:142
 [23] #3985#back at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:49 [inlined]
 [24] (::Zygote.var"#177#178"{Zygote.var"#3985#back#1167"{Zygote.var"#1156#1163"{Tuple{Array{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},1}},Val{2},Array{typeof(∂(λ)),1}}},Tuple{Tuple{Nothing,Nothing,Nothing},Tuple{}}})(::Array{CUDA.CuArray{Float32,2},1}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\lib.jl:178
 [25] #1730#back at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:49 [inlined]
 [26] broadcasted at .\broadcast.jl:1257 [inlined]
 [27] (::typeof(∂(broadcasted)))(::Array{CUDA.CuArray{Float32,2},1}) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [28] loss at C:\Users\Fenmore\github\model-zoo\text\char-rnn\char-rnn.jl:60 [inlined]
 [29] (::typeof(∂(λ)))(::Float32) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [30] #177 at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\lib\lib.jl:178 [inlined]
 [31] #1730#back at C:\Users\Fenmore\.julia\packages\ZygoteRules\6nssF\src\adjoint.jl:49 [inlined]
 [32] #14 at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\optimise\train.jl:83 [inlined]
 [33] (::typeof(∂(λ)))(::Float32) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface2.jl:0
 [34] (::Zygote.var"#54#55"{Zygote.Params,Zygote.Context,typeof(∂(λ))})(::Float32) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface.jl:177
 [35] gradient(::Function, ::Zygote.Params) at C:\Users\Fenmore\.julia\packages\Zygote\rqvFi\src\compiler\interface.jl:54
 [36] macro expansion at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\optimise\train.jl:82 [inlined]
 [37] macro expansion at C:\Users\Fenmore\.julia\packages\Juno\hEPx8\src\progress.jl:119 [inlined]
 [38] train!(::Function, ::Zygote.Params, ::Base.Iterators.Zip{Tuple{Array{Array{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},1},1},Array{Array{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},1},1}}}, ::ADAM; cb::Flux.var"#throttled#42"{Flux.var"#throttled#38#43"{Bool,Bool,var"#25#27"{var"#loss#26",Array{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},1},Array{Flux.OneHotMatrix{CUDA.CuArray{Flux.OneHotVector,1}},1}},Int64}}) at C:\Users\Fenmore\.julia\packages\Flux\05b38\src\optimise\train.jl:80
 [39] train(; kws::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at C:\Users\Fenmore\github\model-zoo\text\char-rnn\char-rnn.jl:69
 [40] train() at C:\Users\Fenmore\github\model-zoo\text\char-rnn\char-rnn.jl:47
 [41] top-level scope at C:\Users\Fenmore\github\model-zoo\text\char-rnn\char-rnn.jl:91
 [42] include_string(::Function, ::Module, ::String, ::String) at .\loading.jl:1088
in expression starting at C:\Users\Fenmore\github\model-zoo\text\char-rnn\char-rnn.jl:91

I guess I can't just simply use these gpu() operations, but I have no idea where to start to figure out what else I have to do. I appreciate any help. Thanks in advance. =)