How to use GridSearchCV to do hyperparameter tuning with a custom estimator and custom cross validation?

Question

I am learning how to apply GRU to a time series data but found unable to solve this problem on my own.

import time

import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime
from datetime import timedelta, date
import math
import sklearn

from sklearn import model_selection
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import cross_val_score, TimeSeriesSplit, GridSearchCV

import torch
import pytorchtools
import torch.nn as nn
from torch.nn import MSELoss
from torch.utils.data import TensorDataset, DataLoader
import torch.optim as optim
import torch.nn.functional as F

class EncoderGRU(nn.Module):
    def __init__(self, encode_dim, output_dim, hidden_dim, n_layers, epoch, batch_size, **kwargs):
        super(EncoderGRU, self).__init__()
        self.lr = kwargs.get('lr', False)
        self.dropout = kwargs.get('dropout', False)
        self.epoch = epoch
        self.batch_size = batch_size
        # Below is a GRU model
        self.fc = GRUNet(encode_dim, hidden_dim, output_dim, n_layers, dropout)

    def forward(self, x, h):
        return self.fc(x, h)

    def dataloader(self, train_x, train_y):
        features = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))
        data_Loader = DataLoader(features, batch_size = self.batch_size, shuffle = True)
        return data_Loader

    def get_device():
        if torch.cuda.is_available():
           device = 'cuda:0'
        else:
           device = 'cpu'
        return device

    def fit(self, train_x, train_y):
        device = get_device()
        self.to(device)
        data_Loader = self.dataloader(train_x, train_y)
        loss = nn.MSELoss()
        optimizer = optim.Adam(self.parameters(), lr = self.lr)

        self.train()
        print("Start Training of GRU model")
        epoch_times = []

        for ep in range(1, self.epoch + 1):
            start_time = time.process_time()
            h = self.fc.init_hidden(self.batch_size)
            counter = 0
            avg_loss = 0
        
            for x, y in data_Loader:
                counter += 1
                self.zero_grad()
                out, h = self.fc(x, h)
                l = loss(out, y.to(device).float())
                l.backward(retain_graph = True)
                torch.nn.utils.clip_grad_norm_(self.parameters(), 5)
                optimizer.step()
                avg_loss += l.item()
                if counter%200 == 0:
                   print("Epoch {}......Step: {}/{}....... Average Loss for Epoch: {}".format(ep, counter, len(data_Loader), avg_loss/counter))
            current_time = time.process_time()

            print("Epoch {}/{} Done, Total Loss: {}".format(ep, self.epoch, avg_loss/len(data_Loader)))
            print("Total Time Elapsed: {} seconds".format(str(current_time-start_time)))
                epoch_times.append(current_time-start_time)
    
        print("Total Training Time: {} seconds".format(str(sum(epoch_times))))
        return self

I am a little confused with how to use **kwargs properly. For example, **kwargs contains hyperparameter candidates.

params = {
'lr': (0.001, 0.005, 0.01, 0.015),
'dropout': (0.1, 0.15, 0.2, 0.25)}  

Can I initiate the model with such a command without inputing params?

model = EncoderGRU(input_dim, encode_dim, output_dim, hidden_dim, n_layers, epoch, batch_size)

On the other hand, if there is a custom cross validation tscv for window rolling split to time series data by years. Can I directly use GridSearchCV as below?

finder = GridSearchCV(
    estimator = model,
    param_grid = params,
    scoring = 'accuracy',
    cv = tscv.split(X, y)
)
best_model = finder.fit(X, y)

Currently, the system returns an error saying raise ValueError("dropout should be a number in range [0, 1] " ValueError: dropout should be a number in range [0, 1] representing the probability of an element being zeroed

Answer 1

Your model is a PyTorch model, whereas GridSearchCV expects sklearn estimators - you can only use it with models built using/compatible with sklearn.

You can tune PyTorch models using Optuna. They have code examples for PyTorch at: https://optuna.org/ (go to Code Examples, and select PyTorch).