Using Keras Tuner for time series split

Question

Is it possible to use Keras tuner for tuning a NN using Time Series Split , similar to sklearn.model_selection.TimeSeriesSplit in sklearn.

For example consider a sample tuner class from https://towardsdatascience.com/hyperparameter-tuning-with-keras-tuner-283474fbfbe

from kerastuner import HyperModel
class SampleModel(HyperModel):
    def __init__(self, input_shape):
        self.input_shape = input_shape
    def build(self, hp):
        model = Sequential()
        model.add(
            layers.Dense(
                units=hp.Int('units', 8, 64, 4, default=8),
                activation=hp.Choice(
                    'dense_activation',
                    values=['relu', 'tanh', 'sigmoid'],
                    default='relu'),
                input_shape=input_shape
            )
        )
    
        model.add(layers.Dense(1))
        
        model.compile(
            optimizer='rmsprop',loss='mse',metrics=['mse']
        )
        
        return model

tuner:

tuner_rs = RandomSearch(
            hypermodel,
            objective='mse',
            seed=42,
            max_trials=10,
            executions_per_trial=2)


tuner_rs.search(x_train_scaled, y_train, epochs=10, validation_split=0.2, verbose=0)

So instead of validation_split = 0.2, in the above line is it possible to do the following

from sklearn.model_selection import TimeSeriesSplit

#defining a time series split object
tscv = TimeSeriesSplit(n_splits = 5)

#using that in Keras Tuner
tuner_rs.search(x_train, y_train, epochs=10, validation_split=tscv, verbose=0)

Answer 1

I solved in this way:

First I have istanciated a class that allows to perform Blocking Time Series Split. I found out that it might be better to use this time series split rather than Sklearn TimeSeriesSplit because we won't make our model train on instances with already seen data. As you can see from the picture, if number of splits is 5, BTSS will divide your training data in 5 parts with only the validation data in common across the splits. (Since StackOverflow doesn't allow me to upload images i'll post a reference link: https://hub.packtpub.com/cross-validation-strategies-for-time-series-forecasting-tutorial/)

class BlockingTimeSeriesSplit():
  def __init__(self, n_splits):
      self.n_splits = n_splits

  def get_n_splits(self, X, y, groups):
      return self.n_splits

  def split(self, X, y=None, groups=None):
      n_samples = len(X)
      k_fold_size = n_samples // self.n_splits
      indices = np.arange(n_samples)

      margin = 0
      for i in range(self.n_splits):
          start = i * k_fold_size
          stop = start + k_fold_size
          mid = int(0.8 * (stop - start)) + start
          yield indices[start: mid], indices[mid + margin: stop]

Then you will proceed by creating your own model:

def build_model(hp):
   pass

Finally you can create your CVtuner as a class which will call back BlockingTimeSeriesSplit.

class CVTuner(kt.engine.tuner.Tuner):
    def run_trial(self, trial, x, y, *args, **kwargs):
        cv = BlockingTimeSeriesSplit(n_splits=5)
        val_accuracy_list = []
        batch_size = trial.hyperparameters.Int('batch_size', 0, 64, step=8)
        epochs = trial.hyperparameters.Int('epochs', 10, 100, step=10)

        for train_indices, test_indices in cv.split(x):
            x_train, x_test = x[train_indices], x[test_indices]
            y_train, y_test = y[train_indices], y[test_indices]
            model = self.hypermodel.build(trial.hyperparameters)
            model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs)
            val_loss, val_accuracy, val_auc = model.evaluate(x_test, y_test)
            val_accuracy_list.append(val_accuracy)
        
            self.oracle.update_trial(trial.trial_id, {'val_accuracy': np.mean(val_accuracy_list)})
            self.save_model(trial.trial_id, model)

  
tuner = CVTuner(oracle=kt.oracles.BayesianOptimization(objective='val_accuracy',max_trials=1), hypermodel=create_model)

stop_early = tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=10)

tuner.search(X, Y, callbacks=[stop_early])

best_model = tuner.get_best_models()[0]

best_model.summary()

best_model.evaluate(x_out_of_sample, y_out_of_sample)