Hello!
I am new to the field of deep learning and have attempted to create a Transformer model using time series data (weather data) with the help of the reference code (https://github.com/CVxTz/time_series_forecasting/tree/main). I had to customize some aspects to suit my needs.
I am working on this code for my bachelor's thesis to compare Transformer models with LSTM models to determine which is better suited for weather data. I'm unsure whether it's normal for my training, with around 13,000 parameters, to take more than 8 hours or if it's just stuck at some point. I have reduced the model from an initial 8 million parameters to only 13,000 parameters. Additionally, my data has been significantly reduced. I am currently stuck in the training process and need assistance.
Here my Code:
Model:
class TimeSeriesForcasting(pl.LightningModule):
def __init__(self, n_encoder_inputs, n_decoder_inputs, channels, dropout, lr):
super().__init__()
self.save_hyperparameters()
self.lr = lr
self.dropout = dropout
self.input_pos_embedding = torch.nn.Embedding(1024, embedding_dim=channels//2)
self.target_pos_embedding = torch.nn.Embedding(1024, embedding_dim=channels//2)
encoder_layer = nn.TransformerEncoderLayer(
d_model=channels,
nhead=2, #8
dropout=self.dropout,
dim_feedforward=4 * channels,
)
decoder_layer = nn.TransformerDecoderLayer(
d_model=channels,
nhead=2, #8
dropout=self.dropout,
dim_feedforward=4 * channels,
)
self.encoder = torch.nn.TransformerEncoder(encoder_layer, num_layers=2)#8
self.decoder = torch.nn.TransformerDecoder(decoder_layer, num_layers=2)#8
self.input_projection = Linear(n_encoder_inputs, channels //2)
self.output_projection = Linear(n_decoder_inputs, channels //2)
self.linear = Linear(channels //2, 1)
self.do = nn.Dropout(p=self.dropout)
def encode_src(self, src):
src_start = self.input_projection(src).permute(1, 0, 2)
in_sequence_len, batch_size = src_start.size(0), src_start.size(1)
pos_encoder = (
torch.arange(0, in_sequence_len, device=src.device)
.unsqueeze(0)
.repeat(batch_size, 1)
)
pos_encoder = self.input_pos_embedding(pos_encoder).permute(1, 0, 2)
src = src_start + pos_encoder
src = self.encoder(src) + src_start
return src
def decode_trg(self, trg, memory):
trg_start = self.output_projection(trg).permute(1, 0, 2)
out_sequence_len, batch_size = trg_start.size(0), trg_start.size(1)
pos_decoder = (
torch.arange(0, out_sequence_len, device=trg.device)
.unsqueeze(0)
.repeat(batch_size, 1)
)
pos_decoder = self.target_pos_embedding(pos_decoder).permute(1, 0, 2)
trg = pos_decoder + trg_start
trg_mask = gen_trg_mask(out_sequence_len, trg.device)
out = self.decoder(tgt=trg, memory=memory, tgt_mask=trg_mask) + trg_start
out = out.permute(1, 0, 2)
out = self.linear(out)
return out
def forward(self, x):
src, trg = x
src = self.encode_src(src)
out = self.decode_trg(trg=trg, memory=src)
return out
def training_step(self, batch, batch_idx):
src, trg_in, trg_out = batch
y_hat = self((src, trg_in))
y_hat = y_hat.view(-1)
y = trg_out.view(-1)
loss = smape_loss(y_hat, y)
self.log("train_loss", loss)
return loss
def validation_step(self, batch, batch_idx):
src, trg_in, trg_out = batch
y_hat = self((src, trg_in))
y_hat = y_hat.view(-1)
y = trg_out.view(-1)
loss = smape_loss(y_hat, y)
self.log("valid_loss", loss)
return loss
def test_step(self, batch, batch_idx):
src, trg_in, trg_out = batch
y_hat = self((src, trg_in))
y_hat = y_hat.view(-1)
y = trg_out.view(-1)
loss = smape_loss(y_hat, y)
self.log("test_loss", loss)
return loss
def configure_optimizers(self):
optimizer = torch.optim.Adam(self.parameters(), lr=self.lr)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, patience=10, factor=0.1
)
return {
"optimizer": optimizer,
"lr_scheduler": scheduler,
"monitor": "valid_loss",
}
train-function:
def train(
data_csv_path: str,
feature_target_names_path: str,
output_json_path: str,
log_dir: str,
model_dir: str ,
batch_size: int,
epochs: int ,
horizon_size: int,
channels: int,
lr: float,
dropout: float
):
data = pd.read_csv(data_csv_path)
with open(feature_target_names_path) as f:
feature_target_names = json.load(f)
data_train = data[~data[feature_target_names["target"]].isna()]
grp_by_train = data_train.groupby(by=feature_target_names["group_by_key"])
full_groups = []
groups = list(grp_by_train.groups)
for grp in groups:
#print(grp_by_train.get_group(grp).shape[0], horizon_size)
if grp_by_train.get_group(grp).shape[0] > 2 * horizon_size:
full_groups.append(grp)
#full_groups = [
# grp for grp in groups if grp_by_train.get_group(grp).shape[0] > 2 * horizon_size
#]
train_data = Dataset(
groups=full_groups,
grp_by=grp_by_train,
split="train",
features=feature_target_names["features"],
target=feature_target_names["target"],
)
val_data = Dataset(
groups=full_groups,
grp_by=grp_by_train,
split="val",
features=feature_target_names["features"],
target=feature_target_names["target"],
)
print("len(train_data)", len(train_data))
print("len(val_data)", len(val_data))
train_loader = DataLoader(
train_data,
batch_size=batch_size,
num_workers=12,
persistent_workers=True,
shuffle=True,
)
val_loader = DataLoader(
val_data,
batch_size=batch_size,
num_workers=12,
persistent_workers=True,
shuffle=False,
)
model = TimeSeriesForcasting(
n_encoder_inputs=len(feature_target_names["features"]) + 1,
n_decoder_inputs=len(feature_target_names["features"]) + 1,
lr=lr,
dropout=dropout,
channels=channels
)
logger = TensorBoardLogger(
save_dir=log_dir,
)
checkpoint_callback = ModelCheckpoint(
monitor="valid_loss",
mode="min",
dirpath=model_dir,
filename="ts",
)
trainer = pl.Trainer(
max_epochs=epochs,
accelerator='auto',
logger=logger,
callbacks=[checkpoint_callback],
)
trainer.fit(model, train_loader, val_loader)
result_val = trainer.test(test_dataloaders=val_loader)
output_json = {
"val_loss": result_val[0]["test_loss"],
"best_model_path": checkpoint_callback.best_model_path,
}
if output_json_path is not None:
with open(output_json_path, "w") as f:
json.dump(output_json, f, indent=4)
return output_json
Class-Dataset:
class Dataset(torch.utils.data.Dataset):
def __init__(self, groups, grp_by, split, features, target):
self.groups = groups
self.grp_by = grp_by
self.split = split
self.features = features
self.target = target
def __len__(self):
return len(self.groups)
def __getitem__(self, idx):
group = self.groups[idx]
df = self.grp_by.get_group(group)
src, trg = split_df(df, split=self.split)
src = src[self.features + [self.target]]
src = df_to_np(src)
trg_in = trg[self.features + [f"{self.target}_lag_1"]]
trg_in = np.array(trg_in)
trg_out = np.array(trg[self.target])
src = torch.tensor(src, dtype=torch.float)
trg_in = torch.tensor(trg_in, dtype=torch.float)
trg_out = torch.tensor(trg_out, dtype=torch.float)
return src, trg_in, trg_out
Other-functions:
def pad_arr(arr: np.ndarray, expected_size: int = 120):
"""
Pad top of array when there is not enough history
:param arr:
:param expected_size:
:return:
"""
arr = np.pad(arr, [(expected_size - arr.shape[0], 0), (0, 0)], mode="edge")
return arr
def df_to_np(df):
arr = np.array(df)
arr = pad_arr(arr)
return arr
def split_df(
df: pd.DataFrame, split: str, history_size: int = 120, horizon_size: int = 30
):
"""
Create a training / validation samples
Validation samples are the last horizon_size rows
:param df:
:param split:
:param history_size:
:param horizon_size:
:return:
"""
if split == "train":
end_index = random.randint(horizon_size + 1, df.shape[0] - horizon_size)
elif split in ["val", "test"]:
end_index = df.shape[0]
else:
raise ValueError
label_index = end_index - horizon_size
start_index = max(0, label_index - history_size)
history = df[start_index:label_index]
targets = df[label_index:end_index]
return history, targets
def gen_trg_mask(length, device):
mask = torch.tril(torch.ones(length, length, device=device)) == 1
mask = (
mask.float()
.masked_fill(mask == 0, float("-inf"))
.masked_fill(mask == 1, float(0.0))
)
return mask
def smape_loss(y_pred, target):
loss = 2 * (y_pred - target).abs() / (y_pred.abs() + target.abs() + 1e-8)
return loss.mean()
Data i used 
This is my output on Training-Start and also after 8hours of training (i stopped it at 8 Hours):
len(train_data) 5
len(val_data) 5
c:\Users\Luca\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\nn\modules\transformer.py:282: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.self_attn.batch_first was not True(use batch_first for better inference performance)
warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
GPU available: True (cuda), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
You are using a CUDA device ('NVIDIA GeForce RTX 3080 Ti') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
| Name | Type | Params
------------------------------------------------------------
0 | input_pos_embedding | Embedding | 4.1 K
1 | target_pos_embedding | Embedding | 4.1 K
2 | encoder | TransformerEncoder | 1.7 K
3 | decoder | TransformerDecoder | 2.4 K
4 | input_projection | Linear | 28
5 | output_projection | Linear | 28
6 | linear | Linear | 5
7 | do | Dropout | 0
------------------------------------------------------------
12.3 K Trainable params
0 Non-trainable params
12.3 K Total params
0.049 Total estimated model params size (MB)