Reinforcement learning does nothing when using test forex data

Question

I am experimenting with RL and I am trying to write an AI so it can learn to trade the Forex market. Here is my code below:

from gym import Env
from gym.spaces import Discrete, Box
import numpy as np
import random
import pandas as pd
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Dropout
from tensorflow.keras.optimizers import Adam
from rl.agents import DQNAgent
from rl.policy import BoltzmannQPolicy
from rl.memory import SequentialMemory
import matplotlib.pyplot as plt


class ForexEnv(Env):
    def __init__(self, all_candle_data):
        self.amount_of_candles = 101
        self.spread = 0.00015
        self.all_candle_data = all_candle_data
        # Actions we can take, buy, sell, wait/close
        self.action_space = Discrete(3)
        self.observation_space = Box(
            low=0, high=10000, shape=(1, self.amount_of_candles, 5)
        )

    def reset(self):
        # 2 no trade, 0 buy, 1 sell
        self.current_trade = [2, 0]
        self.time_given = 1440
        self.candle_pos = random.randint(
            self.amount_of_candles, len(self.all_candle_data) - self.time_given
        )
        self.banked_profit = 0
        self.state = self.all_candle_data.values[
            self.candle_pos - self.amount_of_candles : self.candle_pos
        ]
        self.state[0] = [
            0,
            0,
            self.current_trade[0],
            self.current_trade[1],
            self.time_given,
        ]
        return self.state

    def step(self, action):
        # action
        current_close = self.all_candle_data.values[self.candle_pos][3]
        unrealised_profit = 0
        if self.current_trade[0] == 0:
            # buy
            unrealised_profit = (current_close - self.current_trade[1]) - (
                self.spread / 2
            )
        elif self.current_trade[0] == 1:
            # sell
            unrealised_profit = (self.current_trade[1] - current_close) - (
                self.spread / 2
            )

        if action == 0:
            # buy
            if self.current_trade[0] != 0:
                self.banked_profit += unrealised_profit
                self.current_trade = [0, current_close]
        elif action == 1:
            # sell
            if self.current_trade[0] != 1:
                self.banked_profit += unrealised_profit
                self.current_trade = [1, current_close]
        elif action == 2:
            # close
            self.banked_profit += unrealised_profit
            self.current_trade = [2, 0]

        # state
        self.candle_pos += 1
        self.state = self.all_candle_data.values[
            self.candle_pos - self.amount_of_candles : self.candle_pos
        ]
        self.state[0] = [
            self.banked_profit,
            unrealised_profit,
            self.current_trade[0],
            self.current_trade[1],
            self.time_given,
        ]

        # extras
        self.time_given -= 1
        if self.time_given <= 0:
            session_complete = True
        else:
            session_complete = False

        # Return step information
        return (
            self.state,
            self.banked_profit,
            session_complete,
            {"banked_profit": self.banked_profit},
        )


all_candle_data = pd.read_csv("./EURUSD_M5.csv")
all_candle_data.drop("Time", 1, inplace=True)

train_env = ForexEnv(all_candle_data.head(25000))
train_env.reset()

test_env = ForexEnv(all_candle_data.head(25000))
test_env.reset()


def build_model(states, actions):
    model = Sequential()
    model.add(Flatten(input_shape=states))
    model.add(Dropout(0.2))
    model.add(Dense(24, activation="elu"))
    model.add(Dense(24, activation="elu"))
    model.add(Dense(24, activation="elu"))
    model.add(Dense(actions, activation="softmax"))
    return model


states = train_env.observation_space.shape
actions = train_env.action_space.n
model = build_model(states, actions)


def build_agent(model, actions):
    policy = BoltzmannQPolicy()
    memory = SequentialMemory(
        limit=15000, window_length=1
    )  # this stores states,actions and rewards and is randomly sampled for training
    dqn = DQNAgent(
        model=model,
        memory=memory,
        policy=policy,
        enable_double_dqn=True,
        enable_dueling_network=False,
        dueling_type="avg",
        nb_actions=actions,
        nb_steps_warmup=100000,
        gamma=0.95,
    )
    return dqn


dqn = build_agent(model, actions)
dqn.compile(Adam(learning_rate=1e-4))

try:
    dqn.load_weights("saved_agent")
except:
    print('No saved weights')
history = dqn.fit(train_env, nb_steps=1000000, visualize=False, verbose=1)
dqn.save_weights("saved_agent", overwrite=True)

plt.plot(history.history["nb_steps"], history.history["episode_reward"])
plt.title("model reward")
plt.ylabel("reward")
plt.xlabel("step")
plt.show()

scores = dqn.test(test_env, nb_episodes=100, visualize=False)
print("average episode rewards", np.mean(scores.history["episode_reward"]))

The problem that I am having is that during training i get a wide range of rewards which gradually increase over time, so it looks all good. however when it comes to testing the AI on test data there agent decides to do nothing all the time and never place a trade. Here is a small amount of training:

Here is the code text output:

7 episodes - episode_reward: -23.952 [-33.885, -10.217] - banked_profit: -0.016

Interval 9 (80000 steps performed)
10000/10000 [==============================] - 11s 1ms/step - reward: -0.0160
7 episodes - episode_reward: -21.410 [-33.667, -11.435] - banked_profit: -0.016

Interval 10 (90000 steps performed)
10000/10000 [==============================] - 11s 1ms/step - reward: -0.0189
7 episodes - episode_reward: -28.482 [-39.062, -22.516] - banked_profit: -0.019

Interval 11 (100000 steps performed)
10000/10000 [==============================] - 84s 8ms/step - reward: -0.0178
7 episodes - episode_reward: -25.365 [-37.543, -12.404] - loss: 0.182 - mean_q: 1.000 - banked_profit: -0.018

Interval 12 (110000 steps performed)
10000/10000 [==============================] - 88s 9ms/step - reward: -0.0142
7 episodes - episode_reward: -20.698 [-28.274, -11.942] - loss: 0.185 - mean_q: 1.000 - banked_profit: -0.014

Interval 13 (120000 steps performed)
 7177/10000 [====================>.........] - ETA: 24s - reward: -0.0172^Cdone, took 350.249 seconds
Testing for 100 episodes ...
Episode 1: reward: 0.000, steps: 1440
Episode 2: reward: 0.000, steps: 1440
Episode 3: reward: 0.000, steps: 1440
Episode 4: reward: 0.000, steps: 1440
Episode 5: reward: 0.000, steps: 1440
Episode 6: reward: 0.000, steps: 1440
Episode 7: reward: 0.000, steps: 1440
Episode 8: reward: 0.000, steps: 1440
Episode 9: reward: 0.000, steps: 1440
Episode 10: reward: 0.000, steps: 1440

As shown above the reward changes and improves on the training, but does nothing while testing the data. Thank you for your time, and any other advice would be appriciated.

Happy Coding, Josh

Answer 1

i'm not professional but i fund type error that may cues of problem

in dense layer you should use relu insted of elu

model.add(layers.Dense(64, activation='relu'))

Answer 2

The purpose of a DQN network is to predict Q values for a given state action pair, which implies performing regression. Therefore, using a softmax activation function in your output layer doesn't seem correct. In addition, I think if you could give me a source of your dataset (I believe it's a generic non-proprietary dataset), I could run your code and give a more directed suggestion.