final_project_rl(2).py

# -*- coding: utf-8 -*-
"""Final_project_RL.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1ovIPStgN5Bt3EdqqQVKe4aHgLFwqwnnS

#install packages and import them
"""

!pip install Cmake

!pip install --upgrade numpy

!pip install swig
!pip install box2d-py==2.3.5
!pip install gymnasium stable-baselines3 yfinance tensorflow scikit-learn==1.2.2 Cython

!pip install git+https://github.com/AI4Finance-LLC/FinRL-Library.git

# Commented out IPython magic to ensure Python compatibility.
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import yfinance as yf
import gymnasium as gym
from gymnasium import spaces
# %matplotlib inline


from finrl import config
from finrl import config_tickers
from finrl.meta.preprocessor.yahoodownloader import YahooDownloader
from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split
from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv
from finrl.agents.stablebaselines3.models import DRLAgent
from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline

from sklearn.preprocessing import MinMaxScaler

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout
from stable_baselines3 import PPO
from stable_baselines3.common.vec_env import DummyVecEnv
from stable_baselines3.common.env_util import make_vec_env

"""#Download Data and Preprocess Train and Test"""

# This function is used for fetch stocks data
def fetch_data(stock, start, end):
    data = yf.download(stock, start=start, end=end)
    data = data[['Open', 'High', 'Low', 'Close', 'Volume']]
    return data

# Fetch 5 stocks for train 2009-01-01 to 2022-01-01
nvidia_df_train = fetch_data("NVDA", '2009-01-01', '2022-01-01')
apple_df_train = fetch_data("AAPL", '2009-01-01', '2022-01-01')
microsoft_df_train = fetch_data("MSFT", '2009-01-01', '2022-01-01')
google_df_train = fetch_data("GOOGL", '2009-01-01', '2022-01-01')
amazon_df_train = fetch_data("AMZN", '2009-01-01', '2022-01-01')

# Fetch 5 stocks for test 2022-01-02 to 2024-01-01
nvidia_df_test = fetch_data("NVDA", '2022-01-02', '2024-06-01')
apple_df_test = fetch_data("AAPL", '2022-01-02', '2024-06-01')
microsoft_df_test = fetch_data("MSFT", '2022-01-02', '2024-06-01')
google_df_test = fetch_data("GOOGL", '2022-01-02', '2024-06-01')
amazon_df_test = fetch_data("AMZN", '2022-01-02', '2024-06-01')

nvidia_df_train[nvidia_df_train.columns[0]].count()
  date = nvidia_df_train.copy()  # Copying the DataFrame to avoid modifying the original one
  date['Date'] = date.index
  print(date.shape[0])
  Date_x = []
  for i in range(date.shape[0]):
      Date_x.append(date.iloc[i, 5].strftime('%Y-%m-%d'))
  len(Date_x)

# Check if any of train stocks has a null value
print("Train:")
print(nvidia_df_train.isnull().values.any())
print(apple_df_train.isnull().values.any())
print(microsoft_df_train.isnull().values.any())
print(google_df_train.isnull().values.any())
print(amazon_df_train.isnull().values.any())
print("Tests:")
# Check if any of test stocks has a null value
print(nvidia_df_test.isnull().values.any())
print(apple_df_test.isnull().values.any())
print(microsoft_df_test.isnull().values.any())
print(google_df_test.isnull().values.any())
print(amazon_df_test.isnull().values.any())

# This function is used for scale each stock's data using min max scaler
def normalizer(data):
    scaler = MinMaxScaler()
    scaled_data = scaler.fit_transform(data)
    return scaled_data

# Normalize train datas
nvidia_df_train = pd.DataFrame(normalizer(nvidia_df_train))
apple_df_train = pd.DataFrame(normalizer(apple_df_train))
microsoft_df_train = pd.DataFrame(normalizer(microsoft_df_train))
google_df_train = pd.DataFrame(normalizer(google_df_train))
amazon_df_train = pd.DataFrame(normalizer(amazon_df_train))
# Normalize test datas
nvidia_df_test = pd.DataFrame(normalizer(nvidia_df_test))
apple_df_test = pd.DataFrame(normalizer(apple_df_test))
microsoft_df_test = pd.DataFrame(normalizer(microsoft_df_test))
google_df_test = pd.DataFrame(normalizer(google_df_test))
amazon_df_test = pd.DataFrame(normalizer(amazon_df_test))

# Add suitable columns to the train stocks
nvidia_df_train.columns = ['open', 'high', 'low', 'close', 'volume']
apple_df_train.columns = ['open', 'high', 'low', 'close', 'volume']
microsoft_df_train.columns = ['open', 'high', 'low', 'close', 'volume']
google_df_train.columns = ['open', 'high', 'low', 'close', 'volume']
amazon_df_train.columns = ['open', 'high', 'low', 'close', 'volume']

# Add suitable columns to the test stocks
nvidia_df_test.columns = ['open', 'high', 'low', 'close', 'volume']
apple_df_test.columns = ['open', 'high', 'low', 'close', 'volume']
microsoft_df_test.columns = ['open', 'high', 'low', 'close', 'volume']
google_df_test.columns = ['open', 'high', 'low', 'close', 'volume']
amazon_df_test.columns = ['open', 'high', 'low', 'close', 'volume']

"""#Learning"""

# Neural network model
def create_feature_extractor(input_shape):
    model = Sequential()
    model.add(LSTM(50, return_sequences=True, input_shape=input_shape))
    model.add(Dropout(0.2))
    model.add(LSTM(50, return_sequences=False))
    model.add(Dropout(0.2))
    model.add(Dense(1))
    model.compile(optimizer='adam', loss='mean_squared_error')
    return model

import gym
from gym import spaces
import numpy as np

class StockTradingEnv(gym.Env):
    metadata = {'render.modes': ['human']}

    def __init__(self, features, prices, initial_balance=10000):
        super(StockTradingEnv, self).__init__()

        self.features = features
        self.prices = prices
        self.initial_balance = initial_balance
        self.current_step = 0

        self.action_space = spaces.Discrete(3)
        self.observation_space = spaces.Box(low=-1, high=1, shape=(features.shape[1],), dtype=np.float32)

        self.reset()

    def reset(self, seed=None, options=None):
        super().reset(seed=seed)
        self.balance = self.initial_balance
        self.shares = 0
        self.current_step = 0
        self.done = False
        return self.features[self.current_step], {}

    def step(self, action):
        current_price = self.prices[self.current_step]

        if action == 1:  # Buy
            shares_bought = self.balance // current_price
            self.balance -= shares_bought * current_price
            self.shares += shares_bought
        elif action == 2:  # Sell
            self.balance += self.shares * current_price
            self.shares = 0

        self.current_step += 1
        if self.current_step >= len(self.features) - 1:
            self.done = True

        total_value = self.balance + self.shares * current_price
        reward = (total_value - self.initial_balance) / self.initial_balance

        next_state = self.features[self.current_step] if not self.done else np.zeros(self.features.shape[1])

        return next_state, reward, self.done, False, {}

    def render(self, mode='human'):
        profit = self.balance + self.shares * self.prices[self.current_step] - self.initial_balance
        print(f'Step: {self.current_step}')
        print(f'Balance: {self.balance}')
        print(f'Shares held: {self.shares}')
        print(f'Profit: {profit}')

    def close(self):
        pass

# Actor and Critic Network Definitions
def create_actor(input_shape):
    inputs = Input(shape=input_shape)
    x = LSTM(50, return_sequences=False)(inputs)
    x = Dropout(0.2)(x)
    outputs = Dense(1, activation='sigmoid')(x)
    model = Model(inputs, outputs, name='actor')
    return model

def create_critic(input_shape):
    inputs = Input(shape=input_shape)
    x = LSTM(50, return_sequences=False)(inputs)
    x = Dropout(0.2)(x)
    outputs = Dense(1)(x)
    model = Model(inputs, outputs, name='critic')
    return model

import torch
import torch.nn as nn
import torch.nn.functional as F
from stable_baselines3.common.policies import ActorCriticPolicy

class SimpleLSTMActorCritic(nn.Module):
    def __init__(self, observation_space, action_space):
        super(SimpleLSTMActorCritic, self).__init__()
        self.lstm = nn.LSTM(input_size=observation_space.shape[0], hidden_size=50, num_layers=1, batch_first=True)
        self.dropout = nn.Dropout(0.2)

        # Action space handling
        if isinstance(action_space, gym.spaces.Box):
            self.actor_fc = nn.Linear(50, action_space.shape[0])  # Continuous action space
            self.actor_activation = torch.tanh  # Use tanh for bounded continuous actions
        elif isinstance(action_space, gym.spaces.Discrete):
            self.actor_fc = nn.Linear(50, action_space.n)  # Discrete action space
            self.actor_activation = F.softmax  # Use softmax for discrete actions
        else:
            raise NotImplementedError("Unsupported action space type")

        self.critic_fc = nn.Linear(50, 1)

    def forward(self, x):
        lstm_out, _ = self.lstm(x)

        # Ensure lstm_out has the correct dimensions
        if len(lstm_out.shape) != 3:
            raise ValueError(f"Unexpected shape for LSTM output: {lstm_out.shape}")

        # Take the output from the last timestep
        lstm_out = lstm_out[:, -1, :]  # Shape: (batch_size, hidden_size)
        lstm_out = self.dropout(lstm_out)  # Apply dropout

        action = self.actor_activation(self.actor_fc(lstm_out), dim=-1)
        value = self.critic_fc(lstm_out)
        return action, value

class CustomActorCriticPolicy(ActorCriticPolicy):
    def _build(self, lr_schedule):
        self.mlp_extractor = SimpleLSTMActorCritic(self.observation_space, self.action_space)

def sharpe_ratio(returns, risk_free_rate=0.0):
    excess_returns = returns - risk_free_rate
    return np.mean(excess_returns) / np.std(excess_returns)

def appt(returns):
    return np.mean(returns)

def mpb(returns):
    return np.min(returns)

def mer(returns):
    return np.max(returns)

def cr(returns):
    return np.sum(returns)

def normalize_rewards(rewards, method='standard'):
    if method == 'standard':
        return (rewards - np.mean(rewards)) / np.std(rewards)
    elif method == 'minmax':
        return (rewards - np.min(rewards)) / (np.max(rewards) - np.min(rewards))
    return rewards

datasets = {
    'nvidia': (nvidia_df_train, nvidia_df_test),
    'apple': (apple_df_train, apple_df_test),
    'microsoft': (microsoft_df_train, microsoft_df_test),
    'google': (google_df_train, google_df_test),
    'amazon': (amazon_df_train, amazon_df_test)
}

time_window_list = [5, 15, 30, 50]

for name, (train_df, test_df) in datasets.items():
    scaler = StandardScaler()
    train_df_normalized = scaler.fit_transform(train_df)
    test_df_normalized = scaler.transform(test_df)

    rewards_dict = {time_window: [] for time_window in time_window_list}
    dates_dict = {time_window: [] for time_window in time_window_list}

    for time_window in time_window_list:
        X_train = []
        y_train = []
        for j in range(time_window, len(train_df_normalized)):
            X_train.append(train_df_normalized[j-time_window:j])
            y_train.append(train_df_normalized[j, 3])  # Close price

        X_train, y_train = np.array(X_train), np.array(y_train)

        lstm_model = create_lstm_model((X_train.shape[1], X_train.shape[2]))
        lstm_model.compile(optimizer='adam', loss='mse')  # Compile model
        lstm_model.fit(X_train, y_train, batch_size=128, epochs=15)

        features_train = lstm_model.predict(X_train)
        features_train = np.expand_dims(features_train, axis=1)  # Ensure correct shape
        env = DummyVecEnv([lambda: StockTradingEnv(features_train, train_df_normalized[:, 3])])

        model = PPO(CustomActorCriticPolicy, env, verbose=1, learning_rate=3e-4, seed=9, clip_range=0.2, gamma=0.99)
        model.learn(total_timesteps=10000)
        model.save(f'ppo_stock_{name}_{time_window}.zip')

        obs = env.reset()
        all_rewards = []
        for i in range(len(features_train)):
            action, _ = model.predict(obs)
            obs, reward, done, info = env.step(action)
            all_rewards.append(reward[0])
            dates_dict[time_window].append(train_df.index[time_window + i])
            if done:
                break

        rewards_dict[time_window] = all_rewards

    plt.figure(figsize=(14, 8))
    for time_window in time_window_list:
        dates = dates_dict[time_window]
        rewards = rewards_dict[time_window]
        cumulative_rewards = np.cumsum(rewards)

        cumulative_rewards = (cumulative_rewards - np.min(cumulative_rewards)) / (np.max(cumulative_rewards) - np.min(cumulative_rewards))


        if len(dates) != len(cumulative_rewards):
            print(f"Warning: Dates and rewards lengths do not match for time_window {time_window}.")
            min_length = min(len(dates), len(cumulative_rewards))
            dates = dates[:min_length]
            cumulative_rewards = cumulative_rewards[:min_length]

        plt.plot(dates, cumulative_rewards, label=f'Time Window: {time_window}')

    plt.xlabel('Date')
    plt.ylabel('Normalized Cumulative Reward')
    plt.title(f'Normalized Cumulative Rewards over Time for {name.capitalize()}')
    plt.legend()
    plt.grid(True)
    plt.xticks(rotation=45)
    plt.show()

import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from stable_baselines3 import PPO
import matplotlib.pyplot as plt
import os
from stable_baselines3.common.vec_env import DummyVecEnv

# Constant
TIME_WINDOWS = [5, 15, 30, 50]

def calculate_metrics(y_true, y_pred):
    trades = y_pred[1:] - y_pred[:-1]
    profitable_trades = [trade for trade in trades if trade > 0]
    APPT = np.mean(profitable_trades) if profitable_trades else 0

    returns = (y_pred - y_true) / y_true
    CR = (y_pred[-1] - y_pred[0]) / y_pred[0]
    MER = np.max(returns)
    MPB = np.min(returns)

    annualized_return = np.mean(returns) * 252
    annualized_volatility = np.std(returns) * np.sqrt(252)
    SR = (annualized_return - 0.01) / annualized_volatility if annualized_volatility else 0

    return {
        "APPT": APPT,
        "SR": SR,
        "CR": CR,
        "MER": MER,
        "MPB": MPB
    }

def plot_results(dates, y_true, y_pred, title):
    plt.figure(figsize=(12, 6))
    plt.plot(dates, y_true, label='True')
    plt.plot(dates, y_pred, label='Predicted')
    plt.title(title)
    plt.legend()
    plt.show()

def test_stock_model(stock_name, df_test, extract_dir):
    results = {}
    for window in TIME_WINDOWS:
        print(f"Testing {stock_name} with time window {window}")

        scaler = StandardScaler()
        data_scaled = scaler.fit_transform(df_test)
        X_test, y_test = [], []
        for i in range(window, len(data_scaled)):
            X_test.append(data_scaled[i-window:i])
            y_test.append(data_scaled[i, 3])

        X_test, y_test = np.array(X_test), np.array(y_test)

        model_file_name = f'ppo_stock_{stock_name}_{window}.zip'
        model_path = os.path.join(extract_dir, model_file_name)
        model = PPO.load(model_path)

        features_test = lstm_model.predict(X_test)
        env = DummyVecEnv([lambda: StockTradingEnv(features_test, df_test.iloc[:, 3])])

        y_pred = []
        obs = env.reset()
        for i in range(len(y_test)):
            single_obs = np.expand_dims(X_test[i], axis=0)
            action, _ = model.predict(obs, deterministic=True)
            obs, reward, done, info = env.step(action)
            y_pred.append(reward[0])

            if done:
                break

        y_pred = np.array(y_pred)

        if len(y_pred) < len(y_test):
            y_pred = np.pad(y_pred, (0, len(y_test) - len(y_pred)), 'edge')

        metrics = calculate_metrics(y_test, y_pred)
        results[window] = metrics

        plot_results(df_test.index[window:], y_test, y_pred, f"{stock_name} - Window {window}")

    return results

test_data = {
    'nvidia': nvidia_df_test,
    'apple': apple_df_test,
    'microsoft': microsoft_df_test,
    'google': google_df_test,
    'amazon': amazon_df_test
}

extract_dir = '/content/'
all_results = {}
for stock in test_data.keys():
    all_results[stock] = test_stock_model(stock, test_data[stock], extract_dir)
    print(f"Results for {stock}: {all_results[stock]}")

# Commented out IPython magic to ensure Python compatibility.
# %matplotlib inline
import matplotlib.pyplot as plt
import numpy as np

X = np.linspace(-0.1, 10, 20)
y = 2 * X - 1
plt.scatter(X, y)
plt.show()