train.py

from symphony import Symphony
import gymnasium as gym

import logging
logging.getLogger().setLevel(logging.CRITICAL)
import torch
import numpy as np
import random
import pickle
import time
import os, re

#############################################
# -----------Helper Functions---------------#
#############################################



# random seeds for reproducing the experiment
def seed_reset():
    r1, r2, r3 = random.randint(0,2**32-1), random.randint(0,2**32-1), random.randint(0,2**32-1)
    torch.manual_seed(r1)
    np.random.seed(r2)
    random.seed(r3)
    return r1, r2, r3


def extract_r1_r2_r3():
    pattern = r'history_(\d+)_(\d+)_(\d+)\.csv'

    # Iterate through the files in the given directory
    for filename in os.listdir():
        # Match the filename with the pattern
        match = re.match(pattern, filename)
        if match:
            # Extract the numbers r1, r2, and r3 from the filename
            return map(int, match.groups())
    return None


#write or append to the history log file
class LogFile(object):
    def __init__(self, log_name_main, log_name_opt):
        self.log_name_main = log_name_main
        self.log_name_opt = log_name_opt
    def write(self, text):
        with open(self.log_name_main, 'a+') as file:
            file.write(text)
    def write_opt(self, text):
        with open(self.log_name_opt, 'a+') as file:
            file.write(text)
    def clean(self):
        with open(self.log_name_main, 'w') as file:
            file.write("step,return\n")
        with open(self.log_name_opt, 'w') as file:
            file.write("ep,return,steps,scale\n")


numbers = extract_r1_r2_r3()

if numbers != None:
    # derive random numbers from history file
    r1, r2, r3 = numbers
else:
    # generate new random seeds
    r1, r2, r3 = seed_reset()



print(r1, ", ", r2, ", ", r3)

log_name_main = "history_" + str(r1) + "_" + str(r2) + "_" + str(r3) + ".csv"
log_name_opt = "episodes_" + str(r1) + "_" + str(r2) + "_" + str(r3) + ".csv"
log_file = LogFile(log_name_main, log_name_opt)


def save(algo, total_rewards, total_steps):

    torch.save(algo.nets.online.state_dict(), 'nets_online_model.pt')
    torch.save(algo.nets.target.state_dict(), 'nets_target_model.pt')
    torch.save(algo.nets_optimizer.state_dict(), 'nets_optimizer.pt')
    print("saving... the buffer length = ", algo.replay_buffer.length, end="")
    with open('data', 'wb') as file:
        pickle.dump({'buffer': algo.replay_buffer, 'q_next_ema': algo.nets.q_next_ema, 'total_rewards': total_rewards, 'total_steps': total_steps}, file)
    print(" > done")


def load(algo, Q_learning):

    total_rewards, total_steps = [], 0

    try:
        print("loading models...")
        algo.nets.online.load_state_dict(torch.load('nets_online_model.pt', weights_only=True))
        algo.nets.target.load_state_dict(torch.load('nets_target_model.pt', weights_only=True))
        algo.nets_optimizer.load_state_dict(torch.load('nets_optimizer.pt', weights_only=True))
        print('models loaded')
        #sim_loop(env_valid, 100, True, False, algo, [], total_steps=0)
    except:
        print("problem during loading models")


    try:
        print("loading buffer...")
        with open('data', 'rb') as file:
            dict = pickle.load(file)
            algo.replay_buffer = dict['buffer']
            algo.nets.q_next_ema = dict['q_next_ema']
            total_rewards = dict['total_rewards']
            total_steps = dict['total_steps']
            if algo.replay_buffer.length>=explore_time and not Q_learning: Q_learning = True
        
        print('buffer loaded, Q_ema', round(algo.nets.q_next_ema.item(), 2), ', average_reward = ', round(np.mean(total_rewards[-300:]), 2))
        
    except:
        print("problem during loading buffer")

    return Q_learning, total_rewards, total_steps

#############################################
# ---------------Parametres-----------------#
#############################################

#global parameters
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

print(device)
G = 3
learning_rate = 1e-4
explore_time, times = 10240, 50
capacity = explore_time * times
h_dim = capacity//1000
limit_step = 1000 #max steps per episode
limit_eval = 1000 #max steps per evaluation
num_episodes = 1000000
start_episode = 1 #number for the identification of the current episode
episode_rewards_all, episode_steps_all, test_rewards, Q_learning, total_steps = [], [], [], False, 0

# environment type.
option = 3
pre_valid = True
if option == 0: env_name = '"BipedalWalker-v3'
elif option == 1: env_name = 'HalfCheetah-v4'
elif option == 2: env_name = 'Walker2d-v4'
elif option == 3: env_name = 'Humanoid-v4'
elif option == 4: env_name = 'Ant-v4'
elif option == 5: env_name = 'Swimmer-v4'
elif option == 6: env_name = 'Hopper-v4'
elif option == 7: env_name = 'Pusher-v4'

env = gym.make(env_name)
env_test = gym.make(env_name)
env_valid = gym.make(env_name, render_mode="human")

state_dim = env.observation_space.shape[0]
action_dim= env.action_space.shape[0]
#max_action = torch.FloatTensor(env.action_space.high) if env.action_space.is_bounded() else torch.ones(action_dim)
max_action = torch.ones(action_dim)

print("action_dim: ", action_dim, "state_dim: ", state_dim, "max_action:", max_action)

algo = Symphony(capacity, state_dim, action_dim, h_dim, device, max_action, learning_rate)


# Loop for episodes:[ State -> Loop for one episode: [ Action, Next State, Reward, Done, State = Next State ] ]
def sim_loop(env, episodes, testing, Q_learning, algo, total_rewards, total_steps):


    start_episode = len(total_rewards) + 1


    for episode in range(start_episode, episodes+1):
            
        Return = 0.0     
        state = env.reset()[0]
        
        for steps in range(1,limit_step+1):

            seed_reset()
            total_steps += 1

            # Activate training if explore time is reached and if it is not testing mode:
            if testing:
                Q_learning = False
            else:
                if algo.replay_buffer.length>=explore_time and not Q_learning:
                    Q_learning = True
                    algo.replay_buffer.norm_fill(times)
                    print("started training")

            # if total steps is divisible to 2500 save models, stop training and do testing, return to training:
            if Q_learning and total_steps>=2500 and total_steps%2500==0:
                save(algo, total_rewards, total_steps)
                print("start testing")
                test_return = sim_loop(env_test, 25, True, Q_learning, algo, [], total_steps=0)
                log_file.write(str(total_steps) + "," + str(round(test_return, 2)) + "\n")
                print("end of testing")


            # if steps is close to episode limit (e.g. 950) we shut down actions and leave noise to get Terminal Transition:
            active = steps<(limit_step-50) if Q_learning else True
            action = algo.select_action(state,  action=active, noise=not testing)
            next_state, reward, done, truncated, info = env.step(action)
            if not testing: algo.replay_buffer.add(state, action, reward, next_state, done)
            Return += reward
            
            # actual training
            if Q_learning: [scale := algo.train() for _ in range(G)]
            if done or truncated: break
            state = next_state



        total_rewards.append(Return)
        average_reward = np.mean(total_rewards[-300:])


        print(f"Ep {episode}: Rtrn = {Return:.2f}, Avg = {average_reward:.2f}| ep steps = {steps} | total_steps = {total_steps}") 
        if not testing and Q_learning: log_file.write_opt(str(episode) + "," + str(round(Return, 2)) + "," + str(total_steps) + "," + str(round(scale.mean().item(), 4)) + "\n")
        

    return np.mean(total_rewards).item()




# Loading existing models
Q_learning, total_rewards, total_steps = load(algo, Q_learning)
if not Q_learning: log_file.clean()

# Training
sim_loop(env, num_episodes, False, Q_learning, algo, total_rewards, total_steps)