Skip to main content
 首页 » 编程设计

python之让 DQN 解决 A 是否大于 B

2025年02月15日8小虾米

我最近开始研究神经网络。我目前正在试验 DQN。我已经能够让他们使用大量的 Atari 教程,但发现我无法理解整个问题,所以我为 DQN 编写了一个简单的模式来处理它只是 A 比 B 大,如果不是,则为 1给 0。然后根据神经网络是否正确给它打分 1 或 0。不幸的是,我无法让它学习这个简单的问题。有人可以帮我解决这个问题吗?

# -*- coding: utf-8 -*- 
import random 
import numpy as np 
import env 
import gym 
from collections import deque 
from keras.models import Sequential 
from keras.layers import Dense 
from keras.optimizers import Adam 
 
EPISODES = 1000 
 
 
class DQNAgent: 
    def __init__(self, state_size, action_size): 
        self.state_size = state_size 
        self.action_size = action_size 
        self.memory = deque(maxlen=2000) 
        #self.gamma = 0.95    # discount rate 
        self.gamma = 0  # discount rate 
        self.epsilon = 0.5  # exploration rate 
        self.epsilon_min = 0.01 
        self.epsilon_decay = 0.985 
        self.learning_rate = 0.001 
        self.model = self._build_model() 
 
    def _build_model(self): 
        # Neural Net for Deep-Q learning Model 
        model = Sequential() 
        model.add(Dense(2, input_dim=self.state_size, activation='relu')) 
        model.add(Dense(2, activation='relu')) 
        model.add(Dense(2, activation='relu')) 
        model.add(Dense(self.action_size, activation='relu')) 
        model.compile(loss='mse', 
                      optimizer=Adam(lr=self.learning_rate)) 
        return model 
 
    def remember(self, state, action, reward, next_state, done): 
        self.memory.append((state, action, reward, next_state, done)) 
 
    def act(self, state): 
        if np.random.rand() <= self.epsilon: 
           return random.randrange(self.action_size) 
        action = np.argmax(self.model.predict(state)[0]) 
        return action 
 
    def replay(self, batch_size): 
        minibatch = random.sample(self.memory, batch_size) 
        for state, action, reward, next_state, done in minibatch: 
            print("stating") 
            target = reward 
            if not done: 
                target = reward + self.gamma * np.amax(self.model.predict(next_state)[0]) 
            target_f = self.model.predict(state) 
            target_f[0][action] = target 
            print("Reward: " + str(reward)) 
            print("Target: " + str(target)) 
            print(action) 
            print(self.gamma*np.amax(self.model.predict(next_state)[0])) 
            print(state) 
            print(target_f) 
            self.model.fit(state,target_f, epochs=1, verbose=0) 
        if self.epsilon > self.epsilon_min: 
            self.epsilon *= self.epsilon_decay 
 
    def load(self, name): 
        self.model.load_weights(name) 
 
    def save(self, name): 
        self.model.save_weights(name) 
 
 
if __name__ == "__main__": 
    state_size = 2 
    action_size = 2 
    #timeRange = 1440 
    timeRange = 998 
    agent = DQNAgent(state_size, action_size) 
    # agent.load("./save/cartpole-master.h5") 
    done = False 
    batch_size = 500 
    totalScore=0 
    for e in range(EPISODES): 
        env_state = env.GameState() 
        env_state.reset() 
        state = env_state.step(0)[1] 
        #state = np.reshape(state, [1, state_size]) 
        totalreward = 0 
        for time in range(timeRange): 
            #if time==timeRange-1: 
                #done = True 
            action = agent.act(state) 
            reward, next_state = env_state.step(action)  
 
            totalreward += reward 
            next_state = np.reshape(next_state, [1, state_size]) 
            agent.remember(state, action, reward, next_state, done) 
            state = next_state 
            if done: 
                print("episode: {}/{}, score: {}, e: {:.2}" 
                      .format(e, EPISODES, env_state.money+env_state.shares*env_state.sharePrice, agent.epsilon)) 
                totalScore+=env_state.money+env_state.shares*env_state.sharePrice 
                done = False 
                break 
        if len(agent.memory) > batch_size: 
            print("replaying") 
            print("Total reward: "+str(totalreward)) 
            totalreward = 0 
            #print(agent.memory) 
            agent.replay(batch_size) 
        # if e % 10 == 0: 
        #     agent.save("./save/cartpole.h5") 

请您参考如下方法:

这来自您在模型中使用的激活:

def _build_model(self): 
    # Neural Net for Deep-Q learning Model 
    model = Sequential() 
    model.add(Dense(2, input_dim=self.state_size, activation='relu')) 
    model.add(Dense(2, activation='relu')) 
    model.add(Dense(2, activation='relu')) 
    model.add(Dense(self.action_size, activation='relu')) 
    model.compile(loss='mse', 
                  optimizer=Adam(lr=self.learning_rate)) 
    return model 

使用 RELU 激活无法学习 > 或 < 操作。仔细想想,RELU 无法仅通过线性运算来学习判断一个数是否大于另一个数。

但是,当您更改激活时:

def _build_model(self): 
    # Neural Net for Deep-Q learning Model 
    model = Sequential() 
    model.add(Dense(2, input_dim=self.state_size, activation='sigmoid')) 
    model.add(Dense(2, activation='sigmoid')) 
    model.add(Dense(2, activation='sigmoid')) 
    model.add(Dense(self.action_size, activation='sigmoid')) 
    model.compile(loss='mse', 
                  optimizer=Adam(lr=self.learning_rate)) 
    return model 

现在出现了其他类型的非线性,使网络能够学习那些看似“简单”的比较任务。

我希望这对您有所帮助:-)