这个问题不同于ValueError: Error when checking input: expected lstm_1_input to have 3 dimensions, but got array with shape (10, 1)因为那里的答案不满足我的情况。
按照此处的教程进行操作:https://www.altumintelligence.com/articles/a/Time-Series-Prediction-Using-LSTM-Deep-Neural-Networks , 这是模型摘要:
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
lstm_1 (LSTM) (None, 49, 100) 41200
_________________________________________________________________
dropout_1 (Dropout) (None, 49, 100) 0
_________________________________________________________________
lstm_2 (LSTM) (None, 49, 100) 80400
_________________________________________________________________
lstm_3 (LSTM) (None, 100) 80400
_________________________________________________________________
dropout_2 (Dropout) (None, 100) 0
_________________________________________________________________
dense_1 (Dense) (None, 1) 101
=================================================================
Total params: 202,101
Trainable params: 202,101
Non-trainable params: 0
_________________________________________________________________
None
我的数据集大小为 79005,序列长度为 50,时间步长/批量大小为 32。问题是当我将 epoch 配置为 1 时,一切都很完美。但是,当我将其更改为 2 时,在第二个纪元开始时出现以下错误:
ValueError: Error when checking input: expected lstm_1_input to have 3 dimensions, but got array with shape (0, 1)
我只是想了解为什么这不是 1 个纪元的问题,为什么只有 2 个(或更多)纪元? 编辑:将训练测试拆分设置为 0.80 而不是 0.85 实际上消除了错误!我仍然想知道原因,因为我不明白。
下面是我的数据加载代码:
import math
import numpy as np
import pandas as pd
class DataLoader():
"""A class for loading and transforming data for the lstm model"""
def __init__(self, filename, split, cols):
dataframe = pd.read_csv(filename)
print("data shape:",dataframe.shape)
i_split = int(len(dataframe) * split)
self.data_train = dataframe.get(cols).values[:i_split]
self.data_test = dataframe.get(cols).values[i_split:]
self.len_train = len(self.data_train)
self.len_test = len(self.data_test)
self.len_train_windows = None
def get_test_data(self, seq_len, normalise):
'''
Create x, y test data windows
Warning: batch method, not generative, make sure you have enough memory to
load data, otherwise reduce size of the training split.
'''
data_windows = []
#data_x=[]
#data_y=[]
for i in range(self.len_test - seq_len):
data_windows.append(self.data_test[i:i+seq_len])
data_windows = np.array(data_windows).astype(float)
data_windows = self.normalise_windows(data_windows, single_window=False) if normalise else data_windows
x = data_windows[:, :-1]
y = data_windows[:, -1, [0]]
#x,y=self._next_window(i,seq_len,normalise,train=False)
#data_x.append(x)
#data_y.append(y)
#return np.array(data_x),np.array(data_y)
return x,y
def get_train_data(self, seq_len, normalise):
'''
Create x, y train data windows
Warning: batch method, not generative, make sure you have enough memory to
load data, otherwise use generate_training_window() method.
'''
data_x = []
data_y = []
for i in range(self.len_train - seq_len):
x, y = self._next_window(i, seq_len, normalise)
data_x.append(x)
data_y.append(y)
return np.array(data_x), np.array(data_y)
def generate_train_batch(self, seq_len, batch_size, normalise, epochs):
'''Yield a generator of training data from filename on given list of cols split for train/test'''
i = 0
print("train length:",self.len_train)
#while epoch < epochs:
while i < ((self.len_train - seq_len)*(epochs+1)):
print("i:",i)
x_batch = []
y_batch = []
for b in range(batch_size):
if i >= (self.len_train - seq_len):
# stop-condition for a smaller final batch if data doesn't divide evenly
yield np.array(x_batch), np.array(y_batch)
i = 0
print("i set to 0")
x, y = self._next_window(i, seq_len, normalise)
x_batch.append(x)
y_batch.append(y)
i += 1
print ("x:",np.array(x_batch).shape)
print ("y:",np.array(y_batch).shape)
yield np.array(x_batch), np.array(y_batch)
#epoch += 1
def _next_window(self, i, seq_len, normalise):
'''Generates the next data window from the given index location i'''
window = self.data_train[i:i+seq_len]
#if train:
# window = self.data_train[i:i+seq_len]
#else:
# window = self.data_test[i:i+seq_len]
window = self.normalise_windows(window, single_window=True)[0] if normalise else window
x = window[:-1]
y = window[-1, [0]]
return x, y
def normalise_windows(self, window_data, single_window=False):
'''Normalise window with a base value of zero'''
eps=0.00001
normalised_data = []
window_data = [window_data] if single_window else window_data
for window in window_data:
normalised_window = []
for col_i in range(window.shape[1]):
normalised_col = [((float(p) / (float(window[0, col_i])+eps)) - 1) for p in window[:, col_i]]
normalised_window.append(normalised_col)
normalised_window = np.array(normalised_window).T # reshape and transpose array back into original multidimensional format
normalised_data.append(normalised_window)
return np.array(normalised_data)
下面是模型构建代码:
class Model():
"""A class for an building and inferencing an lstm model"""
def __init__(self):
self.model = Sequential()
def load_model(self, filepath):
print('[Model] Loading model from file %s' % filepath)
self.model = load_model(filepath)
def build_model(self, configs):
timer = Timer()
timer.start()
for layer in configs['model']['layers']:
neurons = layer['neurons'] if 'neurons' in layer else None
dropout_rate = layer['rate'] if 'rate' in layer else None
activation = layer['activation'] if 'activation' in layer else None
return_seq = layer['return_seq'] if 'return_seq' in layer else None
input_timesteps = layer['input_timesteps'] if 'input_timesteps' in layer else None
input_dim = layer['input_dim'] if 'input_dim' in layer else None
if layer['type'] == 'dense':
self.model.add(Dense(neurons, activation=activation))
if layer['type'] == 'lstm':
self.model.add(LSTM(neurons, input_shape=(input_timesteps, input_dim), return_sequences=return_seq))
if layer['type'] == 'dropout':
self.model.add(Dropout(dropout_rate))
self.model.compile(loss=configs['model']['loss'], optimizer=configs['model']['optimizer'],metrics=['mean_squared_error'])
print(self.model.summary())
plot_model(self.model, to_file='model.png')
print('[Model] Model Compiled')
timer.stop()
return self.model
def train(self, x, y, epochs, batch_size, save_dir=""):
timer = Timer()
timer.start()
print('X shape:', (x.shape))
print('[Model] Training Started')
print('[Model] %s epochs, %s batch size' % (epochs, batch_size))
save_fname = os.path.join(save_dir, '%s-e%s.h5' % (dt.datetime.now().strftime('%d%m%Y-%H%M%S'), str(epochs)))
callbacks = [
EarlyStopping(monitor='val_loss', patience=2),
ModelCheckpoint(filepath=save_fname, monitor='val_loss', save_best_only=True)
]
modelhistory=self.model.fit(
x,
y,
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks
)
self.model.save(save_fname)
print('[Model] Training Completed. Model saved as %s' % save_fname)
timer.stop()
return modelhistory
def train_generator(self, data_gen, epochs, batch_size, steps_per_epoch, save_dir=""):
timer = Timer()
timer.start()
#print('X shape:', (x.shape))
print('[Model] Training Started')
print('[Model] %s epochs, %s batch size, %s batches per epoch' % (epochs, batch_size, steps_per_epoch))
save_fname = os.path.join(save_dir, '%s-e%s.h5' % (dt.datetime.now().strftime('%d%m%Y-%H%M%S'), str(epochs)))
callbacks = [
ModelCheckpoint(filepath=save_fname, monitor='loss', save_best_only=True)
]
modelhistory=self.model.fit_generator(
data_gen,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks,
workers=1
)
print('[Model] Training Completed. Model saved as %s' % save_fname)
timer.stop()
return modelhistory
下面是定义了所有参数的模型配置:
configJson={
"data": {
"filename": "C:/projects!/Experiments/2015-02-02-To-2019-5-19-5-Min.csv",
"columns": [
"Close","Volume"
],
"sequence_length": 50,
"train_test_split": 0.85,
"normalise": True
},
"training": {
"epochs": 2,
"batch_size": 32
},
"model": {
"loss": "mse",
"optimizer": "adam",
"layers": [
{
"type": "lstm",
"neurons": 100,
"input_timesteps": 49,
"input_dim": 2,
"return_seq": True
},
{
"type": "dropout",
"rate": 0.2
},
{
"type": "lstm",
"neurons": 100,
"return_seq": True
},
{
"type": "lstm",
"neurons": 100,
"return_seq": False
},
{
"type": "dropout",
"rate": 0.2
},
{
"type": "dense",
"neurons": 1,
"activation": "linear"
}
]
}
}
以下是我构建模型的方式:
data = DataLoader(
os.path.join(configs['data']['filename']),
configs['data']['train_test_split'],
configs['data']['columns']
)
model = Model()
model.build_model(configs)
# out-of memory generative training
steps_per_epoch = math.ceil((data.len_train - configs['data']['sequence_length']) / configs['training']['batch_size'])
modelhistory=model.train_generator(
data_gen = data.generate_train_batch(
seq_len = configs['data']['sequence_length'],
batch_size = configs['training']['batch_size'],
normalise = configs['data']['normalise'],
epochs = configs['training']['epochs']
),
epochs = configs['training']['epochs'],
batch_size = configs['training']['batch_size'],
steps_per_epoch = steps_per_epoch
)
请帮忙。
请您参考如下方法:
好的,我想我知道你的问题是什么(或曾经是什么)。
因此,在 generate_train_batch 中,这一行:
if i >= (self.len_train - seq_len):
检查何时将计数器重置为 0,但不向批处理中添加任何其他内容以进行计算。当您使用 2 个时期和 0.85 分割运行它时,恰好当该行被执行时(就在第二个时期的开始),它仍然是该批处理生成的开始(即 i =67104 所以 if i >= (self.len_train - seq_len): 一旦 for 循环开始,条件就满足了)。所以 i 永远不会增加,所以你的批处理是空的。
对于所有其他配置,此错误不会发生,因为上述特定情况不会发生。在你这边,为了确保这不会发生,我建议只删除上面提到的 if 语句之后的 yield np.array(x_batch), np.array(y_batch) 行。如果您的最后一批重复使用一些第一批元素也没关系。我认为这是解决此问题的最简单方法。
