交叉验证
for epoch in range(500):
idx = tf.range(60000)
idx = tf.random.shuffle(idx)
x_train, y_train = tf.gather(x, idx[:50000]), tf.gather(y, idx[:50000])
x_val, y_val = tf.gather(x, idx[-10000:]), tf.gather(y, idx[-10000:])
db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
db_train.map(prepocess).shuffle(50000).batch(128)
db_val = tf.data.Dataset.from_tensor_slices((x_val, y_val))
db_val.map(prepocess).shuffle(50000).batch(128)
network.fit(db_train_val, epochs=6, validation_split=0.1, validation_freq=2)
Regularization
l2_model = keras.models.Sequential([
keras.layers.Dense(16, kernel_regularizer=keras.regularizers_l2(0.001), activation=tf.nn.relu, input_shape=(NUM_WORDS,)),
keras.layers.Dense(16, kernel_regularizer=keras.regularizers_l2(0.001), activation=tf.nn.relu),
keras.layers.Dense(1, activation=tf.nn.sigmoid)
])
for step, (x, y) in enumerate(db):
with tf.GradientTape() as tape:
loss = tf.reduce_mean(tf.losses.categorical_crossent ropy(y_onehot, out, from_logits=True))
loss_regularization = []
for p in network.trainable_variables:
loss_regularization.append(tf.nn.l2_loss(p))
loss_regularization = tf.reduce_sum(tf.stack(loss_regularization))
loss = loss + 0.0001 * loss_regularization
grads = tape.gradient(loss, network.trainable_variables)
optimizer.apply_gradients(zip(grads, network.trainable_variables))
动量与学习率
optimizer = optimizers.SGD(learning_rate=0.02, momentum=0.9)
optimizer = optimizers.RMSprop(learning_rate=0.02, momentum=0.9)
optimizer = optimizers.Adam(learning_rate=0.02, beta_1=0.9, beta_2=0.999)
optimizer = optimizers.SGD(learning_rate=0.2)
for epoch in range(100):
optimizer.learning_rate = 0.2 * (100-epoch)/100
Dropout
network = Sequential([layers.Dense(256, activation='relu'),
layers.Dropout(0.5),
layers.Dense(128, activation='relu'),
layers.Dropout(0.5),
layers.Dense(64, activation='relu'),
layers.Dense(32, activation='relu'),
layers.Dense(10)])
for step, (x, y) in enumerate(db):
with tf.GradientTape() as tape:
x = tf.reshape(x, (-1, 28*28))
out = network(x, training=True)
out = network(x, training=False)