这段代码尝试使用自定义的dropout实现：
%reset -fimport torchimport torch.nn as nn# import torchvision# import torchvision.transforms as transformsimport torchimport torch.nn as nnimport torch.utils.data as data_utilsimport numpy as npimport matplotlib.pyplot as pltimport torch.nn.functional as Fnum_epochs = 1000number_samples = 10from sklearn.datasets import make_moonsfrom matplotlib import pyplotfrom pandas import DataFrame# 生成2D分类数据集X, y = make_moons(n_samples=number_samples, noise=0.1)# 散点图，点按类别值着色x_data = [a for a in enumerate(X)]x_data_train = x_data[:int(len(x_data) * .5)]x_data_train = [i[1] for i in x_data_train]x_data_trainy_data = [y[i[0]] for i in x_data]y_data_train = y_data[:int(len(y_data) * .5)]y_data_trainx_test = [a[1] for a in x_data[::-1][:int(len(x_data) * .5)]]y_test = [a for a in y_data[::-1][:int(len(y_data) * .5)]]x = torch.tensor(x_data_train).float() # <2>print(x)y = torch.tensor(y_data_train).long()print(y)x_test = torch.tensor(x_test).float()print(x_test)y_test = torch.tensor(y_test).long()print(y_test)class Dropout(nn.Module): def __init__(self, p=0.5, inplace=False):# print(p) super(Dropout, self).__init__() if p < 0 or p > 1: raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p)) self.p = p self.inplace = inplace def forward(self, input): print(list(input.shape)) return np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1-dropout_percent)[0] * (1.0/(1-self.p)) def __repr__(self): inplace_str = ', inplace' if self.inplace else '' return self.__class__.__name__ + '(' \ + 'p=' + str(self.p) \ + inplace_str + ')'class MyLinear(nn.Linear): def __init__(self, in_feats, out_feats, drop_p, bias=True): super(MyLinear, self).__init__(in_feats, out_feats, bias=bias) self.custom_dropout = Dropout(p=drop_p) def forward(self, input): dropout_value = self.custom_dropout(self.weight) return F.linear(input, dropout_value, self.bias)my_train = data_utils.TensorDataset(x, y)train_loader = data_utils.DataLoader(my_train, batch_size=2, shuffle=True)my_test = data_utils.TensorDataset(x_test, y_test)test_loader = data_utils.DataLoader(my_train, batch_size=2, shuffle=True)# 设备配置device = 'cpu'print(device)# 超参数 input_size = 2hidden_size = 100num_classes = 2learning_rate = 0.0001pred = []# 带有一个隐藏层的全连接神经网络class NeuralNet(nn.Module): def __init__(self, input_size, hidden_size, num_classes, p): super(NeuralNet, self).__init__()# self.drop_layer = nn.Dropout(p=p)# self.drop_layer = MyLinear()# self.fc1 = MyLinear(input_size, hidden_size, p) self.fc1 = MyLinear(input_size, hidden_size , p) self.relu = nn.ReLU() self.fc2 = nn.Linear(hidden_size, num_classes) def forward(self, x):# out = self.drop_layer(x) out = self.fc1(x) out = self.relu(out) out = self.fc2(out) return outmodel = NeuralNet(input_size, hidden_size, num_classes, p=0.9).to(device)# 损失函数和优化器criterion = nn.CrossEntropyLoss()optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) # 训练模型total_step = len(train_loader)for epoch in range(num_epochs): for i, (images, labels) in enumerate(train_loader): # 将张量移动到配置的设备上 images = images.reshape(-1, 2).to(device) labels = labels.to(device) # 前向传播 outputs = model(images) loss = criterion(outputs, labels) # 反向传播和优化 optimizer.zero_grad() loss.backward() optimizer.step() if (epoch) % 100 == 0: print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, i+1, total_step, loss.item()))自定义dropout的实现如下：
class Dropout(nn.Module): def __init__(self, p=0.5, inplace=False):# print(p) super(Dropout, self).__init__() if p < 0 or p > 1: raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p)) self.p = p self.inplace = inplace def forward(self, input): print(list(input.shape)) return np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1-dropout_percent)[0] * (1.0/(1-self.p)) def __repr__(self): inplace_str = ', inplace' if self.inplace else '' return self.__class__.__name__ + '(' \ + 'p=' + str(self.p) \ + inplace_str + ')'class MyLinear(nn.Linear): def __init__(self, in_feats, out_feats, drop_p, bias=True): super(MyLinear, self).__init__(in_feats, out_feats, bias=bias) self.custom_dropout = Dropout(p=drop_p) def forward(self, input): dropout_value = self.custom_dropout(self.weight) return F.linear(input, dropout_value, self.bias)我似乎错误地实现了dropout函数？
np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1-dropout_percent)[0] * (1.0/(1-self.p))如何修改以正确使用dropout？
以下帖子在达到这一点时很有帮助：
Hinton的Dropout用三行Python实现：https://iamtrask.github.io/2015/07/28/dropout/
创建自定义Dropout函数：https://discuss.pytorch.org/t/making-a-custom-dropout-function/14053/2
回答：
我似乎错误地实现了dropout函数？
np.random.binomial([np.ones((len(input),np.array(list(input.shape))))],1 dropout_percent)[0] * (1.0/(1-self.p))
实际上，以上实现被称为反向Dropout。反向Dropout是各种深度学习框架中实际实现Dropout的方式。
什么是反向Dropout？
在深入了解反向Dropout之前，了解Dropout如何对单个神经元起作用可能会有所帮助：
由于在训练阶段，神经元以概率q（=1-p）保持开启，因此在测试阶段，我们需要模拟训练阶段使用的网络集合的行为。为此，作者建议在测试阶段通过因子q缩放激活函数，以便使用训练阶段产生的预期输出作为测试阶段所需的单一输出（第10节，多重高斯噪声）。因此：
反向Dropout有点不同。这种方法在训练阶段缩放激活，测试阶段保持不变。缩放因子是保持概率的倒数1/1-p = 1/q，因此：
反向Dropout有助于定义模型一次，只需更改一个参数（保持/丢弃概率）即可在同一模型上运行训练和测试。相反，直接Dropout迫使你在测试阶段修改网络，因为如果你不乘以q，神经元将产生比后续神经元预期的高值（因此后续神经元可能会饱和或爆炸）：这就是为什么反向Dropout是更常见的实现方式。
参考资料：
如何在Pytorch中实现反向Dropout？
class MyDropout(nn.Module): def __init__(self, p: float = 0.5): super(MyDropout, self).__init__() if p < 0 or p > 1: raise ValueError("dropout probability has to be between 0 and 1, " "but got {}".format(p)) self.p = p def forward(self, X): if self.training: binomial = torch.distributions.binomial.Binomial(probs=1-self.p) return X * binomial.sample(X.size()) * (1.0/(1-self.p)) return X如何在Numpy中实现？
import numpy as nppKeep = 0.8weights = np.ones([1, 5])binary_value = np.random.rand(weights.shape[0], weights.shape[1]) < pKeepres = np.multiply(weights, binary_value)res /= pKeep # 这一行称为反向Dropout技术print(res)如何在Tensorflow中实现？
import tensorflow as tftf.enable_eager_execution()weights = tf.ones(shape=[1, 5])keep_prob = 0.8random_tensor = keep_probrandom_tensor += tf.random_uniform(weights.shape)# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)binary_tensor = tf.floor(random_tensor)ret = tf.div(weights, keep_prob) * binary_tensorprint(ret)