我使用theano编写了一个MLP分类器。使用反向传播算法的训练函数如下所示：

self.weights=[theano.shared(numpy.random.random((network.architecture[i+1],network.architecture[i]))) for i in range(len(network.architecture)-1)]self.bias=[theano.shared(numpy.random.random(network.architecture[i+1])) for i in range(len(network.architecture)-1)]self.layers=network.layersself.prev_rate=[theano.shared(numpy.zeros((network.architecture[i+1],network.architecture[i]))) for i in range(len(network.architecture)-1)]+[theano.shared(numpy.zeros(network.architecture[i+1])) for i in range(len(network.architecture)-1)]prediction=T.dmatrix()output=T.dmatrix()reg_lambda=T.dscalar()alpha=T.dscalar()momentum=T.dscalar()cost=T.nnet.categorical_crossentropy(prediction,output).mean()for i,j in zip(self.weights,self.bias):    cost+=T.sum(i**2)*reg_lambda    cost+=T.sum(j**2)*reg_lambdaparameters=self.weights+self.biasrates=[(alpha*T.grad(cost,parameter)+momentum*prev_rate) for parameter,prev_rate in zip(parameters,self.prev_rate)]updates=[(weight,weight-rate) for weight,rate in zip(parameters,rates)]+[(prev_rate,rate) for prev_rate,rate in zip(self.prev_rate,rates)]self.backprop=theano.function([prediction,output,reg_lambda,alpha,momentum],cost,updates=updates)

我尝试对XOR问题进行训练，实现在于：

network=FeedForwardNetwork([2,2,2])network.initialize()network.train(numpy.array([[0.,0.],[0.,1.],[1.,0.],[1.,1.],[0.,0.],[0.,1.],[1.,0.],[1.,1.]]),numpy.array([[0.,1.],[1.,0.],[1.,0.],[0.,1.],[0.,1.],[1.,0.],[1.,0.],[0.,1.]]),alpha=0.01,epochs=1000000000000000,momentum=0.9)print network.predict(numpy.array([[1.,0.]]))print network.predict(numpy.array([[0.,0.]]))

initialize()方法只是在后端编译所有函数，即反向传播函数、前向传递函数用于计算预测以及其他几个theano函数。现在，当我运行这段代码时，训练过程停滞在局部最小值。

0.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.693147180560.69314718056

在训练开始时，损失大约为0.92。它稳步下降到上述值并停在那里。我尝试更改alpha和动量的数值。我做错了什么？

附注：完整代码在这里：networks.py

import theanoimport theano.tensor as Timport numpyfrom layers import *from backend import NetworkBackendclass Network:    def __init__(self,architecture):        self.architecture=architecture        self.layers=[]        self.weights=[]        self.bias=[]    def __str__(self):        banner=''        for i in range(len(self.weights)):            banner+=str(self.weights[i])+'\n'            banner+=str(self.bias[i])+'\n'        return bannerclass FeedForwardNetwork(Network):    def initialize(self):        self.layers.append(InputLayer(units=self.architecture[0]))        for i in range(1,len(self.architecture[:-1])):            self.layers.append(SigmoidLayer(units=self.architecture[i]))        self.layers.append(SoftmaxLayer(units=self.architecture[-1]))        self.backend=NetworkBackend(self)    def predict(self,inputs):        return self.backend.activate(inputs)    def train(self,X,y,alpha=100,reg_lambda=0.0001,epochs=10000,momentum=0.9):        cost=1        while cost>0.01 and epochs:            prediction=self.predict(X)            cost=self.backend.backprop(prediction,y,reg_lambda,alpha,momentum)            print cost            epochs-=1if __name__=='__main__':    network=FeedForwardNetwork([2,2,2])    network.initialize()    network.train(numpy.array([[0.,0.],[0.,1.],[1.,0.],[1.,1.],[0.,0.],[0.,1.],[1.,0.],[1.,1.]]),numpy.array([[0.,1.],[1.,0.],[1.,0.],[0.,1.],[0.,1.],[1.,0.],[1.,0.],[0.,1.]]),alpha=0.01,epochs=1000000000000000,momentum=0.9)    print network.predict(numpy.array([[1.,0.]]))    print network.predict(numpy.array([[0.,0.]]))

layers.py

import theanoimport theano.tensor as Timport scipyfrom backend import ComputationBackendclass Layer:    def __init__(self,units):        self.units=units        self.backend=ComputationBackend()    def __str__(self):        banner=self.__class__.__name__        banner+=" Units:%d"%self.units        return bannerclass SigmoidLayer(Layer):    def forwardPass(self,inputs):        return self.backend.sigmoid(inputs)class InputLayer(Layer):    def forwardPass(self,inputs):        return inputsclass SoftmaxLayer(Layer):    def forwardPass(self,inputs):        return self.backend.softmax(inputs)

backend.py

import theanoimport theano.tensor as Timport numpyclass NetworkBackend:    def __init__(self,network):        # 初始化共享变量        self.weights=[theano.shared(numpy.random.random((network.architecture[i+1],network.architecture[i]))) for i in range(len(network.architecture)-1)]        self.bias=[theano.shared(numpy.random.random(network.architecture[i+1])) for i in range(len(network.architecture)-1)]        self.layers=network.layers        self.prev_rate=[theano.shared(numpy.zeros((network.architecture[i+1],network.architecture[i]))) for i in range(len(network.architecture)-1)]+[theano.shared(numpy.zeros(network.architecture[i+1])) for i in range(len(network.architecture)-1)]        # 网络层的激活        inputs=T.dmatrix()        temp=self.layers[0].forwardPass(inputs)        for i in range(1,len(self.layers[:-1])):            temp=self.layers[i].forwardPass(T.dot(temp,self.weights[i-1].transpose())+self.bias[i-1])        output=self.layers[-1].forwardPass(T.dot(temp,self.weights[-1].transpose())+self.bias[-1])        self.activate=theano.function([inputs],output)        prediction=T.dmatrix()        output=T.dmatrix()        reg_lambda=T.dscalar()        alpha=T.dscalar()        momentum=T.dscalar()        cost=T.nnet.categorical_crossentropy(prediction,output).mean()        for i,j in zip(self.weights,self.bias):            cost+=T.sum(i**2)*reg_lambda            cost+=T.sum(j**2)*reg_lambda        parameters=self.weights+self.bias        rates=[(alpha*T.grad(cost,parameter)+momentum*prev_rate) for parameter,prev_rate in zip(parameters,self.prev_rate)]        updates=[(weight,weight-rate) for weight,rate in zip(parameters,rates)]+[(prev_rate,rate) for prev_rate,rate in zip(self.prev_rate,rates)]        self.backprop=theano.function([prediction,output,reg_lambda,alpha,momentum],cost,updates=updates)class ComputationBackend:    def __init__(self):        # sigmoid激活        self.sigmoid=T.nnet.sigmoid        # softmax激活        self.softmax=T.nnet.softmax

回答：

终于搞明白了！在NetworkBackend中，计算成本时，我计算的是期望输出与作为theano函数参数传递的预测之间的交叉熵，而不是使用activate函数计算的预测。因此，theano图不包含前向传递。结果，theano.tensor.grad只能找到正则化函数的梯度，而不能找到实际成本函数的梯度！所以正确的实现应该是：

inputs=T.dmatrix()temp=self.layers[0].forwardPass(inputs)for i in range(1,len(self.layers[:-1])):    temp=self.layers[i].forwardPass(T.dot(temp,self.weights[i-1].transpose())+self.bias[i-1])output=self.layers[-1].forwardPass(T.dot(temp,self.weights[-1].transpose())+self.bias[-1])self.activate=theano.function([inputs],output)label=T.dmatrix()reg_lambda=T.dscalar()alpha=T.dscalar()momentum=T.dscalar()cost=T.nnet.categorical_crossentropy(output,label).mean()for i,j in zip(self.weights,self.bias):    cost+=T.sum(i**2)*reg_lambda    cost+=T.sum(j**2)*reg_lambdaparameters=self.weights+self.biasrates=[(alpha*T.grad(cost,parameter)+momentum*prev_rate) for parameter,prev_rate in zip(parameters,self.prev_rate)]updates=[(weight,weight-rate) for weight,rate in zip(parameters,rates)]+[(prev_rate,rate) for prev_rate,rate in zip(self.prev_rate,rates)]self.backprop=theano.function([inputs,label,reg_lambda,alpha,momentum],cost,updates=updates)

所以，我没有声明一个新的预测矩阵，而是使用训练函数中的输入，并使用与激活函数中相同的方程计算预测。这完成了theano图，现在theano.tensor.grad()可以计算成本函数和正则化的梯度了。