我在实现神经网络中的反向传播时遇到了一些问题。这个实现借鉴了Andrew Ng在Coursera上的机器学习课程的幻灯片中的想法（这里是链接 https://www.coursera.org/course/ml）。我认为我已经理解了这个算法，但在代码中存在一些细微的错误。
我使用了一个具有1个输入层、1个隐藏层和1个输出层的网络。它们分别有2 + 1、2 + 1、1个神经元（+1表示偏置）。当我尝试实现逻辑与和逻辑或时，一切运作良好，网络学会了给出正确的值。但当我尝试实现XNOR（a XNOR b = NOT (a XOR b)）时，
我使用了4个示例：
- 0 0 1
- 0 1 0
- 1 0 0
- 1 1 1
但突然间，在这个函数上梯度没有任何变化。开始时，我用随机的小数（从-0.01到0.01）初始化权重。输出接近0.5。然后我进行梯度下降。无论输入是什么，输出始终接近0.5。
我想知道如何解决这个问题。
这是代码：
#include <iostream>#include <fstream>#include <vector>#include <algorithm>// contains matrix and Vector classes.// Vector is just like std::valarray, but is compatible with my matrix.#include "matrix.hpp" size_t L;std::vector< Vector<double> > layers;std::vector< matrix<double> > theta;struct Example{ Vector<double> x; Vector<double> y;};using TrainingSet = std::vector<Example>;TrainingSet examples;double g(double x){ return 1 / (1 + exp(-x));}void forwardPropagate(Vector<double> x){ for ( size_t i = 1; i < layers[0].size(); ++i ) layers[0][i] = x[i - 1]; for ( size_t i = 0; i < L - 1; ++i ) { auto z = theta[i] * layers[i]; for ( size_t j = 1; j < layers[i + 1].size(); ++j ) layers[i + 1][j] = g(z[j - 1]); }}void backwardPropagate(Vector<double> y, std::vector< matrix<double> >& delta){ auto err = layers.back().slice(1) - y; for ( int i = L - 2; i >= 0; --i ) { delta[i] += asMatrix(err) * asMatrix(layers[i]).transpose(); auto gdz = layers[i] * (Vector<double>(layers[i].size(), 1.0) - layers[i]); auto tmp = theta[i].transpose() * err * gdz; err = tmp.slice(1); }}double costFunction(const TrainingSet& examples){ double result = 0.0; for ( const auto& example : examples ) { std::cout << layers.back()[1] << '\n'; forwardPropagate(example.x); for ( size_t k = 1; k < layers.back().size(); ++k ) { auto h = layers.back()[k]; auto y = example.y[k - 1]; result += y * log(h) + (1 - y) * log(1 - h); } } return (-result) / examples.size();}void computeGradient(std::vector< matrix<double> >& delta, const TrainingSet& examples){ for ( auto& m : delta ) m.fillWith(0); for ( auto example : examples ) { forwardPropagate(example.x); backwardPropagate(example.y, delta); } for ( auto& m : delta ) m /= examples.size();}void gradientDescentStep(const std::vector< matrix<double> >& gradient){ const double alpha = 0.01; for ( size_t i = 0; i < L - 1; ++i ) theta[i] -= alpha / examples.size() * gradient[i];}double gradientDescent(const TrainingSet& examples){ const double eps = 0.0000001; double prev, cur; cur = costFunction(examples); size_t iterations = 0; const size_t max_iterations = 200000000; std::vector< matrix<double> > delta; delta.reserve(L - 1); for ( size_t i = 0; i < L - 1; ++i ) delta.emplace_back(theta[i].rows(), theta[i].cols()); do { prev = cur; computeGradient(delta, examples); gradientDescentStep(delta); cur = costFunction(examples); } while ( fabs(cur - prev) >= eps && iterations++ < max_iterations ); std::cout << "Made " << iterations << " iterations\n"; return cur;}int main(){ std::ifstream fin("input.txt"); std::istream& in = fin; std::cout.sync_with_stdio(false); in >> L; std::vector<size_t> architecture(L); for ( size_t i = 0; i < L; ++i ) in >> architecture[i]; layers.reserve(L); for ( size_t i = 0; i < L; ++i ) { layers.emplace_back(1 + architecture[i]); layers.back()[0] = 1; } const double eps = 0.01; theta.reserve(L - 1); for ( size_t i = 0; i < L - 1; ++i ) { theta.emplace_back(layers[i + 1].size() - 1, layers[i].size()); theta[i].randomInitialize(eps); } size_t number_of_examples; in >> number_of_examples; examples.reserve(number_of_examples); for ( size_t i = 0; i < number_of_examples; ++i ) { auto x = Vector<double>(architecture.front()); auto y = Vector<double>(architecture.back()); for ( size_t j = 0; j < architecture.front(); ++j ) in >> x[j]; for ( size_t j = 0; j < architecture.back(); ++j ) in >> y[j]; examples.emplace_back(Example{x, y}); } for ( auto example : examples ) { forwardPropagate(example.x); std::cout << layers.back()[1] << '\n'; } for ( size_t i = 0; i < theta.size(); ++i ) std::cout << "θ[" << i << "] = " << theta[i]; gradientDescent(examples); for ( size_t i = 0; i < theta.size(); ++i ) std::cout << "θ[" << i << "] = " << theta[i]; std::cout << "\n\n\n"; for ( auto example : examples ) { forwardPropagate(example.x); std::cout << layers.back()[1] << '\n'; } return 0;}回答：
最后我终于明白了问题出在哪里。问题不在代码本身。而是这种网络配置在处理XOR时，成本函数存在局部最小值。因此，我陷入了这个局部最小值而无法脱身。
解决方案是朝随机方向迈出一步，直到你脱离局部最小值。这允许你相当快地达到全局最小值。
