實現偏倚神經網絡神經網絡

Question

我實現了具有梯度下降的神經網絡的偏倚單位。但我不是100％確定如果我已經以正確的方式實施它。如果你可以快速查看我的代碼，那麼將是空曠的。只有

如果偏置部分：

是很重要的。

而我的第二個問題： 不應該softmax函數的導數爲1-x，因爲x是softmax函數的輸出嗎？ 我用1-x試過了我的網，但其性能更差。

每一個幫助表示讚賞。 在此先感謝。

import numpy as np 
import pickle 
import time 
import math 

class FeedForwardNetwork(): 

    def __init__(self, input_dim, hidden_dim, output_dim, dropout=False, dropout_prop=0.5, bias=False): 
     np.random.seed(1) 
     self.input_layer = np.array([]) 
     self.hidden_layer = np.array([]) 
     self.output_layer = np.array([]) 
     self.hidden_dim = hidden_dim 
     self.dropout = dropout 
     self.dropout_prop = dropout_prop 
     self.bias = bias 

     r_input_hidden = math.sqrt(6/(input_dim + hidden_dim)) 
     r_hidden_output = math.sqrt(6/(hidden_dim + output_dim)) 

     #self.weights_input_hidden = np.random.uniform(low=-r_input_hidden, high=r_input_hidden, size=(input_dim, hidden_dim)) 
     #self.weights_hidden_output = np.random.uniform(low=-r_hidden_output, high=r_hidden_output, size=(hidden_dim, output_dim)) 

     self.weights_input_hidden = np.random.uniform(low=-0.01, high=0.01, size=(input_dim, hidden_dim)) 
     self.weights_hidden_output = np.random.uniform(low=-0.01, high=0.01, size=(hidden_dim, output_dim)) 

     self.validation_data = np.array([]) 
     self.validation_data_solution = np.array([]) 

     self.velocities_input_hidden = np.zeros(self.weights_input_hidden.shape) 
     self.velocities_hidden_output = np.zeros(self.weights_hidden_output.shape) 

     if bias: 
      self.weights_bias_hidden = np.random.uniform(low=-0.01, high=0.01, size=((1, hidden_dim))) 
      self.weights_bias_output = np.random.uniform(low=-0.01, high=0.01, size=((1, output_dim))) 
      self.velocities_bias_hidden = np.zeros(self.weights_bias_hidden.shape) 
      self.velocities_bias_output = np.zeros(self.weights_bias_output.shape) 

    def _tanh(self, x, deriv=False): 
     #The derivate is: 1-np.tanh(x)**2; Because x is already the output of tanh(x) 1-x*x is the correct derivate. 
     if not deriv: 
      return np.tanh(x) 
     return 1-x*x 

    def _softmax(self, x, deriv=False): 
     if not deriv: 
      return np.exp(x)/np.sum(np.exp(x), axis=0) 
     return 1 - np.exp(x)/np.sum(np.exp(x), axis=0) 

    def set_training_data(self, training_data_input, training_data_target, validation_data_input=None, validation_data_target=None): 
     """Splits the data up into training and validation data with a ratio of 0.85/0.15 if no validation data is given. 
     Sets the data for training.""" 
     if len(training_data_input) != len(training_data_target): 
      raise ValueError(
       'Number of training examples and' 
       ' training targets does not match!' 
      ) 
     if (validation_data_input is None) and (validation_data_target is None): 
      len_training_data = int((len(training_data_input)/100*85//1)) 
      self.input_layer = training_data_input[:len_training_data] 
      self.output_layer = training_data_target[:len_training_data] 
      self.validation_data = training_data_input[len_training_data:] 
      self.validation_data_solution = training_data_target[len_training_data:] 
     else: 
      self.input_layer = training_data_input 
      self.output_layer = training_data_target 
      self.validation_data = validation_data_input 
      self.validation_data_solution = validation_data_target 

    def save(self, filename): 
     """Saves the weights into a pickle file.""" 
     with open(filename, "wb") as network_file: 
      pickle.dump(self.weights_input_hidden, network_file) 
      pickle.dump(self.weights_hidden_output, network_file) 

    def load(self, filename): 
     """Loads network weights from a pickle file.""" 
     with open(filename, "rb") as network_file: 
      weights_input_hidden = pickle.load(network_file) 
      weights_hidden_output = pickle.load(network_file) 

     if (
      len(weights_input_hidden) != len(self.weights_input_hidden) 
      or len(weights_hidden_output) != len(self.weights_hidden_output) 
     ): 
      raise ValueError(
       'File contains weights that does not' 
       ' match the current networks size!' 
      )   
     self.weights_input_hidden = weights_input_hidden 
     self.weights_hidden_output = weights_hidden_output 

    def measure_error(self, input_data, output_data): 
     return 1/2 * np.sum((output_data - self.forward_propagate(input_data))**2) 
     #return np.sum(np.nan_to_num(-output_data*np.log(self.forward_propagate(input_data))-(1-output_data)*np.log(1-self.forward_propagate(input_data)))) 

    def forward_propagate(self, input_data, dropout=False): 
     """Proceds the input data from input neurons up to output neurons and returns the output layer. 
      If dropout is True some of the neurons are randomly turned off.""" 
     input_layer = input_data 
     self.hidden_layer = self._tanh(np.dot(input_layer, self.weights_input_hidden)) 
     if self.bias: 
      self.hidden_layer += self.weights_bias_hidden 
     if dropout: 
      self.hidden_layer *= np.random.binomial([np.ones((len(input_data),self.hidden_dim))],1-self.dropout_prop)[0] * (1.0/(1-self.dropout_prop)) 
     if self.bias: 
      return self._softmax((np.dot(self.hidden_layer, self.weights_hidden_output) + self.weights_bias_output).T).T 
     else: 
      return self._softmax(np.dot(self.hidden_layer, self.weights_hidden_output).T).T 
     #return self._softmax(output_layer.T).T 

    def back_propagate(self, input_data, output_data, alpha, beta, momentum): 
     """Calculates the difference between target output and output and adjusts the weights to fit the target output better. 
      The parameter alpha is the learning rate. 
      Beta is the parameter for weight decay which penaltizes large weights.""" 
     sample_count = len(input_data) 
     output_layer = self.forward_propagate(input_data, dropout=self.dropout) 
     output_layer_error = output_layer - output_data 
     output_layer_delta = output_layer_error * self._softmax(output_layer, deriv=True) 
     print("Error: ", np.mean(np.abs(output_layer_error))) 
     #How much did each hidden neuron contribute to the output error? 
     #Multiplys delta term with weights 
     hidden_layer_error = output_layer_delta.dot(self.weights_hidden_output.T) 

     #If the prediction is good, the second term will be small and the change will be small 
     #Ex: target: 1 -> Slope will be 1 so the second term will be big 
     hidden_layer_delta = hidden_layer_error * self._tanh(self.hidden_layer, deriv=True) 
     #The both lines return a matrix. A row stands for all weights connected to one neuron. 
     #E.g. [1, 2, 3] -> Weights to Neuron A 
     #  [4, 5, 6] -> Weights to Neuron B 
     hidden_weights_gradient = input_data.T.dot(hidden_layer_delta)/sample_count 
     output_weights_gradient = self.hidden_layer.T.dot(output_layer_delta)/sample_count 
     velocities_input_hidden = self.velocities_input_hidden 
     velocities_hidden_output = self.velocities_hidden_output 

     self.velocities_input_hidden = velocities_input_hidden * momentum - alpha * hidden_weights_gradient 
     self.velocities_hidden_output = velocities_hidden_output * momentum - alpha * output_weights_gradient 

     #Includes momentum term and weight decay; The weight decay parameter is beta 
     #Weight decay penalizes large weights to prevent overfitting 
     self.weights_input_hidden += -velocities_input_hidden * momentum + (1 + momentum) * self.velocities_input_hidden 
     - alpha * beta * self.weights_input_hidden/sample_count 
     self.weights_hidden_output += -velocities_hidden_output * momentum + (1 + momentum) * self.velocities_hidden_output 
     - alpha * beta * self.weights_hidden_output/sample_count 

     if self.bias: 
      velocities_bias_hidden = self.velocities_bias_hidden 
      velocities_bias_output = self.velocities_bias_output 
      hidden_layer_delta = np.sum(hidden_layer_delta, axis=0) 
      output_layer_delta = np.sum(output_layer_delta, axis=0) 
      self.velocities_bias_hidden = velocities_bias_hidden * momentum - alpha * hidden_layer_delta 
      self.velocities_bias_output = velocities_bias_output * momentum - alpha * output_layer_delta 

      self.weights_bias_hidden += -velocities_bias_hidden * momentum + (1 + momentum) * self.velocities_bias_hidden 
      - alpha * beta * self.weights_bias_hidden/sample_count 
      self.weights_bias_output += -velocities_bias_output * momentum + (1 + momentum) * self.velocities_bias_output 
      - alpha * beta * self.weights_bias_output/sample_count 

    def batch_train(self, epochs, alpha, beta, momentum, patience=10): 
     """Trains the network in batch mode that means the weights are updated after showing all training examples. 
      alpha is the learning rate and patience is the number of epochs that the validation error is allowed to increase before aborting. 
      Beta is the parameter for weight decay which penaltizes large weights.""" 
     #The weight decay parameter is beta 
     validation_error = self.measure_error(self.validation_data, self.validation_data_solution) 
     for epoch in range(epochs): 
      self.back_propagate(self.input_layer, self.output_layer, alpha, beta, momentum) 
      validation_error_new = self.measure_error(self.validation_data, self.validation_data_solution) 
      if validation_error_new < validation_error: 
       validation_error = validation_error_new 
      else: 
       patience -= 1 
       if patience == 0: 
        print("Abort Training. Overfitting has started! Epoch: {0}. Error: {1}".format(epoch, validation_error_new)) 
        return 
      print("Epoch: {0}, Validation Error: {1}".format(epoch, validation_error)) 
      self.save("Network_Mnist.net") 

    def mini_batch_train(self, batch_size, epochs, alpha, beta, momentum, patience=10): 
     """Trains the network in mini batch mode, that means the weights are updated after showing only a bunch of training examples. 
      alpha is the learning rate and patience is the number of epochs that the validation error is allowed to increase before aborting.""" 
     validation_error = self.measure_error(self.validation_data, self.validation_data_solution) 
     sample_count = len(self.input_layer) 
     epoch_counter = 0 
     for epoch in range(0, epochs*batch_size, batch_size): 
      epoch_counter += 1 
      self.back_propagate(self.input_layer[epoch%sample_count:(epoch%sample_count)+batch_size], 
           self.output_layer[epoch%sample_count:(epoch%sample_count)+batch_size], alpha, beta, momentum) 
      validation_error_new = self.measure_error(self.validation_data, self.validation_data_solution) 
      if validation_error_new < validation_error: 
       validation_error = validation_error_new 
       patience = 20 
      else: 
       patience -= 1 
       if patience == 0: 
        print("Abort Training. Overfitting has started! Epoch: {0}. Error: {1}".format(epoch_counter, validation_error_new)) 
        return 
      print("Epoch: {0}, Validation Error: {1}".format(epoch_counter, validation_error)) 
      self.save("Network_Mnist.net")    

if __name__ == "__main__": 
    #If the first row is a one the first output neuron should be on the second off 
    x = np.array([ [0, 0, 1, 1, 0], 
        [0, 1, 1, 1, 1], 
        [1, 0, 1, 1, 1], 
        [1, 1, 1, 1, 0], 
        [0, 1, 1, 1, 0], 
        [1, 1, 0, 0, 0], 
        [1, 1, 0, 0, 0], 
        [1, 0, 1, 0, 0] ]) 

    y = np.array([ [0, 1], 
        [0, 1], 
        [1, 0], 
        [1, 0], 
        [0, 1], 
        [1, 0], 
        [1, 0], 
        [1, 0] ]) 

    #x = np.array([ [0, 0, 1, 1] ]) 
    #y = np.array([[0]]).T 

    a = FeedForwardNetwork(input_dim=5, hidden_dim=200, output_dim=2, bias=False) 
    a.set_training_data(x, y) 
    start = time.time() 
    a.batch_train(epochs=2000, alpha=0.05, beta=0.0001, momentum=0.99, patience=20) 
    print(time.time()-start) 

Answer 1

與衍生關係...

如果您使用的是tanh激活功能，即the derivative is:y' = 1 - y^2。通常使用tanh，因爲它是零中心的。

如果使用邏輯方程，則the derivative is:y' = y(1+y)。 softmax有一個similar derivative。

好的是，所有這些可以表示爲自己的功能，所以你需要看看def _softmax(self, x, deriv=False)函數，以類似的方式定義它，而不是def _tanh(self, x, deriv=False)。