Toy Neural Networks in Processing

vadsavin · July 5, 2018, 7:02pm

I saw every Coding Train’s video about NN with P5 and i’ve tried to export his code into Prcossessing. Actually, it seems to work correctly with only one output node, otherwise it makes itself crazy.
I’ve rewrite this code for about two time and I havent so the mistake yet : (
The NN here should solve something like XOR problem, but with two colors, so it doesnt.
Waiting for your help
Sorry for my awful English

NeuralNetwork nn = new NeuralNetwork(2,5,2);
float[][] data = {{0,0,0,1},{1,1,1,0}}; //1st two - inputs; others - targets;

void setup(){
  size(600,600);
} 
 
void draw(){
   int rez = 20;
   for(int n =0; n < 1000; n++){
     int r = (int)random(data.length);
     float[] d = {data[r][0],data[r][1]};
     float[] t = {data[r][2],data[r][3]};
     nn.train(d,t);
   }
   
   noStroke();
   for(float y = 0; y < height; y+=rez){
     for(int x = 0; x < width; x+=rez){
      float[] d = {x/float(width), y/float(height)};
      fill(0, nn.feedForward(d)[0]*255, nn.feedForward(d)[1]*255 );
      rect(x,y,rez,rez);
     }
   }
 
class NeuralNetwork{
  int ni, nh, no, nh2;
  Matrix wih;
  Matrix who;
  Matrix bh;
  Matrix bo;
  
  float learning_rate = 0.01;
  int layers;
  
  NeuralNetwork(int i, int h, int o){
    ni = i;
    nh = h;
    no = o;
    
    wih = new Matrix(nh,ni);
    who = new Matrix(no,nh);
    
    bh = new Matrix(nh,1);
    bo = new Matrix(no,1);
    
    wih.randomize();
    who.randomize();
    bh.randomize();
    bo.randomize();
    
    layers = 3;
  }
  
  
  
  float[] feedForward(float[] input){
   
      Matrix inputs = Matrix.fromArray(input);
      Matrix h = Matrix.multiply(wih,inputs);
      h.addMat(bh);
      h.activate();
      
      Matrix guess = Matrix.multiply(who,h);
      guess.addMat(bo);
      guess.activate();
      
      return guess.toArray();
      
   
  }
  
  void train(float[] input, float[] target){
     Matrix targets = Matrix.fromArray(target);
      
      Matrix inputs = Matrix.fromArray(input);
      Matrix h = Matrix.multiply(wih,inputs);
      h.addMat(bh);
      h.activate();
      
      Matrix outputs = Matrix.multiply(who,h);
      outputs.addMat(bo);
      outputs.activate();
      
      Matrix outputs_er = Matrix.substruct(targets,outputs);
      Matrix gradients = Matrix.dsig(outputs);
      gradients = Matrix.multiply(gradients,outputs_er);
      gradients.multiply(learning_rate);
      
      Matrix hidden_T = Matrix.transpose(h);
      Matrix weight_ho_deltas = Matrix.multiply(gradients, hidden_T);
      who.addMat(weight_ho_deltas);
      bo.addMat(gradients);
      
      Matrix who_t = Matrix.transpose(who);
      Matrix hidden_errors = Matrix.multiply(who_t, outputs_er);
      
      Matrix hidden_gradient = Matrix.dsig(h);
      hidden_gradient = Matrix.multiply(hidden_gradient,hidden_errors);
      hidden_gradient.multiply(learning_rate);
      
      Matrix inputs_T = Matrix.transpose(inputs);
      Matrix weight_ih_deltas = Matrix.multiply(hidden_gradient, inputs_T);
      
      wih.addMat(weight_ih_deltas);
      bh.addMat(hidden_gradient);
  }
  

}

static class Matrix {
  int rows;
  int cols;
  float data[][];

  Matrix(int r, int c) {
    rows = r;
    cols = c;

    data = new float[r][c];
    for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = 0;
      }
    }
  }

  void addMat(Matrix m){
    for (int j = 0; j<m.rows; j++) {
      for (int i = 0; i< m.cols; i++) {
        data[j][i] = data[j][i] + m.data[j][i];
      }
    }
  }
  
  void sqM(){
     for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = sq(data[j][i]);
      }
    }
  }
  
  void randomize() {
    for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = (float)Math.random()*2 - 1;
        //data[j][i] = random(1);
      }
    }
  }
  static Matrix randomize(Matrix a,float x) {
    for (int j = 0; j<a.rows; j++) {
      for (int i = 0; i< a.cols; i++) {
        a.data[j][i] = (float)Math.random()*2*x-x;
      }
    }
    return a;
  }

  void multiply(float n){
    for (int i = 0; i<rows; i++) {
      for (int j = 0; j<cols; j++) {
        data[i][j] = data[i][j] * n;
      }
    }
  }
  
  static Matrix multiply(Matrix a, Matrix b) {
    Matrix result = new Matrix(a.rows, b.cols);

    for (int i = 0; i<result.rows; i++) {
      for (int j = 0; j<result.cols; j++) {

        float sum = 0;
        for (int k = 0; k< a.cols; k++) {
          sum += a.data[i][k]*b.data[k][j];
        }
        result.data[i][j] = sum;
      }
    }

    return result;
  }
  
  static Matrix substruct(Matrix a, Matrix b){
    if(a.rows == b.rows && a.cols == b.cols){
    for (int i = 0; i<a.rows; i++) {
      for (int j = 0; j< a.cols; j++) {
        a.data[i][j] -= b.data[i][j];
      }
    }
    }else{
      println("Matrix error");
    }
    return a;
    
  }
  
  static Matrix transpose(Matrix m){
    Matrix result = new Matrix(m.cols, m.rows);
    
    for (int i = 0; i<m.rows; i++) {
      for (int j = 0; j< m.cols; j++) {
        result.data[j][i] = m.data[i][j];
      }
    }
    
    return result;
  }
  
  void activate(){
   for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = sigmoid(data[j][i]);
      }
    } 
    
  }
  
  static Matrix fromArray(float[] ar){
    Matrix result = new Matrix(ar.length,1);
    for(int i = 0; i< ar.length; i++){
      result.data[i][0] = ar[i];
    }
    return result;
  }
  
  float[] toArray(){
    float[] ar = new float[rows];
    for(int i = 0; i< ar.length; i++){
      ar[i] = data[i][0];
    }
    return ar;
  }
  
  
  
  void printMat(){
    for (int j = 0; j<rows; j++) {
      print("["+j+"]: | ");
      for (int i = 0; i< cols; i++) {
        print(data[j][i] + " | ");
      }
      println();
    } 
  }
  static Matrix dsig(Matrix a){
    Matrix result = new Matrix(a.rows,a.cols);
    for (int j = 0; j<a.rows; j++) {
      for (int i = 0; i< a.cols; i++) {
        result.data[j][i] = (a.data[j][i]) * (1.0 - a.data[j][i]);
        //result.data[j][i] = 1-(a.data[j][i] * a.data[j][i]);
      }
    } 
      
    return result;
  }
}

void saveMat(Matrix a,String name){
      PrintWriter output;
      output = createWriter(name); 
      
      output.print(a.rows + " " + a.cols);
      output.println();
      
      for (int i = 0; i < a.rows; i++) {
          for (int j = 0; j < a.cols; j++) {
            output.print(a.data[i][j] + " ");
         }
         output.println();
      }
      
      output.flush();
      output.close();
    //println("saved");
}

public Matrix loadMat(String name){
  Matrix mat;
  
  BufferedReader reader;
  reader = createReader(name);
  
  String line;
  int cols_, rows_;
  try{
  line = reader.readLine();
  } catch(IOException e){
    e.printStackTrace();
    line = null;
  } 
  
  
  String pieces[] = split(line, " ");
  rows_ = Integer.parseInt(pieces[0]);
  cols_ = Integer.parseInt(pieces[1]);
  
  mat = new Matrix(rows_,cols_);
  
  for(int i = 0; i< rows_; i++){
    try{
    line = reader.readLine();
    } catch(IOException e){
      e.printStackTrace();
      line = null;
    }
    String piecesInLine[] = split(line, " ");
    
      for(int j = 0; j< cols_; j++){
        mat.data[i][j] = parseFloat(piecesInLine[j]);   
      }
    
  }
  
  println("loaded");
  return mat;
}


static float sigmoid(float x){
  return 1.0/(1.0 + (float)Math.exp(-x));
  //return (float)Math.tanh(x);
}



}```

kfrajer · July 5, 2018, 9:26pm

Please provide a link to the repo where you get this code from or a link to Shiffman’s video.

Kf

vadsavin-a · July 6, 2018, 11:20am

https://github.com/CodingTrain/Toy-Neural-Network-JS/blob/master/lib/nn.js - code for NNs using JS (Shiffman’s version)

vadsavin-a · July 10, 2018, 5:56pm

Any ideas? Honestly, i still dont know where te problem is

Trilobyte · July 10, 2018, 7:15pm

Hey Vadsavin,
I have code for writing neural networks in Processing (written before the CodingTrain series, so it’s a bit rough). My code is essentially a template, so you can set your own number of layers with any number of neurons. You would just have to rework the training algorithm (you can probably work backwards off of Mr. Shiffman’s example). PM me if you want the code/an explanation as to how I wrote it. If not, I would say you should start from the top, with a piece of paper, and track how everything works. I assume you have at least moderate experience with Processing and neural networks- tracing neural nets without experience can be quite frustrating. If you do not have that much experience with neural nets, I would suggest doing some research into basic feed-forward networks- this part is just matrix math. The training algorithm is where things get very tricky- study that a lot, and write down the math. If you can follow the math, compare it to Shiffman’s example and your port of the example. Check out this resource- it’s very helpful: http://neuralnetworksanddeeplearning.com/
Best of luck with your project- I know from experience it’s not easy, and Processing doesn’t make it easier at all.

Trilobyte

vadsavin-a · July 12, 2018, 11:34am

Yay, I wonder if you show me your code

vadsavin-a · July 15, 2018, 7:05pm

Man, please, share your code

Trilobyte · July 17, 2018, 2:32pm

Hey Vadsavin,
I sent you a PM a few days ago. Check your inbox by clicking your profile picture in the top right corner, then clicking on the mail symbol. It’s a bit hidden and doesn’t always pop up notifications.

Topic		Replies	Views
Neural network not working (Dan Shiffman, The Nature of Code 10) Coding Questions	3	624	October 7, 2018
Switching javascript neural network to java Coding Questions	17	526	August 6, 2018
Another Neural Network Trainer Gallery	2	311	August 1, 2020
Neural network training Coding Questions	5	729	May 15, 2020
Neural Network help Project Guidance	6	788	August 24, 2018

Toy Neural Networks in Processing

Related topics