Toy Neural Networks in Processing


#1

I saw every Coding Train’s video about NN with P5 and i’ve tried to export his code into Prcossessing. Actually, it seems to work correctly with only one output node, otherwise it makes itself crazy.
I’ve rewrite this code for about two time and I havent so the mistake yet : (
The NN here should solve something like XOR problem, but with two colors, so it doesnt.
Waiting for your help :sunny:
Sorry for my awful English

NeuralNetwork nn = new NeuralNetwork(2,5,2);
float[][] data = {{0,0,0,1},{1,1,1,0}}; //1st two - inputs; others - targets;

void setup(){
  size(600,600);
} 
 
void draw(){
   int rez = 20;
   for(int n =0; n < 1000; n++){
     int r = (int)random(data.length);
     float[] d = {data[r][0],data[r][1]};
     float[] t = {data[r][2],data[r][3]};
     nn.train(d,t);
   }
   
   noStroke();
   for(float y = 0; y < height; y+=rez){
     for(int x = 0; x < width; x+=rez){
      float[] d = {x/float(width), y/float(height)};
      fill(0, nn.feedForward(d)[0]*255, nn.feedForward(d)[1]*255 );
      rect(x,y,rez,rez);
     }
   }
 
class NeuralNetwork{
  int ni, nh, no, nh2;
  Matrix wih;
  Matrix who;
  Matrix bh;
  Matrix bo;
  
  float learning_rate = 0.01;
  int layers;
  
  NeuralNetwork(int i, int h, int o){
    ni = i;
    nh = h;
    no = o;
    
    wih = new Matrix(nh,ni);
    who = new Matrix(no,nh);
    
    bh = new Matrix(nh,1);
    bo = new Matrix(no,1);
    
    wih.randomize();
    who.randomize();
    bh.randomize();
    bo.randomize();
    
    layers = 3;
  }
  
  
  
  float[] feedForward(float[] input){
   
      Matrix inputs = Matrix.fromArray(input);
      Matrix h = Matrix.multiply(wih,inputs);
      h.addMat(bh);
      h.activate();
      
      Matrix guess = Matrix.multiply(who,h);
      guess.addMat(bo);
      guess.activate();
      
      return guess.toArray();
      
   
  }
  
  void train(float[] input, float[] target){
     Matrix targets = Matrix.fromArray(target);
      
      Matrix inputs = Matrix.fromArray(input);
      Matrix h = Matrix.multiply(wih,inputs);
      h.addMat(bh);
      h.activate();
      
      Matrix outputs = Matrix.multiply(who,h);
      outputs.addMat(bo);
      outputs.activate();
      
      Matrix outputs_er = Matrix.substruct(targets,outputs);
      Matrix gradients = Matrix.dsig(outputs);
      gradients = Matrix.multiply(gradients,outputs_er);
      gradients.multiply(learning_rate);
      
      Matrix hidden_T = Matrix.transpose(h);
      Matrix weight_ho_deltas = Matrix.multiply(gradients, hidden_T);
      who.addMat(weight_ho_deltas);
      bo.addMat(gradients);
      
      Matrix who_t = Matrix.transpose(who);
      Matrix hidden_errors = Matrix.multiply(who_t, outputs_er);
      
      Matrix hidden_gradient = Matrix.dsig(h);
      hidden_gradient = Matrix.multiply(hidden_gradient,hidden_errors);
      hidden_gradient.multiply(learning_rate);
      
      Matrix inputs_T = Matrix.transpose(inputs);
      Matrix weight_ih_deltas = Matrix.multiply(hidden_gradient, inputs_T);
      
      wih.addMat(weight_ih_deltas);
      bh.addMat(hidden_gradient);
  }
  

}

static class Matrix {
  int rows;
  int cols;
  float data[][];

  Matrix(int r, int c) {
    rows = r;
    cols = c;

    data = new float[r][c];
    for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = 0;
      }
    }
  }

  void addMat(Matrix m){
    for (int j = 0; j<m.rows; j++) {
      for (int i = 0; i< m.cols; i++) {
        data[j][i] = data[j][i] + m.data[j][i];
      }
    }
  }
  
  void sqM(){
     for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = sq(data[j][i]);
      }
    }
  }
  
  void randomize() {
    for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = (float)Math.random()*2 - 1;
        //data[j][i] = random(1);
      }
    }
  }
  static Matrix randomize(Matrix a,float x) {
    for (int j = 0; j<a.rows; j++) {
      for (int i = 0; i< a.cols; i++) {
        a.data[j][i] = (float)Math.random()*2*x-x;
      }
    }
    return a;
  }

  void multiply(float n){
    for (int i = 0; i<rows; i++) {
      for (int j = 0; j<cols; j++) {
        data[i][j] = data[i][j] * n;
      }
    }
  }
  
  static Matrix multiply(Matrix a, Matrix b) {
    Matrix result = new Matrix(a.rows, b.cols);

    for (int i = 0; i<result.rows; i++) {
      for (int j = 0; j<result.cols; j++) {

        float sum = 0;
        for (int k = 0; k< a.cols; k++) {
          sum += a.data[i][k]*b.data[k][j];
        }
        result.data[i][j] = sum;
      }
    }

    return result;
  }
  
  static Matrix substruct(Matrix a, Matrix b){
    if(a.rows == b.rows && a.cols == b.cols){
    for (int i = 0; i<a.rows; i++) {
      for (int j = 0; j< a.cols; j++) {
        a.data[i][j] -= b.data[i][j];
      }
    }
    }else{
      println("Matrix error");
    }
    return a;
    
  }
  
  static Matrix transpose(Matrix m){
    Matrix result = new Matrix(m.cols, m.rows);
    
    for (int i = 0; i<m.rows; i++) {
      for (int j = 0; j< m.cols; j++) {
        result.data[j][i] = m.data[i][j];
      }
    }
    
    return result;
  }
  
  void activate(){
   for (int j = 0; j<rows; j++) {
      for (int i = 0; i< cols; i++) {
        data[j][i] = sigmoid(data[j][i]);
      }
    } 
    
  }
  
  static Matrix fromArray(float[] ar){
    Matrix result = new Matrix(ar.length,1);
    for(int i = 0; i< ar.length; i++){
      result.data[i][0] = ar[i];
    }
    return result;
  }
  
  float[] toArray(){
    float[] ar = new float[rows];
    for(int i = 0; i< ar.length; i++){
      ar[i] = data[i][0];
    }
    return ar;
  }
  
  
  
  void printMat(){
    for (int j = 0; j<rows; j++) {
      print("["+j+"]: | ");
      for (int i = 0; i< cols; i++) {
        print(data[j][i] + " | ");
      }
      println();
    } 
  }
  static Matrix dsig(Matrix a){
    Matrix result = new Matrix(a.rows,a.cols);
    for (int j = 0; j<a.rows; j++) {
      for (int i = 0; i< a.cols; i++) {
        result.data[j][i] = (a.data[j][i]) * (1.0 - a.data[j][i]);
        //result.data[j][i] = 1-(a.data[j][i] * a.data[j][i]);
      }
    } 
      
    return result;
  }
}

void saveMat(Matrix a,String name){
      PrintWriter output;
      output = createWriter(name); 
      
      output.print(a.rows + " " + a.cols);
      output.println();
      
      for (int i = 0; i < a.rows; i++) {
          for (int j = 0; j < a.cols; j++) {
            output.print(a.data[i][j] + " ");
         }
         output.println();
      }
      
      output.flush();
      output.close();
    //println("saved");
}

public Matrix loadMat(String name){
  Matrix mat;
  
  BufferedReader reader;
  reader = createReader(name);
  
  String line;
  int cols_, rows_;
  try{
  line = reader.readLine();
  } catch(IOException e){
    e.printStackTrace();
    line = null;
  } 
  
  
  String pieces[] = split(line, " ");
  rows_ = Integer.parseInt(pieces[0]);
  cols_ = Integer.parseInt(pieces[1]);
  
  mat = new Matrix(rows_,cols_);
  
  for(int i = 0; i< rows_; i++){
    try{
    line = reader.readLine();
    } catch(IOException e){
      e.printStackTrace();
      line = null;
    }
    String piecesInLine[] = split(line, " ");
    
      for(int j = 0; j< cols_; j++){
        mat.data[i][j] = parseFloat(piecesInLine[j]);   
      }
    
  }
  
  println("loaded");
  return mat;
}


static float sigmoid(float x){
  return 1.0/(1.0 + (float)Math.exp(-x));
  //return (float)Math.tanh(x);
}



}```

#2

Please provide a link to the repo where you get this code from or a link to Shiffman’s video.

Kf


#3

https://github.com/CodingTrain/Toy-Neural-Network-JS/blob/master/lib/nn.js - code for NNs using JS (Shiffman’s version)


#4

Any ideas? Honestly, i still dont know where te problem is


#5

Hey Vadsavin,
I have code for writing neural networks in Processing (written before the CodingTrain series, so it’s a bit rough). My code is essentially a template, so you can set your own number of layers with any number of neurons. You would just have to rework the training algorithm (you can probably work backwards off of Mr. Shiffman’s example). PM me if you want the code/an explanation as to how I wrote it. If not, I would say you should start from the top, with a piece of paper, and track how everything works. I assume you have at least moderate experience with Processing and neural networks- tracing neural nets without experience can be quite frustrating. If you do not have that much experience with neural nets, I would suggest doing some research into basic feed-forward networks- this part is just matrix math. The training algorithm is where things get very tricky- study that a lot, and write down the math. If you can follow the math, compare it to Shiffman’s example and your port of the example. Check out this resource- it’s very helpful: http://neuralnetworksanddeeplearning.com/
Best of luck with your project- I know from experience it’s not easy, and Processing doesn’t make it easier at all.

  • Trilobyte

#7

Yay, I wonder if you show me your code


#8

Man, please, share your code


#9

Hey Vadsavin,
I sent you a PM a few days ago. Check your inbox by clicking your profile picture in the top right corner, then clicking on the mail symbol. It’s a bit hidden and doesn’t always pop up notifications.