Hello. I was following along with Dan’s Youtube series on neural networks and even though I’m only a beginner decided to write it with the added functionality of having multiple hidden layers instead of copying exactly what was shown (although it is still extremely similar).
Unfortunately as a beginner, the amount of scripting required is so overwhelming and when it came to Dan’s XOR coding challenge it consistently output an entirely wrong result, and thus I was hoping some people might quickly review the code.
I write using Visual Studio Code, and debugged with (Nothing fancy) both the Chrome console and the Firefox console, although p5.js stopped working on Chrome today.
Feel free to critique the structure of both the code and of this post.
- The code involved is the main file:
let nn;
function setup() {
createCanvas(400, 400);
nn = new NeuralNetwork(2, 2, 1);
}
function draw() {
background(0);
for (let i = 0; i < 10000; i++) {
let data = randomInputs();
let output = expectedOutput(data);
nn.fullTrain(data, output);
}
let resolution = 10;
let cols = width / resolution;
let rows = height / resolution;
for (let i = 0; i < cols; i++) {
for (let j = 0; j < rows; j++) {
let x1 = i / cols;
let x2 = j / rows;
let inputs = [x1, x2];
let y = nn.fullFeed(inputs)[nn.hidden_layers][0];
fill(y * 255);
rect(i * resolution, j * resolution, resolution, resolution);
}
}
}
function randomInputs() {
let a = Math.random();
let b = Math.random();
a = distributor(a);
b = distributor(b);
return [a, b];
}
function expectedOutput(x) {
let y = [];
y[0] = Math.abs(x[0] - x[1])
return y;
}
function distributor(x) {
if (x < 0.5) {
return 0;
} else return 1;
}
I highly doubt there’s anything wrong with the code in the sketch file ^^
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
The difference between this neural network and the one in the Youtube series is that this.hidden_nodes can take the form of an array of integers, each integer representing the number of neurons in that layer, E.g
let nn = new NeuralNetwork(2,[3,4],1);
Will construct a neural network with 2 inputs, 2 hidden layers with 3 and 4 neurons respectively, and 1 output.
- The neural network class:
function sigmoid(x) {
return 1 / (1 + Math.exp(-x));
}
function dsigmoid(y) {
return y * (1 - y);
}
class NeuralNetwork {
constructor(input_nodes, hidden_nodes, output_nodes) {
this.input_nodes = input_nodes;
this.hidden_nodes = hidden_nodes;
this.output_nodes = output_nodes;
this.hidden_layers = 1;
this.learning_rate = 0.01;
this.weights = [];
this.biases = [];
if (hidden_nodes instanceof Array) {
this.hidden_layers = this.hidden_nodes.length;
console.log(this.hidden_layers);
for (let i = 0; i < this.hidden_layers + 1; i++) {
if (i == 0) {
this.weights[i] = new Matrix(hidden_nodes[i], input_nodes);
this.biases[i] = new Matrix(this.input_nodes,1);
} else if (i < this.hidden_layers) {
this.weights[i] = new Matrix(hidden_nodes[i],hidden_nodes[i-1]);
this.biases[i] = new Matrix(hidden_nodes[i-1],1)
} else {
this.weights[i] = new Matrix(output_nodes, hidden_nodes[i-1]);
this.biases[i] = new Matrix(output_nodes, 1);
}
this.weights[i].randomize();
this.biases[i].randomize();
}
} else {
this.weights[0] = new Matrix(hidden_nodes, input_nodes);
this.weights[1] = new Matrix(output_nodes, hidden_nodes);
this.biases[0] = new Matrix(hidden_nodes,1);
this.biases[1] = new Matrix(output_nodes,1);
this.weights[0].randomize();
this.weights[1].randomize();
this.biases[0].randomize();
this.biases[1].randomize();
}
}
// Move between 1 layer
feedForward(input, x) {
let inputMatrix = Matrix.fromArray(input);
// Weighted Sum
let hidden = Matrix.matMul(this.weights[x],inputMatrix);
// Activation Function
hidden.map(sigmoid);
return Matrix.toArray(hidden);
}
// Loop through each layer and record nodal outputs
fullFeed(inputs) {
let result = [];
result[0] = inputs;
for (let i = 0; i < this.hidden_layers + 1; i++) {
let temp = result[i];
result[i+1] = this.feedForward(temp, i);
}
return result;
}
// Backpropogation between two layers
train(outputsm, errors, x) {
// x starts at 0
// Calculate gradient
let gradients = Matrix.map(outputsm[this.hidden_layers + 1 - x],dsigmoid);
gradients.sMul(errors[this.hidden_layers - x]);
gradients.sMul(this.learning_rate);
// Calculate deltas
let hidden_T = Matrix.transpose(outputsm[this.hidden_layers-1]);
let dw = Matrix.matMul(gradients, hidden_T);
this.weights[this.hidden_layers - x].add(dw);
let db = Matrix.sMul(this.biases[this.hidden_layers - x], gradients);
this.biases[this.hidden_layers - x].add(db);
}
// Computes errors of each node
computeErrors(inputs, targets) {
let outputs = this.fullFeed(inputs);
// Convert outputs and targets to a matrix
let outputsm = [];
for (let i = 0; i < outputs.length; i++) {
outputsm[i] = Matrix.fromArray(outputs[i]);
}
let targetsm = Matrix.fromArray(targets);
// Compute errors for each node
let errors = [];
for (let i = 0; i < this.hidden_layers; i++) {
if (i == 0) {
let error = Matrix.sub(targetsm, outputsm[this.hidden_layers+1]);
errors[0] = error;
}
let weights_T = Matrix.transpose(this.weights[this.hidden_layers - i]);
let hidden_errors = Matrix.matMul(weights_T, errors[0]);
errors.unshift(hidden_errors);
}
return errors;
}
// Computes errors and loops through node backpropogation function
fullTrain(inputs, targets) {
// Find outputs for nodes
let outputs = this.fullFeed(inputs);
// Convert outputs and targets to a matrix
let outputsm = [];
for (let i = 0; i < outputs.length; i++) {
outputsm[i] = Matrix.fromArray(outputs[i]);
}
let errors = this.computeErrors(inputs,targets);
for (let i = 0; i < this.hidden_layers + 1; i++) {
this.train(outputsm, errors, i);
}
}
}
I suspect there may be something wrong with the indexing of for-loops
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
The main differences between my matrix class and Dan’s is that every function logs to the console when an input has the wrong class, and that matrix multiplication falls under matMul() and scalar multiplication falls under sMul().
- The matrix class
// Matrix is constructed and filled with 0s
class Matrix {
constructor(rows, cols) {
this.rows = rows;
this.cols = cols;
this.data = [];
for (let i = 0; i < this.rows; i++) {
this.data[i] = [];
for (let j = 0; j < cols; j++) {
this.data[i][j] = 0;
}
}
}
// Matrix is filled with a random value between -1 and 1
randomize() {
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
this.data[i][j] = 1 - 2 * Math.random()
}
}
}
// Create a matrix from an array
static fromArray(a) {
if (!(a instanceof Array)) {
console.log("Input must be an array");
return undefined;
}
let m = new Matrix(a.length, 1);
for (let i = 0; i < a.length; i++) {
m.data[i][0] = a[i];
}
return m;
}
// Creates a 1d array from a matrix
// Not equivalent to "let a = m.data;"
static toArray(m) {
if (!(m instanceof Matrix)) {
console.log("Input must be a matrix");
return undefined;
}
let a = [];
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
a.push(m.data[i][j]);
}
}
return a;
}
// Matrix Multiplication
static matMul(m, n) {
if (!(m instanceof Matrix) || !(n instanceof Matrix)) {
if (m instanceof Matrix && !(n instanceof Matrix)) {
console.log("Input B must be a matrix");
} else if (!(m instanceof Matrix) && n instanceof Matrix) {
console.log("Input A must be a matrix");
} else console.log("Both inputs A and B must be matrices");
return undefined;
}
if (m.cols !== n.rows) {
console.log("Cols of A must equal rows of B");
return undefined;
}
let result = new Matrix(m.rows, n.cols);
for (let i = 0; i < result.rows; i++) {
for (let j = 0; j < result.cols; j++) {
for (let k = 0; k < m.cols; k++) {
result.data[i][j] += m.data[i][k] * n.data[k][j];
}
}
}
return result;
}
matMul(n) {
if (!(n instanceof Matrix)) {
console.log("Input must be a matrix");
return undefined;
}
if (this.cols !== n.rows) {
console.log("Cols of A must equal rows of B");
return undefined;
}
let result = new Matrix(this.rows, n.cols);
for (let i = 0; i < result.rows; i++) {
for (let j = 0; j < result.cols; j++) {
for (let k = 0; k < this.cols; k++) {
result.data[i][j] += this.data[i][k] * n.data[k][j];
}
}
}
this.data = result.data;
}
// Returns a matrix filled with the function of each input
static map(m, func) {
if (!(m instanceof Matrix) || !(func instanceof Function)) {
if (m instanceof Matrix && !(n instanceof Function)) {
console.log("Input B must be a function");
} else if (!(m instanceof Matrix) && func instanceof Function) {
console.log("Input A must be a matrix");
} else console.log("Input A must be a matrix and input B must be a function");
return undefined;
}
let result = new Matrix(m.rows, m.cols);
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
let val = m.data[i][j];
result.data[i][j] = func(val);
}
}
return result;
}
map(func) {
if (!(func instanceof Function)) {
console.log("Input must be a function");
return undefined;
}
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
let val = this.data[i][j];
this.data[i][j] = func(val);
}
}
}
static transpose(m) {
if (!(m instanceof Matrix)) {
console.log("Input is not a matrix");
return undefined;
} else {
let result = new Matrix(m.cols, m.rows);
for (let i = 0; i < result.rows; i++) {
for (let j = 0; j < result.cols; j++) {
for (let k = 0; k < m.cols; k++) {
result.data[i][j] = m.data[j][i];
}
}
}
return result;
}
}
transpose() {
let result = new Matrix(this.cols, this.rows);
for (let i = 0; i < result.rows; i++) {
for (let j = 0; j < result.cols; j++) {
for (let k = 0; k < this.cols; k++) {
result.data[i][j] = this.data[j][i];
}
}
}
this.data = result.data;
}
static sMul(m, n) {
if (n instanceof Matrix) {
let result = new Matrix(m.rows, m.cols);
if (m.cols !== n.cols || m.rows !== n.rows) {
console.log("Dimensions of A must match dimensions of B");
return undefined;
}
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
result.data[i][j] = m.data[i][j] * n.data[i][j];
}
}
return result;
} else {
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
result.data[i][j] = m.data[i][j] * n;
}
}
return result;
}
}
sMul(n) {
if (n instanceof Matrix) {
if (this.cols !== n.cols || this.rows != n.rows) {
console.log("Dimensions of A must match dimensions of B");
return undefined;
}
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
this.data[i][j] *= n.data[i][j];
}
}
} else {
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
this.data[i][j] *= n;
}
}
}
}
static add(m, n) {
if (n instanceof Matrix) {
if (m.cols !== n.cols || m.rows != n.rows) {
console.log("Dimensions of A must match dimensions of B");
return undefined;
}
let result = m;
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
results.data[i][j] += n.data[i][j];
}
}
return result;
} else {
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
m.data[i][j] += n;
}
}
}
}
static sub(m, n) {
if (n instanceof Matrix) {
if (m.cols !== n.cols || m.rows != n.rows) {
console.log("Dimensions of A must match dimensions of B");
return undefined;
}
let result = m;
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
result.data[i][j] -= n.data[i][j];
}
}
return result;
} else {
for (let i = 0; i < m.rows; i++) {
for (let j = 0; j < m.cols; j++) {
m.data[i][j] -= n;
}
}
}
}
add(n) {
if (n instanceof Matrix) {
if (this.cols !== n.cols || this.rows != n.rows) {
console.log("Dimensions of A must match dimensions of B");
return undefined;
}
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
this.data[i][j] += n.data[i][j];
}
}
} else {
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
this.data[i][j] += n;
}
}
}
}
sub(n) {
if (n instanceof Matrix) {
if (this.cols !== n.cols || this.rows != n.rows) {
console.log("Dimensions of A must match dimensions of B");
return undefined;
}
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
this.data[i][j] -= n.data[i][j];
}
}
} else {
for (let i = 0; i < this.rows; i++) {
for (let j = 0; j < this.cols; j++) {
this.data[i][j] -= n;
}
}
}
}
print() {
console.table(this.data);
}
}
//////////////////////////////////////////////////////////////////////////////////////////////////////
I would upload my program’s solution but I can only upload 1 image, so here is a potential desired solution.
Hope this formatting is good