Issues classifying a drawing with ml5.js

p5.js Libraries
1

Hello,

I’m trying to classify drawings with a custom model generated by Teachable Machine. To do so I’m storing the doodle in a p5.Renderer object using createGraphics(), and sending that object to the classifier.

I notice the classification and confidence is always the same, which means that I’m not sending anything, or the image sent is always the same. When I display or download the graphic however, it displays correctly. It also shows different results when I send just a regular image.png to classify, so the classifier works.

  • Do I need to do anything to the p5.Renderer object before sending it to the classifier?

Adding code below for reference. Thanks!

let classifier;
let imageModel = 'model/model.json';

let img;

let timer = 90;
let time = 0;
let drawing = false;
let entities = [];
let e = 0;
let pg;

function preload() {
  classifier = ml5.imageClassifier(imageModel); 
  img = loadImage('images/0.png');
}

function setup() {
  createCanvas(112, 112);
  //classifier.classify(img, gotResult);
  background(255);
  image(img, 0, 0);
  pg = createGraphics(112, 112);
  entities.push(new Entity());
}

function draw() {
  stroke(0);
  strokeWeight(3);
  if (mouseIsPressed === true) {
    if (drawing === false){ //it's a new entity
      entities.push(new Entity());
      e++;
      drawing = true;
      print(entities.length);
      
    }   
    line(mouseX, mouseY, pmouseX, pmouseY);
    pg.line(mouseX, mouseY, pmouseX, pmouseY);
    image(pg,0,0);
    
    entities[e].update();
    //entities[e].display();
    time = 0;
  } else if ((mouseIsPressed === false) && (drawing === true)) {
    time++;
    if (time >= timer){
      image(pg, 200, 0, 224, 224);
      let img2 = pg;
      classifier.classify(img2, gotResult);
      //img = loadImage('images/0.png');
      //classifier.classify(pg, gotResult);
      print("send");
      let name = "images/someink" + e + ".png";
      //save(pg, name);
      save(img2, name);
      drawing = false;
    }
  }
}

function mouseReleased() {
  entities[e].bounding();
  entities[e].display();
}

class Entity {
  constructor() {
    this.x0 = mouseX;
    this.y0 = mouseY;
    this.x1 = mouseX;
    this.y1 = mouseY;
    this.points = [];
  }

  update() { 
    // this.x0 = mouseX;
    // this.y0 = mouseY;
    // this.x1 = mouseX;
    // this.y1 = mouseY;
    this.points.push(new createVector(mouseX, mouseY));              
  }

  bounding(){
    for (let i=0; i < this.points.length; i++){      
      this.x0 = min(this.x0, this.points[i].x);
      this.y0 = min(this.y0, this.points[i].y);
      this.x1 = max(this.x1, this.points[i].x);
      this.y1 = max(this.y1, this.points[i].y);
    }
  }

  display() {
    noFill();
    stroke(0,255,0);
    rect(this.x0, this.y0, this.x1-this.x0,this.y1-this.y0);
  }
}

// A function to run when we get any errors and the results
function gotResult(error, results) {
  // Display error in the console
  if (error) {
    console.error(error);
  } else {
    // The results are in an array ordered by confidence.
    console.log(results);
    createDiv('Label: ' + results[0].label);
    createDiv('Confidence: ' + nf(results[0].confidence, 0, 2));
  }
}
1 Like
2

I think I found it: I needed to setup the background for the rendered object: pg.background(255).

3

Hi,

your code seems to be ok, so I have maybe an idea concerning your issue, because I discovered it today trying playing with other models : some ML models doesn’t support transparency, so it’s possible that using a PGraphics (and btw adding transparency to the sended image) causes trouble.

I saw that in the ml5 example, a .jpg is used ; did you tried with other formats than .png ?

EDIT : I didn’t see your own response, but according to what you said, it seems that it’s the cause of your trouble

4

Hi Pr0tonX,

While setting the background help get rid of errors, the training does not converge, which makes me think your initial hypothesis about no transparency may be right. Is there a way to convert PGraphics into an image without transparency without saving and loading it again? Maybe with loadPixels?

5

Maybe you can verify in the pixels[] array and loop on it, convert every transparent pixel on white pixel ?