Issues classifying a drawing with ml5.js

Hello,

I’m trying to classify drawings with a custom model generated by Teachable Machine. To do so I’m storing the doodle in a p5.Renderer object using createGraphics(), and sending that object to the classifier.

I notice the classification and confidence is always the same, which means that I’m not sending anything, or the image sent is always the same. When I display or download the graphic however, it displays correctly. It also shows different results when I send just a regular image.png to classify, so the classifier works.

  • Do I need to do anything to the p5.Renderer object before sending it to the classifier?

Adding code below for reference. Thanks!

let classifier;
let imageModel = 'model/model.json';

let img;

let timer = 90;
let time = 0;
let drawing = false;
let entities = [];
let e = 0;
let pg;

function preload() {
  classifier = ml5.imageClassifier(imageModel); 
  img = loadImage('images/0.png');
}

function setup() {
  createCanvas(112, 112);
  //classifier.classify(img, gotResult);
  background(255);
  image(img, 0, 0);
  pg = createGraphics(112, 112);
  entities.push(new Entity());
}

function draw() {
  stroke(0);
  strokeWeight(3);
  if (mouseIsPressed === true) {
    if (drawing === false){ //it's a new entity
      entities.push(new Entity());
      e++;
      drawing = true;
      print(entities.length);
      
    }   
    line(mouseX, mouseY, pmouseX, pmouseY);
    pg.line(mouseX, mouseY, pmouseX, pmouseY);
    image(pg,0,0);
    
    entities[e].update();
    //entities[e].display();
    time = 0;
  } else if ((mouseIsPressed === false) && (drawing === true)) {
    time++;
    if (time >= timer){
      image(pg, 200, 0, 224, 224);
      let img2 = pg;
      classifier.classify(img2, gotResult);
      //img = loadImage('images/0.png');
      //classifier.classify(pg, gotResult);
      print("send");
      let name = "images/someink" + e + ".png";
      //save(pg, name);
      save(img2, name);
      drawing = false;
    }
  }
}

function mouseReleased() {
  entities[e].bounding();
  entities[e].display();
}

class Entity {
  constructor() {
    this.x0 = mouseX;
    this.y0 = mouseY;
    this.x1 = mouseX;
    this.y1 = mouseY;
    this.points = [];
  }

  update() { 
    // this.x0 = mouseX;
    // this.y0 = mouseY;
    // this.x1 = mouseX;
    // this.y1 = mouseY;
    this.points.push(new createVector(mouseX, mouseY));              
  }

  bounding(){
    for (let i=0; i < this.points.length; i++){      
      this.x0 = min(this.x0, this.points[i].x);
      this.y0 = min(this.y0, this.points[i].y);
      this.x1 = max(this.x1, this.points[i].x);
      this.y1 = max(this.y1, this.points[i].y);
    }
  }

  display() {
    noFill();
    stroke(0,255,0);
    rect(this.x0, this.y0, this.x1-this.x0,this.y1-this.y0);
  }
}

// A function to run when we get any errors and the results
function gotResult(error, results) {
  // Display error in the console
  if (error) {
    console.error(error);
  } else {
    // The results are in an array ordered by confidence.
    console.log(results);
    createDiv('Label: ' + results[0].label);
    createDiv('Confidence: ' + nf(results[0].confidence, 0, 2));
  }
}





1 Like

I think I found it: I needed to setup the background for the rendered object: pg.background(255).

Hi,

your code seems to be ok, so I have maybe an idea concerning your issue, because I discovered it today trying playing with other models : some ML models doesn’t support transparency, so it’s possible that using a PGraphics (and btw adding transparency to the sended image) causes trouble.

I saw that in the ml5 example, a .jpg is used ; did you tried with other formats than .png ?

EDIT : I didn’t see your own response, but according to what you said, it seems that it’s the cause of your trouble

Hi Pr0tonX,

While setting the background help get rid of errors, the training does not converge, which makes me think your initial hypothesis about no transparency may be right. Is there a way to convert PGraphics into an image without transparency without saving and loading it again? Maybe with loadPixels?

Maybe you can verify in the pixels[] array and loop on it, convert every transparent pixel on white pixel ?