Fft audio analysis to pimage

Gallery
1

Just thought I would share this. Its a audio to fft image sketch. Not seen one in processing yet, but maybe I haven’t looked in the right place. May be interesting for some. This will be added to the current program I am making, and will be accompanied with filters and neural networks for analysis. More to come soon.

import processing.sound.*;

SoundFile superliminal = null;
Amplitude amp;

FFT fft;
AudioIn in;

//to set volume
Sound s;
int bands = 512;

float[] spectrum = new float[bands];
ArrayList<PVector> audio = new ArrayList<PVector>();
ArrayList<ArrayList<PVector>> audio2 = new ArrayList<ArrayList<PVector>>();
ArrayList<ArrayList<PVector>> visualiser = new ArrayList<ArrayList<PVector>>();

PImage specImg;

int frame=0;
int frame2=0;
int frames=0;
boolean constructImage;
void setup() {
  size(512, 360);
  
  // Create a Sound object for globally controlling the output volume.
  s = new Sound(this);
    
  // Create an Input stream which is routed into the Amplitude analyzer
  amp = new Amplitude(this);
  superliminal = new SoundFile(this,"kid.mp3");
  // Create an Input stream which is routed into the Amplitude analyzer
  fft = new FFT(this, bands);
  
  //amp.input(superliminal);
  fft.input(superliminal);
  superliminal.play();
  //superliminal.amp(0.01);
  frames = superliminal.frames();
}      

void draw() { 
  
  background(50);
  //---------------------------------------------------------------------------------------------------
  // Map vertical mouse position to volume.
  float amplitude = map(mouseY, 0, height, 0.4, 0.0);

  // Instead of setting the volume for every oscillator individually, we can just
  // control the overall output volume of the whole Sound library.
  s.volume(1);
  //------------------------------------------------------------------------------------------------------
  if(superliminal.isPlaying())fft.analyze(spectrum);
  
  if(superliminal.isPlaying())audio2.add(new ArrayList<PVector>());
  if(superliminal.isPlaying())visualiser.add(new ArrayList<PVector>());
  noFill();
  if(superliminal.isPlaying()){
    //beginShape();
  for(int i = 0; i < bands; i++){
  // The result of the FFT is normalized
  // draw the line for frequency band i scaling it up by 5 to get more amplitude.
  float k = map(i,0,bands,0,width);
  float y = map( (spectrum[i]),0,0.01,0,height);
  
  float k1 = i;
  float y1 = map( (spectrum[i]),0,0.01,0,255);
  
  colorMode(HSB);
  stroke(i,random(255),random(255));
  fill(i,random(255),random(255));
  audio.add(new PVector(k,y));
  audio2.get(frame).add(new PVector(k,height-y));
  visualiser.get(frame).add(new PVector(i,y1));
  //vertex(k,-y);
  stroke(i,random(255),random(255));
  //vertex(bar.x,bar.y);
  line(k,height,k,height -y);
  }
  //endShape(OPEN);
  fill(255);
  text("playing",100,100);
  frame ++;
}else{
    //beginShape();
  // this one is not recommended but may be useful to some
  //  for(int i = 0; i < audio.size(); i++){
      
  //    PVector bar = audio.get(i);
      
  //    colorMode(HSB);
  //    noFill();
  //    strokeWeight(1);
  //    stroke(i,random(255),random(255));
  //    //vertex(bar.x,bar.y);
  //    line(bar.x,height,bar.x,height-bar.y);
  //}
  
  //-----------------------------------------------------------------
  // for spectral playback--------note there is no audio, this is just the fft visualisation
  
  //  ArrayList<PVector> a = audio2.get(frame2);
  //  ArrayList<PVector> b = visualiser.get(frame2);
  //    for(int j = 0; j < a.size(); j++){
  //    PVector bar = a.get(j);
      
      
  //    colorMode(HSB);
  //    noFill();
  //    strokeWeight(1);
  //    //stroke(j,random(255),random(255));
  //    //vertex(bar.x,bar.y);
  //    //line(bar.x,height,bar.x,bar.y);
  //}
  //----------------------------------------------------------------
  
  if(!constructImage){
    specImg = createImage(bands,frame,RGB);
  
  specImg.loadPixels();
  for(int i = 0; i < visualiser.size(); i++){
    ArrayList<PVector> b = visualiser.get(i);
    for(int j = 0; j < b.size(); j++){
      
      int pos = j+i*b.size();
      
      PVector p = b.get(j);
      stroke(p.y);
      point(j,height-i);
      colorMode(HSB);
      specImg.pixels[pos] = color(p.y);
      
  }}constructImage = true;specImg.updatePixels();}
  
  if(constructImage){
   image(specImg,0,0); 
  }
  
    fill(255);
    text("stopped",100,100);
    if(frame2<frame-1)frame2++;
    else frame2 = 0;
//endShape(OPEN);

  
}

  fill(0);
  text(frameRate,200,200);
  //text(frames,200,210);
  //text(frame,200,220);
  text(audio2.size(),200,230);
  text(visualiser.size(),200,240);
  
};

this is what it produces

image

image

as some of you may notice I am mapping the fft value from 0, 0.01, as I feel this produces a better visualization for this particular sound snippet, this may not be the case for all audio.

Also note that I have chosen a particularly short piece of audio hence the height. For longer pieces of audio unless it is split into chunks it will create an image of a height equal to the amount of frames the audio has, in this case it is 137.

Further more the speed of playback may be changes by putting a multiplyer value in the play() function increasing it however divides the amount of frames produced by the number chosen, and the file can be analysed with volume on 0 by setting s.volume(0) instead of 1.

this was coded using guidance from the following website.

4 Likes