I am making a flexible LED display system using Processing to generate and transmit imagery via OSC to a set of 10 Arduinos (Adafruit Feather HUZZAH), each controlling a single meter long 30-LED NeoPixel strip using the Adafruit NeoPixel library. I am looking for help with more efficiently unpacking/reading the OSC messages on the Arduinos, as I have a feeling that the method I am using is inefficient and is introducing latency.
In Processing, I am generating animations, and then “downsampling” each frame to match the resolution of the NeoPixels (30X10) by dividing the Processing frames into a 30X10 array of square cell, finding the average colour in each cell, and transmitting each row of cells to a specific Arduino as an OSC message.
I have been struggling with some latency issues, and I’m not entirely sure where the problem is. In the code included, you can see a reduced version of the Processing sketch that shows the pixel downsampling and OSC sending methods, neither of which seem to be an issue for my computer to run (the frame rate is steady and doesn’t show sign of lag).
I think the problem might be arising on the receiving end, because the latency doesn’t start becoming a problem until the sketch has been running for a few minutes. The frame rate of the sketch never drops, and the NeoPixels update quickly with very little noticeable lag at the start, but get gradually more and more out of sync with the Processing sketch.
Could this be a problem with OSC messages getting caught in a buffer on either the sending or receiving end? I’m not very confident with how I’m unpacking the OSC messages on the Arduino end, so I have a feeling that that is where the inefficiency is arising.
The included code shows a reduced but fully functional version of the Processing sketch as well as the Arduino sketch running on each of the 10 arduinos where I read UDP messages from the buffer.
In the processing sketch, please see:
- the Pixel.update() method, which I would have assumed was a source of inefficiency but seems to running fine
- the OSC.sendStrip() method, which breaks a 2D Pixel[][] array into 1D colour arrays and passes them to OSC.sendBytes()
- the OSC.sendBytes() method, which iterates through the 1D colour arrays and adds each RGB value to an OscP5.OscMessage and sends them out via OSC.
In the Arduino sketch, please see:
- the section of loop() where I use an example from the Arduino website to parse received UDP packets (I don’t fully understand how this section works)
- the routeStrips() method where I iterate through the RGB values from each received OSC message and assign them to individual NeoPixels
Processing Sketch:
//=IMPORT LIBRARIES=============
//import osc and networking libraries
import netP5.*;
import oscP5.*;
//==============================
//NUMBER OF LED STRIPS AND COLUMNS IN USE:--
int STRIPS = 6; //number of strips, or "rows"
int COLS = 30; //number of LEDs per strip, or "columns"
//------------------------------------------
//OscP5 object for receiving messages:---------------------
OscP5 oscrec; //receives an integer value and uses it to switch between different animations
int currcue; //stores the received integer^^
//--------------------------------------------------------
//-Custom OSC object for sending messages:----------------
OSC oscout;
//--------------------------------------------------------
//-PIXEL GRID INSTANCE:----------------------
PixelGrid pixelgrid;
//-------------------------------------------
void setup() {
//DISPLAY SETTINGS:-----------------------------------------
size(1350, 450, P2D);//Display ratio is 3:1 to match NeoPixel ratio of 3:1 (10 strips of 30 LEDs);
//-OSC IN SETUP:----------------------------------------------
oscrec = new OscP5(this, 12000);
//-----------------------------------------------------------
//-OSC OUT SETUP:--------------------------------------------
oscout = new OSC(1100, 8889, STRIPS);
//-----------------------------------------------------------
//-PIXELGRID SETUP:------------------------------------------
pixelgrid = new PixelGrid();
//-----------------------------------------------------------
background(0);
}
void draw() {
background(0);
fill(0, 255, 0);
ellipse(mouseX, mouseY, 400, 400);
//UPDATE AND PIXELS:------------------------------------
pixelgrid.run();
//------------------------------------------------------
//SEND PIXELS:------------------------------------------
oscout.sendStrip(pixelgrid.pix);
//------------------------------------------------------
}
void oscEvent(OscMessage theOscMessage) {
println("Received OSC Message:");
if (theOscMessage.checkAddrPattern("/leds")) {
if (theOscMessage.checkTypetag("i")) {
println("with address " +theOscMessage.addrPattern() + " and value " + theOscMessage.get(0).intValue());
currcue = theOscMessage.get(0).intValue();
}
}
}
//A grid of individual pixel objects:
class PixelGrid {
//-PIXEL OVERLAY SETTINGS:---------------------
int rows;
int cols;
int num_pixels;
Pixel[][] pix;
//---------------------------------------------
PixelGrid() {
rows = STRIPS;
cols = 30;
num_pixels = rows*cols;
//-PIXEL SETUPS:-----------------------------------
float w = width/cols;
float h = height/rows;
pix = new Pixel[cols][rows];
for (int r = 0; r < rows; r++) {
for (int c = 0; c < cols; c++) {
pix[c][r] = new Pixel((w/2)+(c*w), (h/2)+(r*h), w/2, h/2);
}
}
//------------------------------------------------
}
void run() {
//LOAD PIXELS AND UPDATE OVERLAY AT BOTTOM OF LOOP:-----
loadPixels();
//update and display pixel overlay:
for (int c = 0; c < cols; c++) {
for (int r = 0; r < rows; r++) {
pix[c][r].update();
pix[c][r].display();
}
}
//-------------------------------------------------------
}
//A downsampled "pixel." Stores and displays the average colour of the pixels within its boundaries
class Pixel {
//position and dimensions:
float x;
float y;
float w;
float h;
//fill color:
float r;
float g;
float b;
color col;
Pixel(float x_, float y_, float w_, float h_) {
x = x_;
y = y_;
w = w_;
h = h_;
//initialize with 0 values:
r = 0;
g = 0;
b = 0;
col = color(r, g, b);
}
void display() {
strokeWeight(1);
stroke(255, 100);
fill(col);
rectMode(RADIUS);
rect(x, y, w, h);
}
void update() {
//find the average color of the pixels contained within this
//object, by adding the constituent R,G,and B values together
//and then dividing by the total number of pixels contained
//within this object:
int av_r = 0;
int av_b = 0;
int av_g = 0;
for (int i = 0; i < w; i++) {
for (int j = 0; j < h; j++) {
float xp = (x - (w))+i;
float yp = (y - (h))+j;
int index = int(xp+(yp*width));
av_r += (pixels[index] >> 16) & 0xFF;
av_g += (pixels[index] >> 8) & 0xFF;
av_b += pixels[index] & 0xFF;
}
}
r = av_r/(w*h);
g = av_g/(w*h);
b = av_b/(w*h);
col = color(r, g, b);
}
}
}
class OSC {
//osc and netadress objects:
OscP5 osc;
NetAddress[] netaddress;
//destination IP, and in/out ports:
int in_port; //an "in port" is needed to make instance of OscP5, but not used beyond Oscp5() constructor
int out_port;
int num_ips;
OSC(int in, int out, int nips) {
in_port = in;
out_port = out;
num_ips = nips;
osc = new OscP5(this, in_port);
netaddress = new NetAddress[10];//destinations of all of the ESP arduinos:
netaddress[0] = new NetAddress("192.168.1.100", out_port);
netaddress[1] = new NetAddress("192.168.1.101", out_port);
netaddress[2] = new NetAddress("192.168.1.102", out_port);
netaddress[3] = new NetAddress("192.168.1.103", out_port);
netaddress[4] = new NetAddress("192.168.1.104", out_port);
netaddress[5] = new NetAddress("192.168.1.105", out_port);
netaddress[6] = new NetAddress("192.168.1.106", out_port);
netaddress[7] = new NetAddress("192.168.1.107", out_port);
netaddress[8] = new NetAddress("192.168.1.108", out_port);
netaddress[9] = new NetAddress("192.168.1.109", out_port);
}
//Builds an array of colors from a passed in array of "Pixel" objects (declared in other tab).
//"strip_index" refers to the horizontal row in the processing window and to the specific
//NeoPixel strip that the colors from that row will be sent to.
void sendStrip(PixelGrid.Pixel[][] p) {
for (int i = STRIPS-1; i>=0; i--) {
color[] strip_cols = new color[COLS];
for (int c = 0; c < COLS; c++) {
strip_cols[c] = p[c][i].col;
}
sendBytes("/strip", strip_cols, netaddress[(STRIPS-1)-i]);//forward the created color array to the "sendBytes()" function
}
}
void sendBytes(String addr, color[] strip_cols_, NetAddress ip) {
OscMessage message = new OscMessage(addr);//make new osc message with address "addr"
for (int i = 0; i < strip_cols_.length; i++) {
int r = (strip_cols_[i] >> 16) & 0xFF;
int g = (strip_cols_[i] >> 8) & 0xFF;
int b = (strip_cols_[i]) & 0xFF;
//add the rgb values of each color to the message...
message.add(r);
message.add(g);
message.add(b);
}
//...and then send the message to the address specified in the argument
osc.send(message, ip);
}
}
Arduino Sketch:
//INCLUDE LIBRARIES:----------
#include <ESP8266WiFi.h>
#include <WiFiUdp.h>
#include <Adafruit_NeoPixel.h>
#include <OSCBundle.h>
#include <OSCData.h>
#include <OSCMessage.h>
//-----------------------------
//UDP-Declarations:-----------
WiFiUDP udp;
//listening port:
unsigned int localPort = 8889;
//char array to store received messages:
char packetBuffer[255];
//Store the received char as a string
String receivedData;
//-----------------------------
//WiFi-Details:----------------
const char* ssid = "***";
const char* password = "***";
//-----------------------------
OSCErrorCode error;
//NeoPixel Declarations:
#define PIX_PIN 4
Adafruit_NeoPixel strip = Adafruit_NeoPixel(30, PIX_PIN, NEO_GRB + NEO_KHZ800);
void setup() {
//Start a serial stream:
Serial.begin(115200);
// CONNECT TO WIFI:*************************
Serial.println();
Serial.println();
Serial.print("Connecting to ");
Serial.println(ssid);
//start a instance of the WiFi object with declared ssid and password:
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}
Serial.println("");
Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());
Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());
Serial.println("Starting UDP");
udp.begin(localPort);
Serial.print("Local port: ");
#ifdef ESP32
Serial.println(localPort);
#else
Serial.println(udp.localPort());
#endif
//************************************
WiFi.setSleepMode(WIFI_NONE_SLEEP);
//INITIALIZE NEOPIXELS:+++++++++++++++
Serial.println("Attempting to Initialize Pixels...");
strip.begin();
strip.show();
Serial.println("Pixels initialized");
//++++++++++++++++++++++++++++++++++++
}
void loop() {
OSCMessage msg;
int packetSize = udp.parsePacket();
if (packetSize > 0)
{
// Serial.print("Received packet of size ");
// Serial.println(packetSize);
// Serial.print("From ");
// IPAddress remoteIp = udp.remoteIP();
// Serial.print(remoteIp);
// Serial.print(", port ");
// Serial.println(udp.remotePort());
while (packetSize--) {
msg.fill(udp.read());
}
if (!msg.hasError()) {
msg.dispatch("/strip", routeStrips);
} else {
error = msg.getError();
Serial.print("error: ");
Serial.println(error);
}
}
}
void routeStrips(OSCMessage &msg) {
for (int i = strip.numPixels()-1; i >= 0; i--) {
int r = msg.getInt(i*3);
int g = msg.getInt((i*3)+1);
int b = msg.getInt((i*3)+2);
strip.setPixelColor((strip.numPixels()-1)-i, strip.Color(r, g, b));
}
strip.show();
}