Hello all,
I’ve got data coming into a serial port at 250Hz and I’d like to save it all to a .csv file. Of course draw() is not going to be able to keep up with that rate of data.
Is there a way to save all incoming data to a file without polling?
Thanks in advance,
Twain
Additional info:
If you are using the serialEvent() function, then my understanding is that when Processing processes this function, you should be able to retrieve all the data in the queu. You should be able to process it as fast as it comes. I wouldn’t save it to the hard drive directly every time the function is fired. Instead, after you get certain amount of data, you will save it in a single operation, possibly in another thread to avoid blocking (I am keeping the picture simple).
Kf
Hi Kf,
Thanks for taking the time to help out (again)
I am indeed using the serialEvent() function to collect AND parse the data coming in. I’m saving the table to the hard drive every 5 seconds and still I’m seeing less data than expected.
I confirmed that the serial port is indeed receiving all the expected data using a simple serial terminal.
Since keeping i didn’t come across any new suggestions with these broad descriptions, i’ll go a bit more specific and post the code below:
Should i have set up a buffer for the saving the data into the buffer rather than parsing and saving the data as it comes in? If so, wouldn’t my program start to lag behind the real-time data pretty badly?
Thanks!
Twain
import processing.serial.*;
import static javax.swing.JOptionPane.*;
Table table;
String Path = "PathProvidedHere.csv";
String message;
//Some time keeping variables
int hours, minutes, seconds, milliseconds;
float SaveTime;
//Serial port selection
Serial myPort;
String COMx, COMlist = "";
final boolean debug = true;
String portName;
// Data variables
float yaw = 0.0; float pitch = 0.0; float roll = 0.0;
float A1, A2, A3, A4;
float E1, E2, E3, E4;
void setup() {
//Set up GIU box
size(1024, 768, P3D);
frameRate(250);
smooth();
//Some other setups like fonts, graphs, etc.
//Set up the logging table
table = new Table();
table.addColumn("A1"); table.addColumn("A2"); table.addColumn("A3"); table.addColumn("A4");
table.addColumn(""); table.addColumn("E1"); table.addColumn("E3"); table.addColumn("E4");
table.addColumn(" "); table.addColumn("min"); table.addColumn("sec"); table.addColumn("milli");
portName = chooseCOM();
delay(1000);
}
void draw() {
SavetoCSV();
//serialEvent(myPort); // read and parse incoming serial message
ACouple();
Unrelated();
FunctionsHere();
if(millis() - SaveTime > 5000){
saveTable(table, Path);
SaveTime=millis();
}
}
String chooseCOM() {
setupP2 = true;
try {
if (debug) printArray(Serial.list());
int i = Serial.list().length;
if (i != 0) {
if (i >= 2) {
// need to check which port the inst uses -
// for now we'll just let the user decide
for (int j = 0; j < i; ) {
COMlist += char(j+'a') + " = " + Serial.list()[j];
if (++j < i) COMlist += ", ";
}
COMx = showInputDialog("Which COM port is correct? (a,b,..):\n"+COMlist);
if (COMx == null) exit();
if (COMx.isEmpty()) exit();
i = int(COMx.toLowerCase().charAt(0) - 'a') + 1;
}
String portName = Serial.list()[i-1];
if (debug) //println(portName + " Selected");
myPort = new Serial(this, portName, 115200); // change baud rate to your liking
myPort.bufferUntil(13); // buffer until CR/LF appears, but not required..
return portName;
} else {
showMessageDialog(frame, "Device is not connected to the PC");
exit();
}
}
catch (Exception e)
{ //Print the type of error
showMessageDialog(frame, "COM port is not available (may\nbe in use by another program)");
//println("Error:", e);
exit();
}
return "noPort";
}
void serialEvent(Serial myPort) {
int newLine = 13; // new line character in ASCII
do {
message = myPort.readStringUntil(newLine); // read from port until new line
if (message != null) {
String[] list = split(trim(message), " ");
if (list.length == 4 && list[0].equals("i")) {
yaw = float(list[1]); // convert to float yaw
pitch = float(list[2]); // convert to float pitch
roll = float(list[3]); // convert to float roll
} else if (list.length == 5 && list[0].equals("s")) {
A1 = float(list[1]);
A2 = float(list[2]);
A3 = float(list[3]);
A4 = float(list[4]);
} else if (list.length >=2 && list[0].equals("b")) {
Battery = int(list[1]);
} else if (list.length >= 2 && list[0].equals("m")) {
MACid = int(list[1]);
} else {
//print anything extra to console
//println(message);
}
}
} while (message != null);
}
void SavetoCSV() {
if (A1 != 0) {
TableRow newRow = table.addRow();
newRow.setFloat("A1", (A1));
newRow.setFloat("A2", (A2));
newRow.setFloat("A3", (A3));
newRow.setFloat("A4", (A4));
//saveTable(table, Path);
}
}
Went the buffer route… still think I’m missing some data but I’m getting awfully close!
I’m unsure that I’m saving the data in the right order.
Thanks all,
Code:
import processing.serial.*;
import static javax.swing.JOptionPane.*;
//Arrays to save the data
LinkedList<Integer> A1c = new LinkedList<Integer>();
LinkedList<Integer> A2c = new LinkedList<Integer>();
LinkedList<Integer> A3c = new LinkedList<Integer>();
LinkedList<Integer> A4c = new LinkedList<Integer>();
int bufferLength = 500;
int bufflen = 0;
//Serial port selection
Serial myPort;
String COMx, COMlist = "";
final boolean debug = true;
String portName;
// Data variables
float yaw = 0.0; float pitch = 0.0; float roll = 0.0;
float A1, A2, A3, A4;
//Data log variables
Table table;
String Path = "PathtoFile.csv";
void setup() {
//Set up GIU box
size(1024, 768, P3D);
frameRate(250);
strokeWeight(50);
smooth();
//Set up the logging table
table = new Table();
table.addColumn("A1"); table.addColumn("A2"); table.addColumn("A3"); table.addColumn("A4");
portName = chooseCOM();
}
void draw() {
//SavetoCSV now called within SerialEvent()
//SavetoCSV();
//serialEvent(myPort); // read and parse incoming serial message
Some();
Unrelated();
FunctionsHere();
}
void serialEvent(Serial myPort) {
int newLine = 13; // new line character in ASCII
do {
message = myPort.readStringUntil(newLine); // read from port until new line
if (message != null) {
String[] list = split(trim(message), " ");
if (list.length == 4 && list[0].equals("i")) {
yaw = float(list[1]); // convert to float yaw
pitch = float(list[2]); // convert to float pitch
roll = float(list[3]); // convert to float roll
} else if (list.length == 5 && list[0].equals("s")) {
A1 = float(list[1]);
A2 = float(list[2]);
A3 = float(list[3]);
A4 = float(list[4]);
if (bufflen < bufferLength) {
A1c.push(int(A1));
A2c.push(int(A2));
A3c.push(int(A3));
A4c.push(int(A4));
bufflen++;
}
else{
bufflen = 0;
SavetoCSV();
}
} else if (list.length >=2 && list[0].equals("b")) {
Battery = int(list[1]);
} else if (list.length >= 2 && list[0].equals("m")) {
MACid = int(list[1]);
} else {
//print anything extra to console
//println(message);
}
}
} while (message != null);
}
void SavetoCSV() {
if (A1 != 0) {
for (int i = bufferLength - 1; i >= 0; i--){
if (i > 0){
TableRow newRow = table.addRow();
newRow.setFloat("A1", (A1c.get(i)));
newRow.setFloat("A2", (A2c.get(i)));
newRow.setFloat("A3", (A3c.get(i)));
newRow.setFloat("A4", (A4c.get(i)));
} else saveTable(table, Path);
}
}
}
String chooseCOM() {
setupP2 = true;
try {
if (debug) printArray(Serial.list());
int i = Serial.list().length;
if (i != 0) {
if (i >= 2) {
// need to check which port the inst uses -
// for now we'll just let the user decide
for (int j = 0; j < i; ) {
COMlist += char(j+'a') + " = " + Serial.list()[j];
if (++j < i) COMlist += ", ";
}
COMx = showInputDialog("Which COM port is correct? (a,b,..):\n"+COMlist);
if (COMx == null) exit();
if (COMx.isEmpty()) exit();
i = int(COMx.toLowerCase().charAt(0) - 'a') + 1;
}
String portName = Serial.list()[i-1];
if (debug) //println(portName + " Selected");
myPort = new Serial(this, portName, 115200); // change baud rate to your liking
myPort.bufferUntil(13); // buffer until CR/LF appears, but not required..
return portName;
} else {
showMessageDialog(frame, "Device is not connected to the PC");
exit();
}
}
catch (Exception e)
{ //Print the type of error
showMessageDialog(frame, "COM port is not available (may\nbe in use by another program)");
//println("Error:", e);
exit();
}
return "noPort";
}