How to avoid blocking while loop reading Serial?

Question

I am currently playing with a MindWave Mobile headset. I am parsing bluetooth data received from a connected BlueSMIRF Silver. The code I started from is a sample from the MindWave wiki and it works using hardware Serial. For reference here is the original code:

////////////////////////////////////////////////////////////////////////
// Arduino Bluetooth Interface with Mindwave
// 
// This is example code provided by NeuroSky, Inc. and is provided
// license free.
////////////////////////////////////////////////////////////////////////
#define LED 13
#define BAUDRATE 57600
#define DEBUGOUTPUT 1
#define GREENLED1 3
#define GREENLED2 4
#define GREENLED3 5
#define YELLOWLED1 6
#define YELLOWLED2 7
#define YELLOWLED3 8
#define YELLOWLED4 9
#define REDLED1 10
#define REDLED2 11
#define REDLED3 12
#define powercontrol 10
// checksum variables
byte generatedChecksum = 0;
byte checksum = 0; 
int payloadLength = 0;
byte payloadData[64] = {
 0};
byte poorQuality = 0;
byte attention = 0;
byte meditation = 0;
// system variables
long lastReceivedPacket = 0;
boolean bigPacket = false;
//////////////////////////
// Microprocessor Setup //
//////////////////////////
void setup() {
 pinMode(GREENLED1, OUTPUT);
 pinMode(GREENLED2, OUTPUT);
 pinMode(GREENLED3, OUTPUT);
 pinMode(YELLOWLED1, OUTPUT);
 pinMode(YELLOWLED2, OUTPUT);
 pinMode(YELLOWLED3, OUTPUT);
 pinMode(YELLOWLED4, OUTPUT);
 pinMode(REDLED1, OUTPUT);
 pinMode(REDLED2, OUTPUT);
 pinMode(REDLED3, OUTPUT);
 pinMode(LED, OUTPUT);
 Serial.begin(BAUDRATE); // USB
}
////////////////////////////////
// Read data from Serial UART //
////////////////////////////////
byte ReadOneByte() {
 int ByteRead;
 while(!Serial.available());
 ByteRead = Serial.read();
#if DEBUGOUTPUT 
 Serial.print((char)ByteRead); // echo the same byte out the USB serial (for debug purposes)
#endif
 return ByteRead;
}
/////////////
//MAIN LOOP//
/////////////
void loop() {
 // Look for sync bytes
 if(ReadOneByte() == 170) {
 if(ReadOneByte() == 170) {
 payloadLength = ReadOneByte();
 if(payloadLength > 169) //Payload length can not be greater than 169
 return;
 generatedChecksum = 0; 
 for(int i = 0; i < payloadLength; i++) { 
 payloadData[i] = ReadOneByte(); //Read payload into memory
 generatedChecksum += payloadData[i];
 } 
 checksum = ReadOneByte(); //Read checksum byte from stream 
 generatedChecksum = 255 - generatedChecksum; //Take one's compliment of generated checksum
 if(checksum == generatedChecksum) { 
 poorQuality = 200;
 attention = 0;
 meditation = 0;
 for(int i = 0; i < payloadLength; i++) { // Parse the payload
 switch (payloadData[i]) {
 case 2:
 i++; 
 poorQuality = payloadData[i];
 bigPacket = true; 
 break;
 case 4:
 i++;
 attention = payloadData[i]; 
 break;
 case 5:
 i++;
 meditation = payloadData[i];
 break;
 case 0x80:
 i = i + 3;
 break;
 case 0x83:
 i = i + 25; 
 break;
 default:
 break;
 } // switch
 } // for loop
#if !DEBUGOUTPUT
 // *** Add your code here ***
 if(bigPacket) {
 if(poorQuality == 0)
 digitalWrite(LED, HIGH);
 else
 digitalWrite(LED, LOW);
 Serial.print("PoorQuality: ");
 Serial.print(poorQuality, DEC);
 Serial.print(" Attention: ");
 Serial.print(attention, DEC);
 Serial.print(" Time since last packet: ");
 Serial.print(millis() - lastReceivedPacket, DEC);
 lastReceivedPacket = millis();
 Serial.print("\n"); 
 }
#endif 
 bigPacket = false; 
 }
 else {
 // Checksum Error
 } // end if else for checksum
 } // end if read 0xAA byte
 } // end if read 0xAA byte
}

One problem I have is the blocking while loop in the ReadOneByte function:

byte ReadOneByte() {
 int ByteRead;
 while(!Serial.available());
 ByteRead = Serial.read();
#if DEBUGOUTPUT 
 Serial.print((char)ByteRead); // echo the same byte out the USB serial (for debug purposes)
#endif
 return ByteRead;
}

I am trying to avoid this so I started drafting a basic state machine approach:

#define BAUDRATE 57600
// checksum variables
byte generatedChecksum = 0;
byte checksum = 0; 
int payloadLength = 0;
byte payloadData[169] = {0};
byte poorQuality = 0;
byte attention = 0;
byte meditation = 0;
// system variables
long lastReceivedPacket = 0;
boolean bigPacket = false;
int payloadIndex;
int state = 0;
const int STATE_WAIT_FOR_FIRST_A = 0;
const int STATE_WAIT_FOR_SECOND_A = 1;
const int STATE_WAIT_FOR_PAYLOAD_LENGTH = 2;
const int STATE_WAIT_FOR_PAYLOAD = 3;
const int STATE_WAIT_FOR_CHECKSUM = 4;
const String stateNames[5] = {"waiting for first A","waiting for second A","waiting for payload length","accumulating payload","waiting for checksum"};
void setup() {
 Serial.begin(BAUDRATE); // USB
}
void loop() {}
void parsePayload(){
 poorQuality = 200;
 attention = 0;
 meditation = 0;
 for(int i = 0; i < payloadLength; i++) { // Parse the payload
 switch (payloadData[i]) {
 case 2:
 i++; 
 poorQuality = payloadData[i];
 bigPacket = true; 
 break;
 case 4:
 i++;
 attention = payloadData[i]; 
 break;
 case 5:
 i++;
 meditation = payloadData[i];
 break;
 case 0x80:
 i = i + 3;
 break;
 case 0x83:
 i = i + 25; 
 break;
 default:
 break;
 } // switch
 } // for loop
 Serial.print("bigPacket:");
 Serial.println(bigPacket);
 if(bigPacket) {
 Serial.print("PoorQuality: ");
 Serial.print(poorQuality, DEC);
 Serial.print(" Attention: ");
 Serial.print(attention, DEC);
 Serial.print(" Time since last packet: ");
 Serial.print(millis() - lastReceivedPacket, DEC);
 lastReceivedPacket = millis();
 Serial.print("\n"); 
 }
 bigPacket = false; 
}
void printState(){
 Serial.print("state:");
 Serial.println(stateNames[state]);
}
void serialEvent(){
 if(Serial.available() > 0){
 switch(state){
 case STATE_WAIT_FOR_FIRST_A:
 printState();
 if(Serial.read() == 170) state = STATE_WAIT_FOR_SECOND_A;
 break;
 case STATE_WAIT_FOR_SECOND_A:
 printState();
 if(Serial.read() == 170) state = STATE_WAIT_FOR_PAYLOAD_LENGTH;
 break;
 case STATE_WAIT_FOR_PAYLOAD_LENGTH:
 printState();
 payloadLength = Serial.read();
 Serial.print("payloadLength:");Serial.println(payloadLength);
 if(payloadLength > 169){
 Serial.println(payloadLength > 169);
 state = STATE_WAIT_FOR_FIRST_A;
 return;
 }
 generatedChecksum = payloadIndex = 0;
 state = STATE_WAIT_FOR_PAYLOAD;
 break;
 case STATE_WAIT_FOR_PAYLOAD:
 printState();
 if(payloadIndex < payloadLength){
 payloadData[payloadIndex] = Serial.read();
 generatedChecksum += payloadData[payloadIndex];
 Serial.print("payloadData[");Serial.print(payloadIndex);Serial.print(" of ");Serial.print(payloadLength);Serial.print("]: ");
 Serial.println(payloadData[payloadIndex]);
 }else{
 state = STATE_WAIT_FOR_CHECKSUM;
 }
 break;
 case STATE_WAIT_FOR_CHECKSUM:
 printState();
 checksum = Serial.read();
 generatedChecksum = 255 - generatedChecksum;
 if(checksum == generatedChecksum) {
 Serial.println("checksum MATCH! parsing payload");
 parsePayload();
 state = STATE_WAIT_FOR_FIRST_A;
 }else{
 Serial.println("checksum FAIL!");
 state = STATE_WAIT_FOR_FIRST_A;
 }
 break;
 }
 }
}

As far as I can understand from the serialEvent() reference this function would be called only when a new byte is available. I've also added a condition to check if Serial.available() > 0.

I can see the messages I expect when parsing the data, but only small packets(usually 4 bytes long) end up having a correct checksum and never receive a payload with the useful EEG data I'm looking for.

How can I check that my approach is correct or not/ I'm not loosing bytes using serialEvent() instead of the blocking while(!Serial.available()) ? If so, how can I rewrite the while loop in a non blocking way ?

I've not super experienced with Arduino, but I started reading on interrupts. Would a USART_RX interrupt help at all ? (or would it do the same as serialEvent -> trigger when a new byte is available?)

Update! Using Wirewrap's suggestion to use read() which returns -1 if there is no data, I've used peek() which does almost the same, except it doesn't remove the character peeked at from the buffer. For reference here is the code used:

#define BAUDRATE 57600
#define DEBUGOUTPUT 1
// checksum variables
byte generatedChecksum = 0;
byte checksum = 0; 
int payloadLength = 0;
int payloadIndex;
byte payloadData[169] = {0};
byte poorQuality = 0;
byte attention = 0;
byte meditation = 0;
// system variables
long lastReceivedPacket = 0;
boolean bigPacket = false;
int state = 0;
const int STATE_WAIT_FOR_FIRST_A = 0;
const int STATE_WAIT_FOR_SECOND_A = 1;
const int STATE_WAIT_FOR_PAYLOAD_LENGTH = 2;
const int STATE_WAIT_FOR_PAYLOAD = 3;
const int STATE_WAIT_FOR_CHECKSUM = 4;
void setup() {
 Serial.begin(BAUDRATE); // USB 
}
void loop() {}
void parsePayload(){
 poorQuality = 200;
 attention = 0;
 meditation = 0;
 for(int i = 0; i < payloadLength; i++) { // Parse the payload
 switch (payloadData[i]) {
 case 2:
 i++; 
 poorQuality = payloadData[i];
 bigPacket = true; 
 break;
 case 4:
 i++;
 attention = payloadData[i]; 
 break;
 case 5:
 i++;
 meditation = payloadData[i];
 break;
 case 0x80:
 i = i + 3;
 break;
 case 0x83:
 i = i + 25; 
 break;
 default:
 break;
 } // switch
 } // for loop
 if(bigPacket) {
 Serial.print("PoorQuality: ");
 Serial.print(poorQuality, DEC);
 Serial.print(" Attention: ");
 Serial.print(attention, DEC);
 Serial.print(" Time since last packet: ");
 Serial.print(millis() - lastReceivedPacket, DEC);
 lastReceivedPacket = millis();
 Serial.print("\n"); 
 }
 bigPacket = false; 
}
void serialEvent(){
 if(Serial.peek() >= 0){
 switch(state){
 case STATE_WAIT_FOR_FIRST_A:
 if(Serial.read() == 170) state = STATE_WAIT_FOR_SECOND_A;
 break;
 case STATE_WAIT_FOR_SECOND_A:
 if(Serial.read() == 170) state = STATE_WAIT_FOR_PAYLOAD_LENGTH;
 break;
 case STATE_WAIT_FOR_PAYLOAD_LENGTH:
 payloadLength = Serial.read();
 if(payloadLength > 169){
 state = STATE_WAIT_FOR_FIRST_A;
 return;
 }
 generatedChecksum = payloadIndex = 0;
 state = STATE_WAIT_FOR_PAYLOAD;
 break;
 case STATE_WAIT_FOR_PAYLOAD:
 if(payloadIndex < payloadLength){
 payloadData[payloadIndex] = Serial.read();
 generatedChecksum += payloadData[payloadIndex];
 payloadIndex++;
 }else{
 state = STATE_WAIT_FOR_CHECKSUM;
 }
 break;
 case STATE_WAIT_FOR_CHECKSUM:
 checksum = Serial.read();
 generatedChecksum = 255 - generatedChecksum;
 if(checksum == generatedChecksum) {
 parsePayload();
 state = STATE_WAIT_FOR_FIRST_A;
 }else{
 state = STATE_WAIT_FOR_FIRST_A;
 }
 break;
 }
 }
}

Answer 1

Interestingly enough, Serial.read() does not work like most of us think.

Serial.read() is not blocking, it always returns immediately with a byte 0-255 or -1 if there is no character to read.

From Arduino Reference

Description

Reads incoming serial data. read() inherits from the Stream utility class.

Syntax

Serial.read()

Parameters

None

Returns

the first byte of incoming serial data available (or -1 if no data is available) - int

• George Profenza: Interesting indeed(+1). I will try to adapt the code without using available() and simply checking if Serial.read() > -1. Is there a change of missing a byte this way ?
• Wirewrap: Serial.read() reads from a 64 bytes buffer. If you can avoid filling up the buffer you will be okay.
• dannyf: "Serial.read() is not blocking, it always returns immediately with a byte 0-255 or -1 if there is no character to read." unless it returns a multi-byte data type, it couldn't return 0-255 or -1 -> as -1 is 128. it likely returns 0-127 and -1 instead.

Answer 2

Nick Gammon, a moderator on the official Arduino site and a very active member of the Arduino community on Stackoverflow, has done a very nice post on reading serial without blocking. I've posted the relevant code. /* Example of processing incoming serial data without blocking.

Author: Nick Gammon
Date: 13 November 2011. 
Modified: 31 August 2013.
Released for public use.
*/
// how much serial data we expect before a newline
const unsigned int MAX_INPUT = 50;
void setup (){
Serial.begin (115200);
} // end of setup
// here to process incoming serial data after a terminator received
void process_data (const char * data)
 {
 // for now just display it
 // (but you could compare it to some value, convert to an integer, etc.)
 Serial.println (data);
 } // end of process_data
void processIncomingByte (const byte inByte)
{
 static char input_line [MAX_INPUT];
 static unsigned int input_pos = 0;
switch (inByte)
 {
 case '\n': // end of text
 input_line [input_pos] = 0; // terminating null byte
 // terminator reached! process input_line here ...
 process_data (input_line);
 // reset buffer for next time
 input_pos = 0; 
 break;
 case '\r': // discard carriage return
 break;
 default:
 // keep adding if not full ... allow for terminating null byte
 if (input_pos < (MAX_INPUT - 1))
 input_line [input_pos++] = inByte;
 break;
 } // end of switch
} // end of processIncomingByte 
void loop()
{
// if serial data available, process it
while (Serial.available () > 0)
 processIncomingByte (Serial.read ());
// do other stuff here like testing digital input (button presses) ...
} // end of loop

Answer 3

I don't like serialEvent() because it only checks to see if it should run that function after the whole loop is executed. With your new example, it's basically the same as the original (except you moved that code to the very bottom of the loop).

I'm not sure exactly how fast the data is sent, but you should first ask yourself if the data is sent fast enough that you don't even notice that it's blocking. Blocking code isn't bad itself (except maybe from a standpoint of the other device freezing up and stalling your program), but blocking code that takes a long time is bad.

Since you need five bytes, the best thing would be to wrap the blocking code in an if statement like this:

if(Serial.available >= 5) {
 //Blocking code here
}

It's simple and readable, yet effective.

• George Profenza: Thanks for the tips (+1). I didn't know about serialEvent being called once at the end of loop. Reading all the bytes in a full packet with a big payload takes between 900ms and 2500ms from what I can observe. Is there a chance of missing bytes using serialEvent instead of while(!Serial.available()) ?
• Anonymous Penguin: @GeorgeProfenza it shouldn't remove any bytes unless it's a major bug.

Answer 4

Since then I've figured out how to rewrite the code to be non-blocking, by replacing the while loop with states, accumulating one byte at a time.

I've written my first Arduino library with this occasion: Mindwave. Here's a basic example:

#include <Mindwave.h> //import the library
Mindwave mindwave; //start using it
void setup() {
 Serial.begin(MINDWAVE_BAUDRATE); //setup serial communication (MindWave mobile is set to 57600 baud rate)
}
//create a function to received new values as soon as they're avaialble
void onMindwaveData(){
 Serial.print("attention: ");
 Serial.println(mindwave.attention()); //access attention value
}
void loop() {
 mindwave.update(Serial,onMindwaveData);//update using the data input(Serial in this case) and the function to call when data is ready
}

It's non blocking, you can see the implementation here

This was used for the Walk with Me art installation.

Walk with Me installation image 1

Walk with Me installation image 2

Bluetooth and DMX Shield for the Walk with Me installation