Reducing Memory Usage lots of Floats

Question

Im using an Arduino Uno - All code at github.com/robbrad/AirQuality

Current sketch usage:

Sketch uses 23186 bytes (71%) of program storage space. Maximum is 32256 bytes.
Global variables use 1428 bytes (69%) of dynamic memory, leaving 620 bytes for local variables. Maximum is 2048 bytes.

I wouldn't normally just spam code for a optimisation - but im looking to get about 10% memory back from this sketch

It basically checks a bunch of sensors sets the values to floats, build a string and posts that to a php page where some more processing happens.

I have tried everything to my knowledge but I just cant think of a smarter way to use less memory.

I tried an array, a function to build the data string, using one float and concatenating it to the string when it set

I even thought of reducing the Math.h include to just the math functions I was using

Maybe im just reaching the little boards limits (which are pretty good anyway :) ! )

#include <SPI.h>
#include <Ethernet.h>
#include "DHT.h"
#include <math.h>
#include <Wire.h>
#include "MutichannelGasSensor.h"
#include "HP20x_dev.h"
#include "KalmanFilter.h"
unsigned char ret = 0;
/* Instance */
KalmanFilter t_filter; //temperature filter
KalmanFilter p_filter; //pressure filter
KalmanFilter a_filter; //altitude filter
// Enter a MAC address for your controller below.
// Newer Ethernet shields have a MAC address printed on a sticker on the shield
byte mac[] = { 0xDE, 0xAD, 0xBE, 0xEF, 0xFE, 0xED };
// if you don't want to use DNS (and reduce your sketch size)
// use the numeric IP instead of the name for the server:
IPAddress server(192, 168, 0, 30); // numeric IP for Google (no DNS)
//char server[] = "www.google.com"; // name address for Google (using DNS)
// Set the static IP address to use if the DHCP fails to assign
IPAddress ip(192, 168, 0, 40);
// Initialize the Ethernet client library
// with the IP address and port of the server
// that you want to connect to (port 80 is default for HTTP):
EthernetClient client;
#define Vc 4.95
//the number of R0 you detected just now
#define R0 35.54
#define DHTPIN A1 // what pin we're connected to
// Uncomment whatever type you're using!
//#define DHTTYPE DHT11 // DHT 11
#define DHTTYPE DHT22 // DHT 22 (AM2302)
//#define DHTTYPE DHT21 // DHT 21 (AM2301)
DHT dht(DHTPIN, DHTTYPE);
int pin = 8;
unsigned long duration;
unsigned long starttime;
unsigned long sampletime_ms = 2000;//sampe 30s&nbsp;;
unsigned long lowpulseoccupancy = 0;
float ratio = 0;
float concentration = 0;
String data = "";
void setup() {
 Serial.begin(9600);
 while (!Serial) {
 ; // wait for serial port to connect. Needed for native USB port only
 }
 delay(150);
 HP20x.begin();
 delay(100);
 /* Determine HP20x_dev is available or not */
 ret = HP20x.isAvailable();
 pinMode(8, INPUT);
 starttime = millis();//get the current time;
 //Serial.println("power on!");
 pinMode(4, OUTPUT);
 digitalWrite(4, HIGH);
 //if (Ethernet.begin(mac) == 0) {
 // Serial.println("Failed to configure Ethernet using DHCP");
 // try to congifure using IP address instead of DHCP:
 Ethernet.begin(mac, ip);
 // give the Ethernet shield a second to initialize:
 delay(10000);
 // Serial.println("connecting...");
 //}
 // give the Ethernet shield a second to initialize:
 //delay(1000);
 //Serial.println("connecting...");
 // Serial.println(Ethernet.localIP());
 gas.begin(0x04);//the default I2C address of the slave is 0x04
 gas.powerOn();
 //Serial.print("Firmware Version = ");
 //Serial.println(gas.getVersion());
 //Serial.println("Particles\tRS\tHCHO (PPM)\tNH3 (PPM)\tCO (PPM)\tNO2 (PPM)\tC3H8 (PPM)\tC4H10 (PPM)\tCH4 (PPM)\tH2 (PPM)\tC2H5OH (PPM)");
 dht.begin();
 delay(10000);
}
void loop() {
 if (client.available()) {
 char c = client.read();
 // Serial.print(c);
 }
 float HATemp2;
 float HAPres2;
 float HAAlt2;
 float d;
 if (OK_HP20X_DEV == ret)
 {
 unsigned long HATemp = HP20x.ReadTemperature();
 d = HATemp / 100.0;
 HATemp2 = t_filter.Filter(d);
 unsigned long HAPres = HP20x.ReadPressure();
 d = HAPres / 100.0;
 HAPres2 = p_filter.Filter(d);
 unsigned long HAAlt = HP20x.ReadAltitude();
 d = HAAlt / 100.0;
 HAAlt2 = a_filter.Filter(d);
 }
 duration = pulseIn(pin, LOW);
 lowpulseoccupancy = lowpulseoccupancy + duration;
 if ((millis() - starttime) >= sampletime_ms) //if the sampel time = = 30s
 {
 ratio = lowpulseoccupancy / (sampletime_ms * 10.0); // Integer percentage 0=&gt;100
 d = 1.1 * pow(ratio, 3) - 3.8 * pow(ratio, 2) + 520 * ratio + 0.62; // using spec sheet curve
 //Serial.print("concentration = ");
 lowpulseoccupancy = 0;
 starttime = millis();
 }
 //Serial.print("\t");
 //HCHO
 int sensorValue = analogRead(A0);
 double Rs = (1023.0 / sensorValue) - 1;
 double ppm = pow(10.0, ((log10(Rs / R0) - 0.0827) / (-0.4807)));
 //Serial.print("HCHO ppm = ");
 //MultiChannel Gas
 float NH3 = gas.measure_NH3();
 float CO = gas.measure_CO();
 float NO2 = gas.measure_NO2();
 float C3H8 = gas.measure_C3H8();
 float C4H10 = gas.measure_C4H10();
 float CH4 = gas.measure_CH4();
 float H2 = gas.measure_H2();
 float C2H5OH = gas.measure_C2H5OH();
 // Reading temperature or humidity takes about 250 milliseconds!
 // Sensor readings may also be up to 2 seconds 'old' (its a very slow sensor)
 float h = dht.readHumidity();
 float t = dht.readTemperature();
 data = String("dust=") + d + "&rs=" + Rs + "&hcho=" + ppm + "&nh3=" + NH3 + "&co=" + CO + "&no2=" + NO2 + "&c3h8=" + C3H8 + "&c4h10=" + C4H10 + "&ch4=" + CH4 + "&h2=" + H2 + "&c2h5oh=" + C2H5OH + "&temp=" + t + "&hum=" + h + "&HATemp=" + HATemp2 + "&HAPres=" + HAPres2 + "&HAAlt=" + HAAlt2;
 //Serial.println(data);
 if (client.connect(server, 80)) {
 Serial.println(F("connected"));
 client.println(F("POST /air_add.php HTTP/1.1"));
 client.println(F("Host: 192.168.0.30"));
 client.println(F("User-Agent: Arduino/1.0"));
 client.println(F("Connection: close"));
 client.println(F("Content-Type: application/x-www-form-urlencoded;"));
 client.print(F("Content-Length: "));
 client.println(data.length());
 client.println();
 client.println(data);
 }
 else
 {
 Serial.println(F("could not connect"));
 }
 Serial.println(data);
 if (client.connected()) {
 client.stop();
 }
 delay(300000);
}

• Jot: For which Arduino board? Are you trying to reduce ram or flash memory? What does the compiler say about ram and flash memory usage? Those few float variables on the stack are not the problem. The libraries might use a lot of memory. If no library uses the 'String' object, then you should try to avoid that. You can also send bare sensor data, and do the calculation and processing in the server.
• Rob: It’s an Uno. The compiler didn’t complain but it’s right on the 70% and 69% limi for both respectively. I had a quick look at the libraries but not seeing anything jumping out at me. Maybe I’ll make a sketch with just the libraries to see just their usage
• Edgar Bonet: Is this your KalmanFilter? I can't quite make sense out of it. To me, it looks silly and very memory intensive. Couldn't you use a basic first-order low-pass instead?
• Nick Gammon: Even if you used 99.9% of PROGMEM, that would not affect stability. However with string fragmentation (particularly caused by the String class) even 50% of RAM used might be too much. It looks like you have found the problem.
• Rob: @Peter your too kind sir, I’m more a jack of all trades master of none. But your right I’m fast realising it’s quite a different arena to work in and not all initially learned principles apply

Answer 1

I counted the floats in the sketch:

Global: 2
setup: 0
loop: 14
Total: 16 floats -> 64 bytes.

If you want to reduce that, don't wait to have all your values before preparing your answer. You will use less memory if you do

 data = String("dust=") + dht.readHumidity();
 data+= "&rs=" + (1023.0 / sensorValue) - 1;
 data+= "&hch0=" + pow(10.0, ((log10(Rs / R0) - 0.0827) / (-0.4807)));
 ... and the rest ...

Your data String and concat's take way more space than the floats. Use char[] instead of string.

As last resort, you don't need to construct the complete POST before sending it to the server. Send it line by line, value by value. Use a fixed, big enough Content-Length:, padding with spaces after you send the values and know the actual lenght.

Or you can switch your technology: MQTT need way less memory than HTTP.

• Rob: MQTT might be the ticket - I might send the value as soon as I have it
• Rob: hmm interesting - I tried char data[15]; and the String("dust=") didn't like it - My understanding is data[] would now be an array - is that right?
• user31481: @Rob. You use char[] instead of String(), not with or intermixed. You have to go low-level, char by char, byte by byte.
• Rob: There have been some great responses on this question. It was hard to accept an answer, but as this one mentioned the String issue first I choose this one

Answer 2

Two basic approaches:

1) use fixed point math. or

1) use int16_t or int32_t in lieu of the floating types. for example, you can use x10 or x100 to designate variables with some decimal points. use x16 or x128 or even x256 for speed gains as well. you will need to make sure that it doesn't overflow.

edit: here is an example that hopefully will help you get it going.

void setup() {
 Serial.begin(9600);
 var1=100; //var1=1.00
 var2=123; //var2=1.23
}
void loop() {
 var1+=var2; //increment var1
 Serial.print("var1 = "); Serial.print(var1 / 100); Serial.print("."); Serial.print(var1 - (var1 / 100) * 100); Serial.println(".");
 delay(100);
}

var1 (1.00 initially) is incremented by var2 (1.23), both are int32_t types.

here is the printout.

enter image description here

• Rob: Ok so what you're saying is I can use int16_t instead of a float? do I have to cast the result in some other way? doing more reading now
• Rob: That’s pretty cool, never done this before. So var1 is of type int32_t. So when it’s taken to the string it’s only converted to the correct precision at the point - is this the essence of fixed point math?
• user31481: So, are you using int32_t instead of float? Both are 4 bytes long.
• Eric Johnson: @dannyf What software are you using in the last picture? Is that an emulator?
• dannyf: " is this the essence of fixed point math?" yeah. fixed point math is in base 2, and what I have shown is in base 10. using base 2 speeds up calculation and using base 10 is more intuitive.

Answer 3

I second the advice that has already been given to you about avoiding String. Other than that, using fixed-point math could be a big win, but given the math you do it won't be any easy.

Assuming you stick with floating point, I just found a couple of easy optimization opportunities:

• Do not use pow to evaluate a polynomial. Use the Horner's method which is way cheaper: d = ((1.1*ratio - 3.8)*ratio + 520)*ratio + 0.62.
• The expression of ppm can be simplified: ppm = 2500.33 * pow(Rs, -2.0803).

---

Edit: You could also try to implement chunked transfer encoding. This way you could send the data to the server on the fly, as you collet it, instead of storing everything in memory and sending it in one go.

Example:

void send_chunk(Print &connection, const String &chunk)
{
 connection.println(chunk.length, 16);
 connection.println(chunk);
}
void loop() {
 // ...
 // last header sent to the client:
 client.println(F("Transfer-Encoding: chunked"));
 client.println();
 // Now the data:
 String chunk;
 chunk = "dust=";
 chunk += ((1.1*ratio - 3.8)*ratio + 520)*ratio + 0.62;
 send_chunk(client, chunk);
 chunk = "&rs=";
 chunk += Rs;
 send_chunk(client, chunk);
 chunk = "&hcho=";
 chunk += 2500.33 * pow(Rs, -2.0803);
 send_chunk(client, chunk);
 chunk = "&nh3=";
 chunk += gas.measure_NH3();
 send_chunk(client, chunk);
 // and so on...
 chunk = "";
 send_chunk(client, chunk); // Empty chunk means end of message.
 // ...
}

Notice that there is no "Content-Length" header. Instead, each chunk is preceded by its size in hex. And, most importantly, notice that this way you can get rid of lots of variables.

---

Edit 2: I came out with a better (more generic) way to implement chunked encoding, which involves sending the data through a "chunker" filter that does the encoding and outputs to the client. Here is my Chunker class:

/*
 * Transfer a message using the HTTP/1.1 "Chunked Transfer Coding".
 * C.f. https://tools.ietf.org/html/rfc7230#section-4.1
 */
class Chunker : public Print
{
public:
 Chunker(Print &downstream) : downstream(downstream), pos(0) {}
 virtual size_t write(uint8_t c)
 {
 buffer[pos++] = c;
 if (pos == BUFFER_SIZE)
 send();
 return 1;
 }
 void end()
 {
 if (pos) send();
 send(); // empty chunk means end of message
 }
private:
 static const uint8_t BUFFER_SIZE = 32;
 Print &downstream;
 uint8_t buffer[BUFFER_SIZE];
 uint8_t pos;
 // Send the buffer as a chunk.
 void send()
 {
 downstream.println(pos, HEX);
 if (pos) {
 downstream.write(buffer, pos);
 downstream.println();
 pos = 0;
 }
 }
};

And here is how you would use it:

Chunker message(client); // instantiate the filter
message.print(F("dust="));
message.print(((1.1*ratio - 3.8)*ratio + 520)*ratio + 0.62);
message.print("&rs=");
message.print(Rs);
message.print(F("&hcho="));
message.print(2500.33 * pow(Rs, -2.0803));
message.print(F("&nh3="));
message.print(gas.measure_NH3());
// and so on...
message.end(); // terminate the chunked message

The main advantage of this method is that you completely avoid using the String class, which means no dynamic memory allocation (a very desirable thing). Also you can choose your buffer size. I've set it to 32 bytes, but you can make it smaller if needed.

• Edgar Bonet: @Rob: I added another suggestion about the usage of the HTTP protocol.
• Rob: Thats really cool - thank you for spending some time on this - I think im edging towards this approach
• Edgar Bonet: @Rob: Here is a more generic way of using chunked encoding, which avoids String, or even the need to guess how mush space you need to sprintf() a number.
• Rob: Thats pretty amazing ... I just need to work out the particulars like the connection part. My Endpoint is a php page/server which does a load more things with the POST request - would this be a good place to start dancingmammoth.com/2009/08/29/…
• Edgar Bonet: @Rob: I don't know much about PHP. Not sure whether you have to take for the encoding yourself. The link you provide is for adding chunked support to PHP streams, which are a useful tool for handling very large payloads. You payload is tiny, so you don't need them. Also, I assume your web server will take care of the decoding step, at least if using mod_php. Note, however, that there is a PHP bug that prevents this to work in fast CGI mode.