Hello everybody,
I would like to print on serial port a json file with weather forecasts on my esp8266.
Host is in SHTTP
For small json files, this works perfectly…
But on this site : https://www.prevision-meteo.ch/services/json/nantes the file is not complete, i get only roughly between 1440 and 3000 chars…
I m’ not sure how to enable wificlient debug strings… but no crash / error is reported when this function is executed:
Thanks a lots !
Matthieu
/////////////////////////////////////////////////////////////////////////////////
// void Recuperer_MTO(void)
// MORE INFOS : https://github.com/esp8266/Arduino/issues/4814
// : https://github.com/bblanchon/ArduinoJson/blob/6.x/examples/JsonHttpClient/JsonHttpClient.ino
//
// Connect on https://prevision-meteo.ch website then get json document and parse it
// We don't care about certificate thanks to ===> Client_web.setInsecure();
// Json File is there: https://www.prevision-meteo.ch/services/json/nantes
// Use libraries bblanchon/ArduinoJson, WiFiClientSecure, ESP8266WiFi
//
// (Fonction qui se connecte a un serveur de données MTO mondial renvoyant un JSON)
// Ignore les certificats grace à la commande ===> Client_web.setInsecure();
/////////////////////////////////////////////////////////////////////////////////
void Recuperer_MTO(void)
{
// FOR DEBUGGING...
#define DEBUGV(fmt, ...) ::printf((PGM_P)PSTR(fmt), ## __VA_ARGS__)
#define DEBUG_BSSL(fmt, ...) ::printf((PGM_P)PSTR(fmt), ## __VA_ARGS__)
const char* host = "prevision-meteo.ch"; // Web site web où se connecter / Host @ to connect
const int httpsPort = 443; // PORT HTTPS
#include <WiFiClientSecure.h>
//if (!WiFi_connecte) return; // Si l'on est pas connecté on quitte / Exit if not connected
#ifndef STASSID
#define STASSID "MY SSID"
#define STAPSK "A_BIG_PASSWORD"
#endif
const char* ssid = STASSID;
const char* password = STAPSK;
// Use WiFiClientSecure class to create TLS connection
Serial.print("Connecting to ");
Serial.println(ssid);
WiFi.mode(WIFI_STA);
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED)
{
delay(500);
Serial.print(".");
}
Serial.println("");
Serial.println("WiFi connected");
Serial.println("IP address: ");
Serial.println(WiFi.localIP());
WiFiClientSecure Client_web;
Client_web.setInsecure(); // Ignore l'autenticité du site sur lequel on se connecte / Discard certificate
Serial.print("Connecting to ");
Serial.println(host);
Client_web.setTimeout(10000); // needed ?
Client_web.setBufferSizes(10000, 1000); // needed ?
if (!Client_web.connect(host, httpsPort)) {Serial.println("Connection failed"); return;}
String url = "/services/json/nantes/";
Serial.print("Requesting URL: ");
Serial.println(url);
// Send HTTP request
Client_web.prinlnt(String("GET ") + url + " HTTP/1.0\r\n" + "Host: " + host + "\r\nConnection: close\r\n"); // HTTP 1.0 pour éviter les réponses découpées (chunked) / Use HTTP 1.0 to avoid chunked answers
if (Client_web.println() == 0) {Serial.println(F("ERROR : Failed to send request")); return;}
Serial.println(F("OK : Request sent"));
while (Client_web.connected()) // Get header: "OK 200... " ==> need to add header validity check (err 404, etc)
{
String line = Client_web.readStringUntil('\n');
if (line == "\r")
{
Serial.println("headers received");
break;
}
}
while (Client_web.available()) // FOR NOW, JUST PRINT RECEIVED RAW JSON FILE, BUF FILE IS NOT COMPLETE !! (+/-1440 chars out of +/-54151 chars)
{
char c = Client_web.read();
Serial.write(c);
}
// Disconnect
Client_web.stop();
}
But if you’re constantly checking for available data, there might be points in time where you have read all currently buffered received data but the connection is still and open and the server will send more. So you are prematurely exiting this loop. Since you’re already specifying Connection: Close in the HTTP header, just make the condtion so that you don’t exit the read loop if the connection is still open with
while (Client_web.available() || Client_web.connected()) {
}
Also this is
- wrong because of no error handling (function returns
int with error code -1)
- extremely inefficient to read and read and write byte-wise. Create a buffer on the stack and read in it, e.g. using the read(buf, size) function
char rxData[256];
while (Client_web.available() || Client_web.connected()) {
int ret = Client_web.read(rxData, sizeof(rxData));
if (ret <= 0) { /* error (-1) or no data (0) */ }
else {
//length of actually received data is ret, if positive.
Serial.print(rxData, ret);
}
}
Also consider using the premade ESP8266HTTPClient library which has HTTPS examples and does streaming much better.
Thanks a lots maxgerhardt, I will try that this evening.
char c = Client_web.read();
This was just for a small test 
Il will use bblanchon/ArduinoJson lib to parse the data, but on my last try, he tolds me “NoMemory” (I gess due to my truncated data)
Does ESP8266HTTPClient allows ignoring fingerprint (like Client_web.setInsecure(); ?
Matthieu
As you can see by reading the code
The HTTP client is instantiated with the client object of type WiFiClientSecure, you can call the same function on that as you do in your initial sketch, e.g. in line 42.
Great! Thanks it’s works ! :o)
The next challenge was to parse the big json file with the little memory of the ESP…
After trying several json parser’s libraries, I wrote my own (minimalist) parser (not a perfect one, but its sufficient for what I need, and it parse incoming stream without store it in ram).
Here I give you the code, hope it can help someone :o)
/////////////////////////////////////////////////////////////////////////////////
// void Recuperer_MTO(void)
//
//
// Connect on prevision-meteo ch website then get json document and parse it
// We don't care about certificate thanks to ===> Client_web.setInsecure();
// Use libraries WiFiClientSecure, ESP8266WiFi
//
// (Fonction qui se connecte a un serveur de données MTO renvoyant un JSON)
// Ignore les certificats grace à la commande ===> Client_web.setInsecure();
/////////////////////////////////////////////////////////////////////////////////
void Recuperer_MTO(void)
{
const char* host = "prevision-meteo.ch"; // Web site web où se connecter / Host @ to connect
const int httpsPort = 443; // PORT HTTPS
//if (!WiFi_connecte) return; // Si l'on est pas connecté on quitte / Exit if not connected
#ifndef STASSID
#define STASSID "myssid"
#define STAPSK "mypassword,"
#endif
const char* ssid = STASSID;
const char* password = STAPSK;
char ville[20] = {0};
char sunrise[6] = {0};
char sunset[6] = {0};
char tmp[20] = {0};
char prev_j0[20] = {0};
float latitude = 0;
float longitude = 0;
short t_min_j0 = 0;
short t_max_j0 = 0;
Serial.print("Connecting to ");
Serial.println(ssid);
WiFi.mode(WIFI_STA);
WiFi.begin(ssid, password);
while (WiFi.status() != WL_CONNECTED)
{
delay(500);
Serial.print(".");
}
Serial.println(F(""));
Serial.println(F("WiFi connected"));
Serial.println(F("IP address: "));
Serial.println(WiFi.localIP());
// Create TLS connection
std::unique_ptr<BearSSL::WiFiClientSecure> client(new BearSSL::WiFiClientSecure);
bool mfln = client->probeMaxFragmentLength(host, 443, 1024);
#ifdef DEBUG_VERBOSE
Serial.printf("Maximum fragment Length negotiation supported: %s\n", mfln ? "yes" : "no");
Serial.print(F("[HTTPS] begin...\n"));
#endif
if (mfln) client->setBufferSizes(1024, 1024);
client->setInsecure(); // Ignore l'autenticité du site sur lequel on se connecte / Discard certificate
HTTPClient https;
if (https.begin(*client, "https://www.prevision-meteo.ch/services/json/nantes/"))
{
#ifdef DEBUG_VERBOSE
Serial.print(F("[HTTPS] GET...\n"));
#endif
// start connection and send HTTP header
int httpCode = https.GET();
if (httpCode > 0)
{
// HTTP header has been send and Server response header has been handled
#ifdef DEBUG_VERBOSE
Serial.printf("[HTTPS] GET... code: %d\n", httpCode);
#endif
// file found at server
if (httpCode == HTTP_CODE_OK || httpCode == HTTP_CODE_MOVED_PERMANENTLY)
{
// get lenght of document (is -1 when Server sends no Content-Length header)
int len = https.getSize();
#define NB_TOKEN 10 // Number of tokens to be identified in the json file (sections and key)
char* tokens[NB_TOKEN] = {"city_info\":{", "name\":\"", "latitude\":\"", "longitude\":\"", "sunrise\":\"", "sunset\":\"", "fcst_day_0\":{", "tmin\":", "tmax\":", "condition_key\":\""}; // Token's names
short tokens_found[NB_TOKEN] = {0}; // If a specified token was found
unsigned short idx_tokens[NB_TOKEN] = {0}; // Position in a specified token to be parsed
char ville_idx = 0; // JSON query result data storaged: City
char sunrise_idx = 0; // JSON query result data storaged: Sunrise
char sunset_idx = 0; // JSON query result data storaged: Sunset
char tmp_idx = 0; // JSON query result data storaged: Temprary file used for float or int conversions
char prev_j0_idx = 0; // JSON query result data storaged: String showing forecast for today
while (https.connected() && (len > 0 || len == -1)) // read all data from server
{
size_t size = client->available(); // get available data size
if (size) // There is still data to be red
{
uint8_t buff[128] = { 0 }; // create buffer for read, (erased on every loop)
int c = client->readBytes(buff, ((size > sizeof(buff)) ? sizeof(buff) : size)); // read up to 128 byte, store data read into buff
for (unsigned short k = 0; k < 128; k++) // Parse the jsonfile piece by piece... (=> Very low memory usage...)
{
for (short tok = 0; tok < NB_TOKEN; tok++) // For each searched token...
{
if((tokens_found[tok] == 0) && (buff[k] == tokens[tok][idx_tokens[tok]])) // Compare each char of not yet found tokens with the received buff data
{
idx_tokens[tok]++; // A char match for current token
if (idx_tokens[tok] == strlen(tokens[tok])) {tokens_found[tok] = 1; k++;} // If a token was totally identified
}
else idx_tokens[tok] = 0; // No matching char for a specified token => Sentence was starting like the token, but eventually this is not the token => erase idx_tokens[tok] to try again later
}
short end = buff[k] != '\"'; // A key end with " char
if (tokens_found[1] == 1) {if (tokens_found[0] == 0) tokens_found[1] = 0; else {if (end) ville[ville_idx++] = buff[k]; else {tokens_found[1] = 2;}}} // Ville / City
if (tokens_found[2] == 1) {if (tokens_found[0] == 0) tokens_found[2] = 0; else {if (end) tmp[tmp_idx++] = buff[k]; else {tokens_found[2] = 2; latitude = atof(tmp); for (unsigned short j = 0; j < 20; j++)tmp[j] = 0; tmp_idx = 0;}}} // Latitude
if (tokens_found[3] == 1) {if (tokens_found[0] == 0) tokens_found[3] = 0; else {if (end) tmp[tmp_idx++] = buff[k]; else {tokens_found[3] = 2; longitude = atof(tmp); for (unsigned short j = 0; j < 20; j++)tmp[j] = 0; tmp_idx = 0;}}} // Longitude
if (tokens_found[4] == 1) {if (tokens_found[0] == 0) tokens_found[4] = 0; else {if (end) sunrise[sunrise_idx++] = buff[k]; else {tokens_found[4] = 2;}}} // Lever du soleil / Sunrise
if (tokens_found[5] == 1) {if (tokens_found[0] == 0) tokens_found[5] = 0; else {if (end) sunset[sunset_idx++] = buff[k]; else {tokens_found[5] = 2;}}} // Coucher du soleil / Sunset
if (tokens_found[7] == 1) {if (tokens_found[6] == 0) tokens_found[7] = 0; else {if (end) tmp[tmp_idx++] = buff[k]; else {tokens_found[7] = 2; t_min_j0 = atoi(tmp); for (unsigned short j = 0; j < 20; j++)tmp[j] = 0; tmp_idx = 0;}}} // Temperature max j0 (today)
if (tokens_found[8] == 1) {if (tokens_found[6] == 0) tokens_found[8] = 0; else {if (end) tmp[tmp_idx++] = buff[k]; else {tokens_found[8] = 2; t_max_j0 = atoi(tmp); for (unsigned short j = 0; j < 20; j++)tmp[j] = 0; tmp_idx = 0;}}} // Temperature max j0 (today)
if (tokens_found[9] == 1) {if (tokens_found[6] == 0) tokens_found[9] = 0; else {if (end) prev_j0[prev_j0_idx++] = buff[k]; else {tokens_found[9] = 2;}}} // Prévision j0 / Forecast (today)
}
yield(); // Avoid watchdog trigger...
if (len > 0) len -= c; // Count the number of chars to be read
if (tokens_found[NB_TOKEN-1]) break; // If we have found all we need, stop parsing file and close connection
}
delay(1);
}
// Serial.print(reponse); // For debug
#ifdef DEBUG_VERBOSE
Serial.print("[HTTPS] connection closed or file end.\n");
#endif
}
#ifdef DEBUG_VERBOSE
// Display results
Serial.println("Ville: " + String(ville)); // City's name
Serial.println("latitude: " + String(latitude)); // Latitude (decimal)
Serial.println("longitude: " + String(longitude)); // Longitude (decimal)
Serial.println("Lever du soleil: " + String(sunrise)); // Sunrise
Serial.println("Coucher du soleil: " + String(sunset)); // Sunset
Serial.println("Temperature min/max: " + String(t_min_j0) + String(t_max_j0)); // Temperature of the day (°C)
Serial.println("Prévision du jour: " + String(prev_j0)); // Today forecast
#endif
}
#ifdef DEBUG_VERBOSE
else Serial.printf("[HTTPS] GET... failed, error: %s\n", https.errorToString(httpCode).c_str());
#endif
}
#ifdef DEBUG_VERBOSE
else Serial.printf("Unable to connect\n");
#endif
https.end();
}