[TerkinData] Add SQL serializer

libraries/TerkinData/README.rst

@@ -19,7 +19,9 @@ TerkinData C++
 Introduction
 ************
 
-TerkinData C++ is a convenient library for handling sensor readings.
+TerkinData C++ is a convenient library for collecting sensor readings, and
+for marshalling them into different output formats, like CSV, JSON, SQL, or
+x-www-form-urlencoded.
 
 It helps to decouple the sensor reading domain from the telemetry domain in a
 typical data logger application. While providing a generic interface, it can
@@ -171,6 +173,20 @@ JSON
     delete measurement;
 
 
+SQL
+---
+
+::
+
+    // Define table name.
+    std::string table_name = "testdrive";
+
+    // Serialize data into SQL CREATE statement.
+    datamgr->sql_create(table_name, *measurement);
+
+    // Serialize data into SQL INSERT statement.
+    datamgr->sql_insert(table_name, *measurement);
+
 
 *****
 Usage
@@ -274,6 +290,29 @@ JSON
 .. seealso:: Full source of `<json_basic.cpp_>`_.
 
 
+.. _terkindata-sql-example:
+
+SQL
+===
+
+::
+
+    ==============================
+    TerkinData SQL DDL+DML example
+    ==============================
+
+    -- Test DDL
+    data:   CREATE TABLE testdrive ('time' DATETIME WITH TIMEZONE,'weight' FLOAT,'temperature_outside' FLOAT,'humidity_outside' FLOAT,'temperature_inside' FLOAT,'voltage' FLOAT);
+
+    -- Test DML single reading (complete)
+    data:   INSERT INTO testdrive (time,weight,temperature_outside,humidity_outside,temperature_inside,voltage) VALUES ('2023-06-03T13:14:24Z',85.000000,42.419998,84.839996,33.330002,3.843000);
+
+    -- Test DML single reading (incomplete)
+    data:   INSERT INTO testdrive (time,temperature_outside,humidity_outside,voltage) VALUES ('2023-06-03T13:14:24Z',42.419998,84.839996,3.843000);
+
+.. seealso:: Full source of `<sql_basic.cpp_>`_.
+
+
 ********
 Download
 ********
@@ -317,3 +356,4 @@ Terkin
 .. _csv_basic.cpp: https://github.com/hiveeyes/arduino/blob/main/libraries/TerkinData/examples/csv/csv_basic.cpp
 .. _json_basic.cpp: https://github.com/hiveeyes/arduino/blob/main/libraries/TerkinData/examples/json/json_basic.cpp
 .. _urlencoded_basic.cpp: https://github.com/hiveeyes/arduino/blob/main/libraries/TerkinData/examples/urlencoded/urlencoded_basic.cpp
+.. _sql_basic.cpp: https://github.com/hiveeyes/arduino/blob/main/libraries/TerkinData/examples/urlencoded/sql_basic.cpp

libraries/TerkinData/TerkinData.cpp

@@ -201,3 +201,85 @@ std::string DataManager::json_data(Measurement& measurement) {
 #endif
 
 }
+
+
+/**
+ *
+ * Serialize measurement data to SQL CREATE statement
+ *
+**/
+std::string DataManager::sql_create(std::string table, Measurement& measurement) {
+
+    // Translate sensor value names to telemetry field names.
+    this->map_fields(measurement);
+
+    // Containers for SQL CREATE statements' columns, with types.
+    std::vector<std::string> fields;
+
+    // Iterate header field names.
+    for (std::string name: *this->field_names) {
+
+        // #include <stdio.h>
+        // std::cout << "name: " << name << std::endl;
+
+        // Handle .time specially
+        if (name == "time") {
+            fields.push_back("'" + name + "'" + " DATETIME WITH TIMEZONE");
+            continue;
+        }
+
+        fields.push_back("'" + name + "'" + " FLOAT");
+    }
+
+    // Serialize to SQL INSERT statement.
+    std::string fields_clause = join(fields, ',');
+    std::string sql = "CREATE TABLE " + table + " (" + fields_clause + ");";
+
+    return sql;
+
+}
+
+
+/**
+ *
+ * Serialize measurement data to SQL INSERT statement
+ *
+**/
+std::string DataManager::sql_insert(std::string table, Measurement& measurement) {
+
+    // Translate sensor value names to telemetry field names.
+    this->map_fields(measurement);
+
+    // Containers for SQL INSERT statements' columns and values clauses.
+    std::vector<std::string> columns;
+    std::vector<std::string> values;
+
+    // Iterate header field names.
+    for (std::string name: *this->field_names) {
+
+        // #include <stdio.h>
+        // std::cout << "name: " << name << std::endl;
+
+        // Handle .time specially
+        if (name == "time") {
+            columns.push_back(name);
+            values.push_back("'" + measurement.time + "'");
+            continue;
+        }
+
+        // Add values of sensor readings
+        if (key_exists(measurement.data, name)) {
+            float value = measurement.data[name];
+            columns.push_back(name);
+            values.push_back(to_string(value));
+        }
+    }
+
+    // Serialize to SQL INSERT statement.
+    std::string columns_clause = join(columns, ',');
+    std::string values_clause = join(values, ',');
+    std::string sql = "INSERT INTO " + table + " (" + columns_clause + ") VALUES (" + values_clause + ");";
+
+    return sql;
+
+}

libraries/TerkinData/TerkinData.h

@@ -93,6 +93,10 @@ namespace TerkinData {
             // Serialize measurement data to JSON format
             std::string json_data(Measurement&);
 
+            // Serialize measurement data to SQL statements
+            std::string sql_create(std::string, Measurement&);
+            std::string sql_insert(std::string, Measurement&);
+
             // Serialize measurement data to x-www-urlencoded format
             std::string urlencode_data(Measurement&);
 

libraries/TerkinData/examples/Makefile

@@ -21,11 +21,11 @@ export BUILD_DIR := $(PWD)/.build
 # =================
 
 # Main entrypoint targets.
-all: run build
+all: setup-virtualenv run build
 run: posix-run
 
 # Run all POSIX program examples.
-posix-run: csv json urlencoded
+posix-run: csv json sql urlencoded
 
 
 # ==================================================
@@ -35,6 +35,7 @@ posix-run: csv json urlencoded
 build: setup-virtualenv
 	PLATFORMIO_SRC_DIR=csv $(pio) run  # --verbose
 	PLATFORMIO_SRC_DIR=json $(pio) run  # --verbose
+	PLATFORMIO_SRC_DIR=sql $(pio) run  # --verbose
 	PLATFORMIO_SRC_DIR=urlencoded $(pio) run  # --verbose
 
 
@@ -44,12 +45,16 @@ build: setup-virtualenv
 
 .PHONY: csv
 csv:
-	PLATFORMIO_SRC_DIR=csv pio run -e native -t exec
+	PLATFORMIO_SRC_DIR=csv $(pio) run -e native -t exec
 
 .PHONY: json
 json:
-	PLATFORMIO_SRC_DIR=json pio run -e native -t exec
+	PLATFORMIO_SRC_DIR=json $(pio) run -e native -t exec
+
+.PHONY: sql
+sql:
+	PLATFORMIO_SRC_DIR=sql $(pio) run -e native -t exec
 
 .PHONY: urlencoded
 urlencoded:
-	PLATFORMIO_SRC_DIR=urlencoded pio run -e native -t exec
+	PLATFORMIO_SRC_DIR=urlencoded $(pio) run -e native -t exec

libraries/TerkinData/examples/sql/sql_basic.cpp

@@ -0,0 +1,165 @@
+/**
+ *
+ * TerkinData: A small data collection framework for decoupling sensor reading and telemetry domains.
+ * SQL DML example: Collect measurement readings and serialize to `SQL INSERT` statement.
+ *
+ *
+ *  Copyright (C) 2023  Andreas Motl <[email protected]>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU Lesser General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  https://www.gnu.org/licenses/lgpl-3.0.txt
+ *
+**/
+
+// Terrine: Application boilerplate.
+#include <Terrine.h>
+Terrine terrine;
+
+// TerkinData: A small data collection framework for decoupling sensor reading and telemetry domains.
+#include <TerkinData.h>
+using namespace TerkinData;
+using namespace TerkinUtil;
+
+// Miscellaneous utilities.
+using std::string;
+std::string separator = std::string(42, '=');
+
+
+// -------------------------
+// DataManager configuration
+// -------------------------
+void DataManager::setup() {
+
+    // List of field names
+    this->field_names  = new DataHeader({"time", "weight", "temperature_outside", "humidity_outside", "temperature_inside", "voltage"});
+
+    // Map names of lowlevel sensor values to highlevel telemetry data fields
+    (*this->sensor_field_mapping)[string("dht.0.temp")]   = string("temperature_outside");
+    (*this->sensor_field_mapping)[string("dht.0.hum")]    = string("humidity_outside");
+    (*this->sensor_field_mapping)[string("ds18b20.0")]    = string("temperature_inside");
+
+}
+
+DataManager *datamgr = new DataManager();
+
+void getTimestamp(Measurement& measurement) {
+    //measurement.time = "2012-03-02T04:07:34.0218628Z";
+    measurement.time = now_iso();
+}
+
+// Forward declarations
+void invoke_create(Measurement *measurement);
+void invoke_insert(Measurement *measurement);
+
+
+void basic_create() {
+
+    terrine.log("-- Test DDL");
+
+    // Data container to define schema.
+    Measurement *measurement = new Measurement();
+
+    // Display SQL CREATE statement.
+    invoke_create(measurement);
+
+    // Free memory
+    delete measurement;
+
+}
+
+void basic_insert_single() {
+
+    terrine.log("-- Test DML single reading (complete)");
+
+    // Data container to collect one reading
+    Measurement *measurement = new Measurement();
+
+    // Fill timestamp
+    getTimestamp(*measurement);
+
+    // Fill dummy values
+    measurement->data["weight"]      = 85.00f;
+    measurement->data["dht.0.temp"]  = 42.42f;
+    measurement->data["dht.0.hum"]   = 84.84f;
+    measurement->data["ds18b20.0"]   = 33.33f;
+    measurement->data["voltage"]     = 3.843f;
+
+    // Display SQL INSERT statement.
+    invoke_insert(measurement);
+
+    // Free memory
+    delete measurement;
+
+}
+
+void basic_insert_missing() {
+
+    terrine.log("-- Test DML single reading (incomplete)");
+
+    // Data container to collect one reading
+    Measurement *measurement = new Measurement();
+
+    // Fill timestamp
+    getTimestamp(*measurement);
+
+    // Fill dummy values
+    measurement->data["dht.0.temp"]  = 42.42f;
+    measurement->data["dht.0.hum"]   = 84.84f;
+    measurement->data["voltage"]     = 3.843f;
+
+    // Display SQL INSERT statement.
+    invoke_insert(measurement);
+
+    // Free memory
+    delete measurement;
+
+}
+
+void invoke_create(Measurement *measurement) {
+
+    // Serialize data into SQL INSERT statement.
+    std::string data_record = datamgr->sql_create("testdrive", *measurement);
+
+    // Output
+    terrine.log("data:   ", false);
+    terrine.log(data_record.c_str());
+
+    terrine.log();
+}
+
+void invoke_insert(Measurement *measurement) {
+
+    // Serialize data into SQL INSERT statement.
+    std::string data_record = datamgr->sql_insert("testdrive", *measurement);
+
+    // Output
+    terrine.log("data:   ", false);
+    terrine.log(data_record.c_str());
+
+    terrine.log();
+}
+
+// Program entrypoint for glibc.
+int main() {
+
+    terrine.log("==============================");
+    terrine.log("TerkinData SQL DDL+DML example");
+    terrine.log("==============================");
+    terrine.log();
+
+    basic_create();
+    basic_insert_single();
+    basic_insert_missing();
+
+}
+
+// Program entrypoints for Arduino.
+void setup() {
+    main();
+}
+void loop() {
+}