feature: interpret load sensors in kW rather than W

config_emhass.yaml

@@ -4,7 +4,9 @@ retrieve_hass_conf:
   freq: 30 # The time step to resample retrieved data from hass in minutes
   days_to_retrieve: 2 # We will retrieve data from now and up to days_to_retrieve days
   var_PV: 'sensor.power_photovoltaics' # Photovoltaic produced power sensor in Watts
+  var_PV_in_kw: False # Photovoltaic sensor in kW rather than W
   var_load: 'sensor.power_load_no_var_loads' # Household power consumption sensor in Watts (deferrable loads should be substracted)
+  var_load_in_kw: False # load_no_var_loads in kW rather than W
   load_negative: False # Set to True if the retrived load variable is negative by convention
   set_zero_min: True # A special treatment for a minimum value saturation to zero. Values below zero are replaced by nans
   var_replace_zero: # A list of retrived variables that we would want  to replace nans with zeros

docs/config.md

@@ -15,7 +15,9 @@ These are the parameters that we will need to define to retrieve data from Home
 - `freq`: The time step to resample retrieved data from hass. This parameter is given in minutes. It should not be defined too low or you will run into memory problems when defining the Linear Programming optimization. Defaults to 30. 
 - `days_to_retrieve`: We will retrieve data from now and up to days_to_retrieve days. Defaults to 2.
 - `var_PV`: This is the name of the photovoltaic produced power sensor in Watts from Home Assistant. For example: 'sensor.power_photovoltaics'.
+- `var_PV_in_kw`: Set this parameter to True if the PV sensor is in kW rather than W. Defaults to False.
 - `var_load`: The name of the household power consumption sensor in Watts from Home Assistant. The deferrable loads that we will want to include in the optimization problem should be substracted from this sensor in HASS. For example: 'sensor.power_load_no_var_loads'
+- `var_load_in_kw`: Set this parameter to True if the load sensor is in kW rather than W. Defaults to False.
 - `load_negative`: Set this parameter to True if the retrived load variable is negative by convention. Defaults to False.
 - `set_zero_min`: Set this parameter to True to give a special treatment for a minimum value saturation to zero for power consumption data. Values below zero are replaced by nans. Defaults to True.
 - `var_replace_zero`: The list of retrieved variables that we would want to replace nans (if they exist) with zeros. For example:

docs/differences.md

@@ -20,6 +20,8 @@ See bellow for a list of associations between the parameters from `config_emhass
 | retrieve_hass_conf |  var_PV |  sensor_power_photovoltaics | |
 | retrieve_hass_conf |  var_load |  sensor_power_load_no_var_loads | |
 | retrieve_hass_conf |  load_negative |  load_negative | |
+| retrieve_hass_conf |  var_PV_in_kw |  var_PV_in_kw | |
+| retrieve_hass_conf |  var_load_in_kw |  var_load_in_kw | |
 | retrieve_hass_conf |  set_zero_min |  set_zero_min | |
 | retrieve_hass_conf |  method_ts_round |  method_ts_round | |
 | params_secrets |  solcast_api_key |  optional_solcast_api_key | |

docs/mlforecaster.md

@@ -26,6 +26,8 @@ The minimum number of `days_to_retrieve` is hard coded to 9 by default. But it i
 
 - `var_model`: the name of the sensor to retrieve data from Home Assistant. Example: `sensor.power_load_no_var_loads`.
 
+- `var_model_in_kw`: whether the sensor is in kW. Example: `False`.
+
 - `sklearn_model`: the `scikit-learn` model that will be used. For now only this options are possible: `LinearRegression`, `ElasticNet` and `KNeighborsRegressor`.
 
 - `num_lags`: the number of auto-regression lags to consider. A good starting point is to fix this as one day. For example if your time step is 30 minutes, then fix this to 48, if the time step is 1 hour the fix this to 24 and so on.

options.json

@@ -22,6 +22,8 @@
   "weight_battery_charge": 1.0,
   "sensor_power_photovoltaics": "sensor.power_photovoltaics",
   "sensor_power_load_no_var_loads": "sensor.power_load_no_var_loads",
+  "var_PV_in_kw": false,
+  "var_load_in_kw": false,
   "load_negative": false,
   "set_zero_min": true,
   "number_of_deferrable_loads": 2,

scripts/load_clustering.py

@@ -47,6 +47,7 @@
     days_to_retrieve = 240
     model_type = "load_clustering"
     var_model = "sensor.power_load_positive"
+    var_model_in_kw = False
 
     data_path = emhass_conf['data_path'] / str('data_train_'+model_type+'.pkl')
     params = None
@@ -65,7 +66,8 @@
 
         days_list = get_days_list(days_to_retrieve)
         var_list = [var_model]
-        rh.get_data(days_list, var_list)
+        sensors_in_kw = [var_model_in_kw]
+        rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw)
 
         with open(data_path, 'wb') as fid:
             pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL)
@@ -144,4 +146,4 @@
     # data_lag['cluster_group_tslearn_sdtw'] = y_pred
 
     # fig = px.scatter(data_lag, x='power_load y(t)', y='power_load y(t+1)', color='cluster_group_tslearn_sdtw', template=template)
-    # fig.show()
+    # fig.show()

scripts/load_forecast_sklearn.py

@@ -54,6 +54,7 @@ def neg_r2_score(y_true, y_pred):
     days_to_retrieve = 240
     model_type = "load_forecast"
     var_model = "sensor.power_load_no_var_loads"
+    var_model_in_kw = False
     sklearn_model = "KNeighborsRegressor"
     num_lags = 48
 
@@ -74,7 +75,8 @@ def neg_r2_score(y_true, y_pred):
 
         days_list = get_days_list(days_to_retrieve)
         var_list = [var_model]
-        rh.get_data(days_list, var_list)
+        sensors_in_kw = [var_model_in_kw]
+        rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw)
 
         with open(data_path, 'wb') as fid:
             pickle.dump((rh.df_final, var_model), fid, pickle.HIGHEST_PROTOCOL)
@@ -257,7 +259,8 @@ def neg_r2_score(y_true, y_pred):
     days_list = get_days_list(days_needed)
     var_model = retrieve_hass_conf['var_load']
     var_list = [var_model]
-    rh.get_data(days_list, var_list)
+    sensor_in_kw_list = [retrieve_hass_conf['var_load_in_kw']]
+    rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list)
     data_last_window = copy.deepcopy(rh.df_final)
 
     data_last_window = add_date_features(data_last_window)
@@ -275,4 +278,4 @@ def neg_r2_score(y_true, y_pred):
     fig.update_yaxes(title_text = "Power (W)")
     fig.update_xaxes(title_text = "Time")
     fig.show()
-    fig.write_image(emhass_conf['root_path'] / "docs/images/load_forecast_production.svg", width=1080, height=0.8*1080)
+    fig.write_image(emhass_conf['root_path'] / "docs/images/load_forecast_production.svg", width=1080, height=0.8*1080)

scripts/optim_results_analysis.py

@@ -130,4 +130,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf,
 
     print(opt_res_dah)
     if save_html:
-        opt_res_dah.to_html('opt_res_dah.html')
+        opt_res_dah.to_html('opt_res_dah.html')

scripts/special_config_analysis.py

@@ -185,4 +185,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf,
     fig_res_mpc.layout.template = template
     fig_res_mpc.update_yaxes(title_text = "Powers (W)")
     fig_res_mpc.update_xaxes(title_text = "Time")
-    fig_res_mpc.show()
+    fig_res_mpc.show()

scripts/use_cases_analysis.py

@@ -55,8 +55,9 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf,
                       params, emhass_conf, logger)
     days_list = get_days_list(retrieve_hass_conf['days_to_retrieve'])
     var_list = [retrieve_hass_conf['var_load'], retrieve_hass_conf['var_PV']]
+    sensors_in_kw = [retrieve_hass_conf['var_load_in_kw'], retrieve_hass_conf['var_PV_in_kw']]
     rh.get_data(days_list, var_list,
-                    minimal_response=False, significant_changes_only=False)
+                    minimal_response=False, significant_changes_only=False,sensor_in_kw_list=sensors_in_kw)
     rh.prepare_data(retrieve_hass_conf['var_load'], load_negative = retrieve_hass_conf['load_negative'],
                             set_zero_min = retrieve_hass_conf['set_zero_min'], 
                             var_replace_zero = retrieve_hass_conf['var_replace_zero'], 
@@ -177,4 +178,4 @@ def get_forecast_optim_objects(retrieve_hass_conf, optim_conf, plant_conf,
                                 width=1080, height=0.8*1080)
 
     print("System with: PV, Battery, two deferrable loads, dayahead optimization, profit >> total cost function sum: "+\
-        str(opt_res_dah['cost_profit'].sum()))
+        str(opt_res_dah['cost_profit'].sum()))

src/emhass/command_line.py

@@ -81,7 +81,9 @@ def set_input_data_dict(emhass_conf: dict, costfun: str,
                 retrieve_hass_conf["days_to_retrieve"])
             var_list = [retrieve_hass_conf["var_load"],
                         retrieve_hass_conf["var_PV"]]
-            if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False):
+            sensors_in_kw = [retrieve_hass_conf["var_load_in_kw"],
+                                 retrieve_hass_conf["var_PV_in_kw"]]
+            if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw):
                 return False
         if not rh.prepare_data(retrieve_hass_conf["var_load"],
                                load_negative=retrieve_hass_conf["load_negative"],
@@ -129,7 +131,9 @@ def set_input_data_dict(emhass_conf: dict, costfun: str,
             days_list = utils.get_days_list(1)
             var_list = [retrieve_hass_conf["var_load"],
                         retrieve_hass_conf["var_PV"]]
-            if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False):
+            sensors_in_kw = [retrieve_hass_conf["var_load_in_kw"],
+                                 retrieve_hass_conf["var_PV_in_kw"]]
+            if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw):
                 return False
         if not rh.prepare_data(retrieve_hass_conf["var_load"],
                                load_negative=retrieve_hass_conf["load_negative"],
@@ -165,6 +169,7 @@ def set_input_data_dict(emhass_conf: dict, costfun: str,
         days_to_retrieve = params["passed_data"]["days_to_retrieve"]
         model_type = params["passed_data"]["model_type"]
         var_model = params["passed_data"]["var_model"]
+        var_model_in_kw = params["passed_data"]["var_model_in_kw"]
         if get_data_from_file:
             days_list = None
             filename = 'data_train_'+model_type+'.pkl'
@@ -175,7 +180,8 @@ def set_input_data_dict(emhass_conf: dict, costfun: str,
         else:
             days_list = utils.get_days_list(days_to_retrieve)
             var_list = [var_model]
-            if not rh.get_data(days_list, var_list):
+            sensors_in_kw = [var_model_in_kw]
+            if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw):
                 return False
             df_input_data = rh.df_final.copy()
     elif set_type == "regressor-model-fit" or set_type == "regressor-model-predict":
@@ -404,6 +410,7 @@ def forecast_model_fit(input_data_dict: dict, logger: logging.Logger,
     data = copy.deepcopy(input_data_dict['df_input_data'])
     model_type = input_data_dict['params']['passed_data']['model_type']
     var_model = input_data_dict['params']['passed_data']['var_model']
+    var_model_in_kw = input_data_dict['params']['passed_data']['var_model_in_kw']
     sklearn_model = input_data_dict['params']['passed_data']['sklearn_model']
     num_lags = input_data_dict['params']['passed_data']['num_lags']
     split_date_delta = input_data_dict['params']['passed_data']['split_date_delta']

src/emhass/forecast.py

@@ -643,7 +643,8 @@ def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: O
                     self.var_load_new = self.var_load+'_positive'
             else:
                 days_list = get_days_list(days_min_load_forecast) 
-                if not rh.get_data(days_list, var_list):
+                sensors_in_kw = [self.retrieve_hass_conf['var_load_in_kw']]
+                if not rh.get_data(days_list, var_list, minimal_response=False, significant_changes_only=False, sensor_in_kw_list=sensors_in_kw):
                     return False
             if  not rh.prepare_data(
                 self.retrieve_hass_conf['var_load'], load_negative = self.retrieve_hass_conf['load_negative'],

src/emhass/retrieve_hass.py

@@ -67,7 +67,7 @@ def __init__(self, hass_url: str, long_lived_token: str, freq: pd.Timedelta,
 
     def get_data(self, days_list: pd.date_range, var_list: list, 
                  minimal_response: Optional[bool] = False, significant_changes_only: Optional[bool] = False,
-                 test_url: Optional[str] = "empty") -> None:
+                 test_url: Optional[str] = "empty", sensor_in_kw_list: Optional[list] = []) -> None:
         r"""
         Retrieve the actual data from hass.
 
@@ -84,6 +84,9 @@ def get_data(self, days_list: pd.date_range, var_list: list,
         :param significant_changes_only: Retrieve significant changes only \
             using the hass restful API, defaults to False
         :type significant_changes_only: bool, optional
+        :param sensor_in_kw_list: Array of flags that determine whether the \
+                var_list elements are in kW or W. False if undefined
+        :type sensor_in_kw_list: list, optional
         :return: The DataFrame populated with the retrieved data from hass
         :rtype: pandas.DataFrame
 
@@ -178,16 +181,37 @@ def get_data(self, days_list: pd.date_range, var_list: list,
                 if i == 0: # Defining the DataFrame container
                     from_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").min()
                     to_date = pd.to_datetime(df_raw['last_changed'], format="ISO8601").max()
-                    ts = pd.to_datetime(pd.date_range(start=from_date, end=to_date, freq=self.freq), 
+                    ts = pd.to_datetime(pd.date_range(start=from_date, end=to_date, freq=self.freq),
                                         format='%Y-%d-%m %H:%M').round(self.freq, ambiguous='infer', nonexistent='shift_forward')
                     df_day = pd.DataFrame(index = ts)
-                # Caution with undefined string data: unknown, unavailable, etc.
-                df_tp = (
-                    df_raw.copy()[["state"]]
-                    .replace(["unknown", "unavailable", ""], np.nan)
-                    .astype(float)
-                    .rename(columns={"state": var})
-                )
+                # if load sensors are in kW, multiply the state entry by 1000
+                if sensor_in_kw_list:
+                    if sensor_in_kw_list[i]:
+                        # Caution with undefined string data: unknown, unavailable, etc.
+                        df_tp = (
+                            df_raw.copy()[["state"]]
+                            .replace(["unknown", "unavailable", ""], np.nan)
+                            .astype(float)
+                            .rename(columns={"state": var})
+                            .mul(1000)
+                        )
+                    else:
+                        # Caution with undefined string data: unknown, unavailable, etc.
+                        df_tp = (
+                            df_raw.copy()[["state"]]
+                            .replace(["unknown", "unavailable", ""], np.nan)
+                            .astype(float)
+                            .rename(columns={"state": var})
+                        )
+                else:
+                    # Caution with undefined string data: unknown, unavailable, etc.
+                    df_tp = (
+                        df_raw.copy()[["state"]]
+                        .replace(["unknown", "unavailable", ""], np.nan)
+                        .astype(float)
+                        .rename(columns={"state": var})
+                    )
+
                 # Setting index, resampling and concatenation
                 df_tp.set_index(
                     pd.to_datetime(df_raw["last_changed"], format="ISO8601"),

src/emhass/utils.py

@@ -325,11 +325,17 @@
         else:
             model_type = runtimeparams["model_type"]
         params["passed_data"]["model_type"] = model_type
+
         if "var_model" not in runtimeparams.keys():
             var_model = "sensor.power_load_no_var_loads"
         else:
             var_model = runtimeparams["var_model"]
         params["passed_data"]["var_model"] = var_model
+        if "var_model_in_kw" not in runtimeparams.keys():
+            var_model_in_kw = False
+        else:
+            var_model_in_kw = eval(str(runtimeparams["var_model_in_kw"]).capitalize())
+        params["passed_data"]["var_model_in_kw"] = var_model_in_kw
         if "sklearn_model" not in runtimeparams.keys():
             sklearn_model = "KNeighborsRegressor"
         else:
@@ -673,7 +679,7 @@
         "<h4>Plotting train/test forecast model results for " + mlf.model_type + "</h4>"
     )
     injection_dict["subsubtitle0"] = (
-        "<h4>Forecasting variable " + mlf.var_model + "</h4>"
+            "<h4>Forecasting variable " + mlf.var_model + " (sensor in kW: " + mlf.var_model_in_kw + ")</h4>"
     )
     injection_dict["figure_0"] = image_path_0
     return injection_dict
@@ -743,6 +749,12 @@
         params["retrieve_hass_conf"]["var_load"] = options.get(
             "sensor_power_load_no_var_loads", params["retrieve_hass_conf"]["var_load"]
         )
+        params["retrieve_hass_conf"]["var_PV_in_kw"] = options.get(
+            "var_PV_in_kw", params["retrieve_hass_conf"]["var_PV_in_kw"]
+        )
+        params["retrieve_hass_conf"]["var_load_in_kw"] = options.get(
+            "var_load_in_kw", params["retrieve_hass_conf"]["var_load_in_kw"]
+        )
         params["retrieve_hass_conf"]["load_negative"] = options.get(
             "load_negative", params["retrieve_hass_conf"]["load_negative"]
         )

tests/test_command_line_utils.py

@@ -279,6 +279,7 @@ def test_forecast_model_fit_predict_tune(self):
             "days_to_retrieve": 20,
             "model_type": "load_forecast",
             "var_model": "sensor.power_load_no_var_loads",
+            "var_model_in_kw": False,
             "sklearn_model": "KNeighborsRegressor",
             "num_lags": 48,
             "split_date_delta": '48h',
@@ -457,6 +458,7 @@ def test_main_forecast_model_fit(self):
             "days_to_retrieve": 20,
             "model_type": "load_forecast",
             "var_model": "sensor.power_load_no_var_loads",
+            "var_model_in_kw": False,
             "sklearn_model": "KNeighborsRegressor",
             "num_lags": 48,
             "split_date_delta": '48h',
@@ -479,6 +481,7 @@ def test_main_forecast_model_predict(self):
             "days_to_retrieve": 20,
             "model_type": "load_forecast",
             "var_model": "sensor.power_load_no_var_loads",
+            "var_model_in_kw": False,
             "sklearn_model": "KNeighborsRegressor",
             "num_lags": 48,
             "split_date_delta": "48h",
@@ -501,6 +504,7 @@ def test_main_forecast_model_tune(self):
             "days_to_retrieve": 20,
             "model_type": "load_forecast",
             "var_model": "sensor.power_load_no_var_loads",
+            "var_model_in_kw": False,
             "sklearn_model": "KNeighborsRegressor",
             "num_lags": 48,
             "split_date_delta": "48h",