From 058a1d704788f39f5fb4bc14436e102f22145423 Mon Sep 17 00:00:00 2001 From: Guy Khmelnitsky Date: Fri, 6 Dec 2024 10:25:47 +0200 Subject: [PATCH] fix: Fix bad KVA tarriff fetching - by a fallback --- custom_components/iec/coordinator.py | 201 ++++++++++++--------------- 1 file changed, 92 insertions(+), 109 deletions(-) diff --git a/custom_components/iec/coordinator.py b/custom_components/iec/coordinator.py index a051a27..8785c4d 100644 --- a/custom_components/iec/coordinator.py +++ b/custom_components/iec/coordinator.py @@ -3,7 +3,6 @@ import calendar import itertools import logging -import traceback import socket from datetime import datetime, timedelta, date from typing import cast, Any # noqa: UP035 @@ -87,7 +86,6 @@ def __init__( # Refresh every 1h to be at most 5h behind. update_interval=timedelta(hours=1), ) - _LOGGER.debug("Initializing IEC Coordinator") self._config_entry = config_entry self._bp_number = config_entry.data.get(CONF_BP_NUMBER) self._contract_ids = config_entry.data.get(CONF_SELECTED_CONTRACTS) @@ -120,10 +118,6 @@ def _dummy_listener() -> None: # _async_update_data not periodically getting called which is needed for _insert_statistics. self.async_add_listener(_dummy_listener) - async def async_unload(self): - """Unload the coordinator, cancel any pending tasks.""" - _LOGGER.info("Coordinator unloaded successfully.") - async def _get_devices_by_contract_id(self, contract_id) -> list[Device]: devices = self._devices_by_contract_id.get(contract_id) if not devices: @@ -194,6 +188,8 @@ async def _get_kva_tariff(self) -> float: try: self._kva_tariff = await self.api.get_kva_tariff() except IECError as e: + _LOGGER.exception("Failed fetching KVA Tariff from IEC", e) + except Exception as e: _LOGGER.exception("Failed fetching KVA Tariff", e) return self._kva_tariff or 0.0 @@ -297,20 +293,15 @@ async def _get_readings( async def _verify_daily_readings_exist( self, - daily_readings: dict[str, list[RemoteReading]], - desired_date: date, + daily_readings: list[RemoteReading], + desired_date: datetime, device: Device, contract_id: int, prefetched_reading: RemoteReadingResponse | None = None, ): - if not daily_readings.get(device.device_number): - daily_readings[device.device_number] = [] - + desired_date = desired_date.replace(hour=0, minute=0, second=0, microsecond=0) daily_reading = next( - filter( - lambda x: find_reading_by_date(x, desired_date), - daily_readings[device.device_number], - ), + filter(lambda x: find_reading_by_date(x, desired_date), daily_readings), None, ) if not daily_reading: @@ -323,7 +314,7 @@ async def _verify_daily_readings_exist( contract_id, device.device_number, device.device_code, - datetime.fromordinal(desired_date.toordinal()), + desired_date, ReadingResolution.MONTHLY, ) else: @@ -331,32 +322,21 @@ async def _verify_daily_readings_exist( f'Daily reading for date: {desired_date.strftime("%Y-%m-%d")} - using existing prefetched readings' ) - if readings and readings.data: - daily_readings[device.device_number] += readings.data - - # Remove duplicates - daily_readings[device.device_number] = list( - dict.fromkeys(daily_readings[device.device_number]) + desired_date_reading = next( + filter( + lambda reading: reading.date.date() == desired_date.date(), + readings.data, + ), + 0, + ) + if desired_date_reading == 0 or desired_date_reading.value <= 0: + _LOGGER.debug( + f'Couldn\'t find daily reading for: {desired_date.strftime("%Y-%m-%d")}' ) - - # Sort by Date - daily_readings[device.device_number].sort(key=lambda x: x.date) - - desired_date_reading = next( - filter( - lambda reading: reading.date.date() == desired_date, - readings.data, - ), - None, + else: + daily_readings.append( + RemoteReading(0, desired_date, desired_date_reading.value) ) - if desired_date_reading is None or desired_date_reading.value <= 0: - _LOGGER.debug( - f'Couldn\'t find daily reading for: {desired_date.strftime("%Y-%m-%d")}' - ) - else: - daily_readings[device.device_number].append( - RemoteReading(0, desired_date, desired_date_reading.value) - ) else: _LOGGER.debug( f'Daily reading for date: {daily_reading.date.strftime("%Y-%m-%d")}' @@ -384,7 +364,6 @@ async def _update_data( if c.status == 1 and int(c.contract_id) in self._contract_ids } localized_today = TIMEZONE.localize(datetime.now()) - localized_first_of_month = localized_today.replace(day=1) kwh_tariff = await self._get_kwh_tariff() kva_tariff = await self._get_kva_tariff() @@ -455,58 +434,63 @@ async def _update_data( for device in devices: attributes_to_add[METER_ID_ATTR_NAME] = device.device_number - reading_type: ReadingResolution | None = None - reading_date: date | None = None - - if localized_today.date() != localized_first_of_month.date(): - reading_type: ReadingResolution | None = ( - ReadingResolution.MONTHLY - ) - reading_date: date | None = localized_first_of_month - elif localized_today.date().isoweekday() != 7: - # If today's the 1st of the month, but not sunday, get weekly from yesterday - yesterday = localized_today - timedelta(days=1) - reading_type: ReadingResolution | None = ( - ReadingResolution.WEEKLY - ) - reading_date: date | None = yesterday - else: - # Today is the 1st and is Monday (since monday.isoweekday==1) - last_month_first_of_the_month = ( - localized_first_of_month - timedelta(days=1) - ).replace(day=1) - - reading_type: ReadingResolution | None = ( - ReadingResolution.MONTHLY - ) - reading_date: date | None = last_month_first_of_the_month - - _LOGGER.debug( - f"Fetching {reading_type.name} readings from {reading_date}" - ) remote_reading = await self._get_readings( contract_id, device.device_number, device.device_code, - reading_date, - reading_type, + localized_today, + ReadingResolution.MONTHLY, ) + if ( + remote_reading and remote_reading.total_import + ): # use total_import as validation that reading is OK + future_consumption[device.device_number] = ( + remote_reading.future_consumption_info + ) + if remote_reading and remote_reading.data: daily_readings[device.device_number] = remote_reading.data else: _LOGGER.warning( - "No %s readings returned for device %s in contract %s on %s", - reading_type.name, + "No Monthly readings returned for device %s in contract %s on %s", device.device_number, contract_id, - reading_date, + localized_today.strftime("%Y-%m-%d"), ) daily_readings[device.device_number] = [] - # Verify today's date appears + weekly_future_consumption = None + if localized_today.day == 1: + # if today's the 1st of the month, "yesterday" is on a different month + yesterday: datetime = localized_today - timedelta(days=1) + remote_reading = await self._get_readings( + contract_id, + device.device_number, + device.device_code, + yesterday, + ReadingResolution.WEEKLY, + ) + if ( + remote_reading and remote_reading.total_import + ): # use total_import as validation that reading OK + daily_readings[device.device_number] += remote_reading.data + weekly_future_consumption = ( + remote_reading.future_consumption_info + ) + + # Remove duplicates + daily_readings[device.device_number] = list( + dict.fromkeys(daily_readings[device.device_number]) + ) + + # Sort by Date + daily_readings[device.device_number].sort( + key=lambda x: x.date + ) + await self._verify_daily_readings_exist( - daily_readings, - localized_today.date(), + daily_readings[device.device_number], + localized_today - timedelta(days=1), device, contract_id, ) @@ -526,12 +510,19 @@ async def _update_data( # fallbacks for future consumption since IEC api is broken :/ if ( - not future_consumption.get(device.device_number) + not future_consumption[device.device_number] or not future_consumption[ device.device_number ].future_consumption ): if ( + weekly_future_consumption + and weekly_future_consumption.future_consumption + ): + future_consumption[device.device_number] = ( + weekly_future_consumption + ) + elif ( self._today_readings.get(today_reading_key) and self._today_readings.get( today_reading_key @@ -560,7 +551,7 @@ async def _update_data( two_days_ago_reading.future_consumption_info ) else: - _LOGGER.warning( + _LOGGER.debug( "Failed fetching FutureConsumption, data in IEC API is corrupted" ) @@ -650,29 +641,23 @@ async def _async_update_data( try: return await self._update_data() except Exception as err: - _LOGGER.error("Failed updating data. Exception: %s", err) - _LOGGER.error(traceback.format_exc()) raise UpdateFailed("Failed Updating IEC data") from err async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> None: if not is_smart_meter: _LOGGER.info( - f"[IEC Statistics] IEC Contract {contract_id} doesn't contain Smart Meters, not adding statistics" + f"IEC Contract {contract_id} doesn't contain Smart Meters, not adding statistics" ) # Support only smart meters at the moment return - _LOGGER.debug( - f"[IEC Statistics] Updating statistics for IEC Contract {contract_id}" - ) + _LOGGER.debug(f"Updating statistics for IEC Contract {contract_id}") devices = await self._get_devices_by_contract_id(contract_id) kwh_price = await self._get_kwh_tariff() localized_today = TIMEZONE.localize(datetime.now()) if not devices: - _LOGGER.error( - f"[IEC Statistics] Failed fetching devices for IEC Contract {contract_id}" - ) + _LOGGER.error(f"Failed fetching devices for IEC Contract {contract_id}") return for device in devices: @@ -686,7 +671,7 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No if not last_stat: _LOGGER.debug( - "[IEC Statistics] No statistics found, fetching today's MONTHLY readings to extract field `meterStartDate`" + "No statistics found, fetching today's MONTHLY readings to extract field `meterStartDate`" ) month_ago_time = localized_today - timedelta(weeks=4) @@ -703,12 +688,12 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No month_ago_time = max(month_ago_time, readings.meter_start_date) else: _LOGGER.debug( - "[IEC Statistics] Failed to extract field `meterStartDate`, falling back to a month ago" + "Failed to extract field `meterStartDate`, falling back to a month ago" ) - _LOGGER.debug("[IEC Statistics] Updating statistic for the first time") + _LOGGER.debug("Updating statistic for the first time") _LOGGER.debug( - f"[IEC Statistics] Fetching consumption from {month_ago_time.strftime('%Y-%m-%d %H:%M:%S')}" + f"Fetching consumption from {month_ago_time.strftime('%Y-%m-%d %H:%M:%S')}" ) last_stat_time = 0 readings = await self._get_readings( @@ -724,7 +709,7 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No # API returns daily data, so need to increase the start date by 4 hrs to get the next day from_date = datetime.fromtimestamp(last_stat_time) _LOGGER.debug( - f"[IEC Statistics] Last statistics are from {from_date.strftime('%Y-%m-%d %H:%M:%S')}" + f"Last statistics are from {from_date.strftime('%Y-%m-%d %H:%M:%S')}" ) if from_date.hour == 23: @@ -732,14 +717,14 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No if localized_today.date() == from_date.date(): _LOGGER.debug( - "[IEC Statistics] The date to fetch is today or later, replacing it with Today at 01:00:00" + "The date to fetch is today or later, replacing it with Today at 01:00:00" ) from_date = localized_today.replace( hour=1, minute=0, second=0, microsecond=0 ) _LOGGER.debug( - f"[IEC Statistics] Fetching consumption from {from_date.strftime('%Y-%m-%d %H:%M:%S')}" + f"Fetching consumption from {from_date.strftime('%Y-%m-%d %H:%M:%S')}" ) readings = await self._get_readings( contract_id, @@ -754,7 +739,7 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No ] = readings if not readings or not readings.data: - _LOGGER.debug("[IEC Statistics] No recent usage data. Skipping update") + _LOGGER.debug("No recent usage data. Skipping update") continue last_stat_hour = ( @@ -768,9 +753,7 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No else (last_stat_hour - timedelta(hours=1)) ) - _LOGGER.debug( - f"[IEC Statistics] Fetching LongTerm Statistics since {last_stat_req_hour}" - ) + _LOGGER.debug(f"Fetching LongTerm Statistics since {last_stat_req_hour}") stats = await get_instance(self.hass).async_add_executor_job( statistics_during_period, self.hass, @@ -783,14 +766,14 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No ) if not stats.get(consumption_statistic_id): - _LOGGER.debug("[IEC Statistics] No recent usage data") + _LOGGER.debug("No recent usage data") consumption_sum = 0 else: consumption_sum = cast(float, stats[consumption_statistic_id][0]["sum"]) if not stats.get(cost_statistic_id): if not stats.get(consumption_statistic_id): - _LOGGER.debug("[IEC Statistics] No recent cost data") + _LOGGER.debug("No recent cost data") cost_sum = 0.0 else: cost_sum = ( @@ -801,10 +784,10 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No cost_sum = cast(float, stats[cost_statistic_id][0]["sum"]) _LOGGER.debug( - f"[IEC Statistics] Last Consumption Sum for C[{contract_id}] D[{device.device_number}]: {consumption_sum}" + f"Last Consumption Sum for C[{contract_id}] D[{device.device_number}]: {consumption_sum}" ) _LOGGER.debug( - f"[IEC Statistics] Last Estimated Cost Sum for C[{contract_id}] D[{device.device_number}]: {cost_sum}" + f"Last Estimated Cost Sum for C[{contract_id}] D[{device.device_number}]: {cost_sum}" ) new_readings: list[RemoteReading] = filter( @@ -827,12 +810,12 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No group_list = list(group) if len(group_list) < 4: _LOGGER.debug( - f"[IEC Statistics] LongTerm Statistics - Skipping {key} since it's partial for the hour" + f"LongTerm Statistics - Skipping {key} since it's partial for the hour" ) continue if key <= last_stat_req_hour: _LOGGER.debug( - f"[IEC Statistics] LongTerm Statistics - Skipping {key} data since it's already reported" + f"LongTerm Statistics - Skipping {key} data since it's already reported" ) continue readings_by_hour[key] = sum(reading.value for reading in group_list) @@ -871,14 +854,14 @@ async def _insert_statistics(self, contract_id: int, is_smart_meter: bool) -> No if readings_by_hour: _LOGGER.debug( - f"[IEC Statistics] Last hour fetched for C[{contract_id}] D[{device.device_number}]: " + f"Last hour fetched for C[{contract_id}] D[{device.device_number}]: " f"{max(readings_by_hour, key=lambda k: k)}" ) _LOGGER.debug( - f"[IEC Statistics] New Consumption Sum for C[{contract_id}] D[{device.device_number}]: {consumption_sum}" + f"New Consumption Sum for C[{contract_id}] D[{device.device_number}]: {consumption_sum}" ) _LOGGER.debug( - f"[IEC Statistics] New Estimated Cost Sum for C[{contract_id}] D[{device.device_number}]: {cost_sum}" + f"New Estimated Cost Sum for C[{contract_id}] D[{device.device_number}]: {cost_sum}" ) async_add_external_statistics(