diff --git a/README.md b/README.md index 73b0e95..08f44c5 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ Features: * Configurable update interval * Configurable traffic unit (bps, Kbps, Mbps, B/s, KB/s, MB/s) * Supports monitoring of multiple mikrotik devices simultaneously - + * RX/TX WAN/LAN traffic sensors per hosts from Mikrotik Accounting feature + # Integration preview ![Tracker and sensors](https://raw.githubusercontent.com/tomaae/homeassistant-mikrotik_router/master/docs/assets/images/ui/device_tracker.png) @@ -39,6 +40,7 @@ Features: ![Queue switch](https://raw.githubusercontent.com/tomaae/homeassistant-mikrotik_router/master/docs/assets/images/ui/queue_switch.png) ![Host tracker](https://raw.githubusercontent.com/tomaae/homeassistant-mikrotik_router/master/docs/assets/images/ui/host_tracker.png) +![Accounting sensor](https://raw.githubusercontent.com/tomaae/homeassistant-mikrotik_router/master/docs/assets/images/ui/accounting_sensor.png) # Setup integration Setup this integration for your Mikrotik device in Home Assistant via `Configuration -> Integrations -> Add -> Mikrotik Router`. @@ -58,3 +60,18 @@ You can add this integration several times for different devices. ## List of detected devices ![Integration options](https://raw.githubusercontent.com/tomaae/homeassistant-mikrotik_router/master/docs/assets/images/ui/integration_devices.png) + +## Accounting +For per-IP throughput tracking Mikrotik's accounting feature is used. + +[Mikrotik support page](https://wiki.mikrotik.com/wiki/Manual:IP/Accounting) + +Feature will be automaticaly used if accounting is enabled in Mikrotik. Feature is present in Winbox IP-Accounting. Make sure that threshold is set to resonable value to store all connections between user defined scan interval. Max value is 8192 so for piece of mind i recommend setting that value. Web Access is not needed, integration is using API access. + +Integration will scan DHCP Lease table and ARP table to generate all known hosts and create two sensors for WAN traffic (mikrotik-XXX-wan-rx and mikrotik-XXX-wan-tx). If the parameter *account-local-traffic* is set in Mikrotik's accounting configuration it will also create two sensors for LAN traffic (mikrotik-XXX-lan-rx and mikrotik-XXX-lan-tx). + +Device's name will be determined by first available string this order: +1. DHCP lease comment +2. DNS static entry +3. DHCP hostname +4. Device's MAC address diff --git a/custom_components/mikrotik_router/__init__.py b/custom_components/mikrotik_router/__init__.py index 61d051b..b84d19d 100644 --- a/custom_components/mikrotik_router/__init__.py +++ b/custom_components/mikrotik_router/__init__.py @@ -54,6 +54,7 @@ async def async_setup_entry(hass, config_entry): traffic_type ) await mikrotik_controller.hwinfo_update() + await mikrotik_controller.async_update() if not mikrotik_controller.data: diff --git a/custom_components/mikrotik_router/config_flow.py b/custom_components/mikrotik_router/config_flow.py index 3e9b3be..ec574aa 100644 --- a/custom_components/mikrotik_router/config_flow.py +++ b/custom_components/mikrotik_router/config_flow.py @@ -81,7 +81,7 @@ class MikrotikControllerConfigFlow(ConfigFlow, domain=DOMAIN): username=user_input["username"], password=user_input["password"], port=user_input["port"], - use_ssl=user_input["ssl"], + use_ssl=user_input["ssl"] ) if not api.connect(): errors[CONF_HOST] = api.error diff --git a/custom_components/mikrotik_router/mikrotik_controller.py b/custom_components/mikrotik_router/mikrotik_controller.py index 25dc09f..ef7678d 100644 --- a/custom_components/mikrotik_router/mikrotik_controller.py +++ b/custom_components/mikrotik_router/mikrotik_controller.py @@ -66,13 +66,17 @@ class MikrotikControllerData: "dhcp-network": {}, "dhcp": {}, "host": {}, + "accounting": {} } + self.local_dhcp_networks = [] + self.listeners = [] self.lock = asyncio.Lock() self.api = MikrotikAPI(host, username, password, port, use_ssl) + self.raw_arp_entries = [] self.nat_removed = {} async_track_time_interval( @@ -205,6 +209,8 @@ class MikrotikControllerData: await self.hass.async_add_executor_job(self.get_system_resource) await self.hass.async_add_executor_job(self.get_script) await self.hass.async_add_executor_job(self.get_queue) + await self.hass.async_add_executor_job(self.get_dhcp) + await self.hass.async_add_executor_job(self.get_accounting) async_dispatcher_send(self.hass, self.signal_update) @@ -348,6 +354,7 @@ class MikrotikControllerData: def update_arp(self, mac2ip, bridge_used): """Get list of hosts in ARP for interface client data from Mikrotik""" data = self.api.path("/ip/arp") + self.raw_arp_entries = data if not data: return mac2ip, bridge_used @@ -561,7 +568,7 @@ class MikrotikControllerData: self.data["resource"]["hdd-usage"] = "unknown" # --------------------------- - # get_system_routerboard + # get_firmware_update # --------------------------- def get_firmware_update(self): """Check for firmware update on Mikrotik""" @@ -736,6 +743,20 @@ class MikrotikControllerData: ] ) + # Build list of local DHCP networks + dhcp_networks = parse_api( + data={}, + source=self.api.path("/ip/dhcp-server/network"), + key="address", + vals=[ + {"name": "address"}, + ], + ensure_vals=[ + {"name": "address"}, + ] + ) + self.local_dhcp_networks = [IPv4Network(network) for network in dhcp_networks] + self.data["dhcp"] = parse_api( data=self.data["dhcp"], source=self.api.path("/ip/dhcp-server/lease"), @@ -747,7 +768,7 @@ class MikrotikControllerData: {"name": "status", "default": "unknown"}, {"name": "last-seen", "default": "unknown"}, {"name": "server", "default": "unknown"}, - {"name": "comment", "default": ""}, + {"name": "comment"}, ], ensure_vals=[ {"name": "interface"}, @@ -827,3 +848,178 @@ class MikrotikControllerData: # Update last seen if self.data["host"][uid]["available"]: self.data["host"][uid]["last-seen"] = utcnow() + self.data["dhcp"][uid]['interface'] = \ + self.data["dhcp-server"][self.data["dhcp"][uid]['server']]["interface"] + + self.data["dhcp"][uid]['available'] = \ + self.api.arp_ping(self.data["dhcp"][uid]['address'], self.data["dhcp"][uid]['interface']) + + def _address_part_of_local_network(self, address): + address = ip_address(address) + for network in self.local_dhcp_networks: + if address in network: + return True + return False + + def _get_accounting_mac_by_ip(self, requested_ip): + for mac, vals in self.data['accounting'].items(): + if vals.get('address') is requested_ip: + return mac + return None + + def get_accounting(self): + """Get Accounting data from Mikrotik""" + # Check if accounting and account-local-traffic is enabled + accounting_enabled, local_traffic_enabled = self.api.is_accounting_and_local_traffic_enabled() + traffic_type, traffic_div = self._get_traffic_type_and_div() + + if not accounting_enabled: + # If any hosts were created return counters to 0 so sensors wont get stuck on last value + for mac, vals in self.data["accounting"].items(): + self.data['accounting'][mac]["tx-rx-attr"] = traffic_type + if 'wan-tx' in vals: + self.data["accounting"][mac]['wan-tx'] = 0.0 + if 'wan-rx' in vals: + self.data["accounting"][mac]['wan-rx'] = 0.0 + if 'lan-tx' in vals: + self.data["accounting"][mac]['lan-tx'] = 0.0 + if 'lan-rx' in vals: + self.data["accounting"][mac]['lan-rx'] = 0.0 + return + + # Build missing hosts from already retrieved DHCP Server leases + for mac, vals in self.data["dhcp"].items(): + if mac not in self.data["accounting"]: + self.data["accounting"][mac] = { + 'address': vals['address'], + 'mac-address': vals['mac-address'], + 'host-name': vals['host-name'], + 'comment': vals['comment'] + } + + # Build missing hosts from already retrieved ARP list + host_update_from_arp = False + for entry in self.raw_arp_entries: + if entry['mac-address'] not in self.data["accounting"]: + self.data["accounting"][entry['mac-address']] = { + 'address': entry['address'], + 'mac-address': entry['mac-address'], + 'host-name': '', + 'comment': '' + } + host_update_from_arp = True + + # If some host was added from ARP table build new host-name for it from static DNS entry. Fallback to MAC + if host_update_from_arp: + dns_data = parse_api( + data={}, + source=self.api.path("/ip/dns/static"), + key="address", + vals=[ + {"name": "address"}, + {"name": "name"}, + ], + ) + + # Try to build hostname from DNS static entry + for mac, vals in self.data["accounting"].items(): + if not str(vals.get('host-name', '')).strip() or vals['host-name'] is 'unknown': + if vals['address'] in dns_data and str(dns_data[vals['address']].get('name', '')).strip(): + self.data["accounting"][mac]['host-name'] = dns_data[vals['address']]['name'] + + # Check if any host still have empty 'host-name'. Default it to MAC. + # Same check for 'comment' (pretty name) + for mac, vals in self.data["accounting"].items(): + if not str(vals.get('host-name', '')).strip() or vals['host-name'] is 'unknown': + self.data["accounting"][mac]['host-name'] = mac + if not str(vals.get('comment', '')).strip() or vals['host-name'] is 'comment': + self.data["accounting"][mac]['comment'] = mac + + _LOGGER.debug(f"Working with {len(self.data['accounting'])} accounting devices") + + # Build temp accounting values dict with ip address as key + # Also set traffic type for each item + tmp_accounting_values = {} + for mac, vals in self.data['accounting'].items(): + tmp_accounting_values[vals['address']] = { + "wan-tx": 0, + "wan-rx": 0, + "lan-tx": 0, + "lan-rx": 0 + } + self.data['accounting'][mac]["tx-rx-attr"] = traffic_type + + time_diff = self.api.take_accounting_snapshot() + if time_diff: + accounting_data = parse_api( + data={}, + source=self.api.path("/ip/accounting/snapshot"), + key=".id", + vals=[ + {"name": ".id"}, + {"name": "src-address"}, + {"name": "dst-address"}, + {"name": "bytes", "default": 0}, + ], + ) + + for item in accounting_data.values(): + source_ip = str(item.get('src-address')).strip() + destination_ip = str(item.get('dst-address')).strip() + bits_count = int(str(item.get('bytes')).strip()) * 8 + + if self._address_part_of_local_network(source_ip) and self._address_part_of_local_network(destination_ip): + # LAN TX/RX + if source_ip in tmp_accounting_values: + tmp_accounting_values[source_ip]['lan-tx'] += bits_count + if destination_ip in tmp_accounting_values: + tmp_accounting_values[destination_ip]['lan-rx'] += bits_count + elif self._address_part_of_local_network(source_ip) and \ + not self._address_part_of_local_network(destination_ip): + # WAN TX + if source_ip in tmp_accounting_values: + tmp_accounting_values[source_ip]['wan-tx'] += bits_count + elif not self._address_part_of_local_network(source_ip) and \ + self._address_part_of_local_network(destination_ip): + # WAN RX + if destination_ip in tmp_accounting_values: + tmp_accounting_values[destination_ip]['wan-rx'] += bits_count + + # Now that we have sum of all traffic in bytes for given period + # calculate real throughput and transform it to appropriate unit + for addr in tmp_accounting_values: + mac = self._get_accounting_mac_by_ip(addr) + if not mac: + _LOGGER.debug(f"Address {addr} not found in accounting data, skipping update") + continue + + self.data['accounting'][mac]['wan-tx'] = round( + tmp_accounting_values[addr]['wan-tx'] / time_diff * traffic_div, 2) + self.data['accounting'][mac]['wan-rx'] = round( + tmp_accounting_values[addr]['wan-rx'] / time_diff * traffic_div, 2) + + if local_traffic_enabled: + self.data['accounting'][mac]['lan-tx'] = round( + tmp_accounting_values[addr]['lan-tx'] / time_diff * traffic_div, 2) + self.data['accounting'][mac]['lan-rx'] = round( + tmp_accounting_values[addr]['lan-rx'] / time_diff * traffic_div, 2) + else: + # If local traffic was enabled earlier and then disabled return counters for LAN traffic to 0 + if 'lan-tx' in self.data['accounting'][mac]: + self.data['accounting'][mac]['lan-tx'] = 0.0 + if 'lan-rx' in self.data['accounting'][mac]: + self.data['accounting'][mac]['lan-rx'] = 0.0 + else: + # No time diff, just initialize/return counters to 0 for all + for addr in tmp_accounting_values: + mac = self._get_accounting_mac_by_ip(addr) + if not mac: + _LOGGER.debug(f"Address {addr} not found in accounting data, skipping update") + continue + + self.data['accounting'][mac]['wan-tx'] = 0.0 + self.data['accounting'][mac]['wan-rx'] = 0.0 + + if local_traffic_enabled: + self.data['accounting'][mac]['lan-tx'] = 0.0 + self.data['accounting'][mac]['lan-rx'] = 0.0 diff --git a/custom_components/mikrotik_router/mikrotikapi.py b/custom_components/mikrotik_router/mikrotikapi.py index e28938c..66f1bbc 100644 --- a/custom_components/mikrotik_router/mikrotikapi.py +++ b/custom_components/mikrotik_router/mikrotikapi.py @@ -60,6 +60,7 @@ class MikrotikAPI: self._connection_retry_sec = 58 self.error = None self.connection_error_reported = False + self.accounting_last_run = None # Default ports if not self._port: @@ -427,6 +428,7 @@ class MikrotikAPI: self.disconnect() self.lock.release() return False + except ( librouteros_custom.exceptions.TrapError, librouteros_custom.exceptions.MultiTrapError, @@ -463,3 +465,95 @@ class MikrotikAPI: return True return False + + @staticmethod + def _current_milliseconds(): + from time import time + return int(round(time() * 1000)) + + def is_accounting_and_local_traffic_enabled(self) -> (bool, bool): + # Returns: + # 1st bool: Is accounting enabled + # 2nd bool: Is account-local-traffic enabled + + if not self.connection_check(): + return False, False + + response = self.path("/ip/accounting") + if response is None: + return False, False + + for item in response: + if 'enabled' not in item: + continue + if not item['enabled']: + return False, False + + for item in response: + if 'account-local-traffic' not in item: + continue + if not item['account-local-traffic']: + return True, False + + return True, True + + # --------------------------- + # take_accounting_snapshot + # Returns float -> period in seconds between last and current run + # --------------------------- + def take_accounting_snapshot(self) -> float: + """Get accounting data""" + if not self.connection_check(): + return 0 + + accounting = self.path("/ip/accounting", return_list=False) + + self.lock.acquire() + try: + # Prepare command + take = accounting('snapshot/take') + except librouteros_custom.exceptions.ConnectionClosed: + self.disconnect() + self.lock.release() + return 0 + except ( + librouteros_custom.exceptions.TrapError, + librouteros_custom.exceptions.MultiTrapError, + librouteros_custom.exceptions.ProtocolError, + librouteros_custom.exceptions.FatalError, + ssl.SSLError, + BrokenPipeError, + OSError, + ValueError, + ) as api_error: + self.disconnect("accounting_snapshot", api_error) + self.lock.release() + return 0 + except: + self.disconnect("accounting_snapshot") + self.lock.release() + return 0 + + try: + list(take) + except librouteros_custom.exceptions.ConnectionClosed as api_error: + self.disconnect("accounting_snapshot", api_error) + self.lock.release() + return 0 + except: + self.disconnect("accounting_snapshot") + self.lock.release() + return 0 + + self.lock.release() + + # First request will be discarded because we cannot know when the last data was retrieved + # prevents spikes in data + if not self.accounting_last_run: + self.accounting_last_run = self._current_milliseconds() + return 0 + + # Calculate time difference in seconds and return + time_diff = self._current_milliseconds() - self.accounting_last_run + self.accounting_last_run = self._current_milliseconds() + return time_diff / 1000 diff --git a/custom_components/mikrotik_router/sensor.py b/custom_components/mikrotik_router/sensor.py index 0fccbc9..1ddf128 100644 --- a/custom_components/mikrotik_router/sensor.py +++ b/custom_components/mikrotik_router/sensor.py @@ -12,6 +12,19 @@ from .const import (DOMAIN, DATA_CLIENT, ATTRIBUTION) _LOGGER = logging.getLogger(__name__) + +# --------------------------- +# format_attribute +# --------------------------- +def format_attribute(attr): + res = attr.replace("-", " ") + res = res.capitalize() + res = res.replace(" ip ", " IP ") + res = res.replace(" mac ", " MAC ") + res = res.replace(" mtu", " MTU") + return res + + ATTR_ICON = "icon" ATTR_LABEL = "label" ATTR_UNIT = "unit" @@ -66,8 +79,54 @@ SENSOR_TYPES = { ATTR_PATH: "interface", ATTR_ATTR: "rx-bits-per-second", }, + "accounting_lan_tx": { + ATTR_DEVICE_CLASS: None, + ATTR_ICON: "mdi:download-network", + ATTR_LABEL: "LAN TX", + ATTR_GROUP: "Accounting", + ATTR_UNIT: "ps", + ATTR_UNIT_ATTR: "tx-rx-attr", + ATTR_PATH: "accounting", + ATTR_ATTR: "lan-tx", + }, + "accounting_lan_rx": { + ATTR_DEVICE_CLASS: None, + ATTR_ICON: "mdi:upload-network", + ATTR_LABEL: "LAN RX", + ATTR_GROUP: "Accounting", + ATTR_UNIT: "ps", + ATTR_UNIT_ATTR: "tx-rx-attr", + ATTR_PATH: "accounting", + ATTR_ATTR: "lan-rx", + }, + "accounting_wan_tx": { + ATTR_DEVICE_CLASS: None, + ATTR_ICON: "mdi:download-network", + ATTR_LABEL: "WAN TX", + ATTR_GROUP: "Accounting", + ATTR_UNIT: "ps", + ATTR_UNIT_ATTR: "tx-rx-attr", + ATTR_PATH: "accounting", + ATTR_ATTR: "wan-tx", + }, + "accounting_wan_rx": { + ATTR_DEVICE_CLASS: None, + ATTR_ICON: "mdi:upload-network", + ATTR_LABEL: "WAN RX", + ATTR_GROUP: "Accounting", + ATTR_UNIT: "ps", + ATTR_UNIT_ATTR: "tx-rx-attr", + ATTR_PATH: "accounting", + ATTR_ATTR: "wan-rx", + }, } +DEVICE_ATTRIBUTES_ACCOUNTING = [ + "address", + "mac-address", + "comment" +] + # --------------------------- # async_setup_entry @@ -101,7 +160,7 @@ def update_items(inst, mikrotik_controller, async_add_entities, sensors): new_sensors = [] for sensor in SENSOR_TYPES: - if "traffic_" not in sensor: + if "system_" in sensor: item_id = f"{inst}-{sensor}" _LOGGER.debug("Updating sensor %s", item_id) if item_id in sensors: @@ -133,6 +192,24 @@ def update_items(inst, mikrotik_controller, async_add_entities, sensors): ) new_sensors.append(sensors[item_id]) + if "accounting_" in sensor: + for uid in mikrotik_controller.data["accounting"]: + + item_id = f"{inst}-{sensor}-{mikrotik_controller.data['accounting'][uid]['mac-address']}" + if item_id in sensors: + if sensors[item_id].enabled: + sensors[item_id].async_schedule_update_ha_state() + continue + + if SENSOR_TYPES[sensor][ATTR_ATTR] in mikrotik_controller.data['accounting'][uid].keys(): + sensors[item_id] = MikrotikAccountingSensor( + mikrotik_controller=mikrotik_controller, + inst=inst, + sensor=sensor, + uid=uid, + ) + new_sensors.append(sensors[item_id]) + if new_sensors: async_add_entities(new_sensors, True) @@ -277,3 +354,65 @@ class MikrotikControllerTrafficSensor(MikrotikControllerSensor): self._data["default-name"], self._sensor, ) + + +# --------------------------- +# MikrotikAccountingSensor +# --------------------------- +class MikrotikAccountingSensor(MikrotikControllerSensor): + """Define an Mikrotik Accounting sensor.""" + + def __init__(self, mikrotik_controller, inst, sensor, uid): + """Initialize.""" + super().__init__(mikrotik_controller, inst, sensor) + self._uid = uid + self._data = mikrotik_controller.data[SENSOR_TYPES[sensor][ATTR_PATH]][uid] + + @property + def name(self): + """Return the name.""" + return f"{self._inst} {self._data['host-name']} {self._type[ATTR_LABEL]} " + + @property + def unique_id(self): + """Return a unique_id for this entity.""" + return f"{self._inst.lower()}-{self._sensor.lower()}-{self._data['mac-address'].lower()}" + + @property + def device_info(self): + """Return a accounting description for device registry.""" + info = { + "identifiers": { + ( + DOMAIN, + "serial-number", + self._ctrl.data["routerboard"]["serial-number"], + "sensor", + "Accounting" + ) + }, + "manufacturer": self._ctrl.data["resource"]["platform"], + "model": self._ctrl.data["resource"]["board-name"], + "name": self._type[ATTR_GROUP], + } + return info + + @property + def device_state_attributes(self): + """Return the state attributes.""" + attributes = self._attrs + for variable in DEVICE_ATTRIBUTES_ACCOUNTING: + if variable in self._data: + attributes[format_attribute(variable)] = self._data[variable] + + return attributes + + async def async_added_to_hass(self): + """Port entity created.""" + _LOGGER.debug( + "New sensor %s (%s [%s] %s)", + self._inst, + self._data["host-name"], + self._data["mac-address"], + self._sensor, + ) diff --git a/docs/assets/images/ui/accounting_sensor.jpg b/docs/assets/images/ui/accounting_sensor.jpg new file mode 100644 index 0000000..10583ed Binary files /dev/null and b/docs/assets/images/ui/accounting_sensor.jpg differ diff --git a/docs/assets/images/ui/setup_integration.PNG b/docs/assets/images/ui/setup_integration.PNG new file mode 100644 index 0000000..42958f9 Binary files /dev/null and b/docs/assets/images/ui/setup_integration.PNG differ diff --git a/docs/assets/images/ui/setup_integration.png b/docs/assets/images/ui/setup_integration.png deleted file mode 100644 index 2eada0d..0000000 Binary files a/docs/assets/images/ui/setup_integration.png and /dev/null differ