Relase 1.0.8 and Change logs

Enhanced processing of DHCP logs
Added some redis functions

py/_version.py

@@ -1 +1 @@
-__version__ = "1.0.7"
+__version__ = "1.0.8"

py/api/api_dev.py

@@ -8,12 +8,11 @@
 from flask import request,redirect ,session
 import datetime
 import html
-
 import config
 import re
 from libs.red import RedisDB
 from libs.webutil import app,buildResponse,login_required,get_myself,get_ip,get_agent
-from libs import util
+from libs import util,ping
 from libs.db import db_device,db_groups,db_user_group_perm,db_user_tasks,db_sysconfig,db_syslog
 import logging
 import json
@@ -299,12 +298,16 @@ def dev_info():
     res=db_device.get_device(devid)
     options=util.build_api_options(db_device.get_devices_by_id([res['id'],])[0])
     network_info=[]
+    res['online']=True
     try:
         if util.check_port(options['host'],options['port']):
             router=util.RouterOSCheckResource(options)
             network_info=util.get_network_data(router)
             del network_info['total']
+        else:
+            res['online']=False
     except:
+        res['online']=False
         pass
     interfaces=[]
     for iface in network_info:
@@ -325,6 +328,45 @@ def dev_info():
         log.error(e)
         return buildResponse({'status': 'failed'}, 200, error="Wrong Data")
         pass
+    try:
+        res['active_users']=[]
+        if res['online']:
+            res['active_users']=tuple(router.api("/user/active/print"))
+    except:
+        res['active_users']=[]
+    try:
+        res['ping']=ping.get_ping_results(res['ip'], 5, 1)
+    except Exception as e:
+        res['ping']=[]
+    return buildResponse(res,200)
+
+@app.route('/api/dev/kill_session', methods = ['POST'])
+@login_required(role='admin',perm={'device':'full'})
+def dev_kill_session():
+    """return dev info"""
+    input = request.json
+    devid=input.get('devid',False)
+    item=input.get('item',False)
+    if not devid or not isinstance(devid, int):
+        return buildResponse({'status': 'failed'},200,error="Wrong Data")
+    try:
+        dev=db_device.get_devices_by_id([devid,])[0]
+    except:
+        return buildResponse({'status': 'failed'},200,error="Wrong Data")
+    if not dev:
+        return buildResponse({'status': 'failed'},200,error="Wrong Data")
+    options=util.build_api_options(dev)
+    router=util.RouterOSCheckResource(options)
+    # active_users=tuple(router.api("/user/active/print"))
+    # if item in active_users:
+    try:
+        acturl=router.api.path("user","active")
+        res=tuple(acturl('request-logout', **{'.id': item['.id']}))
+        log.error(res)
+    except Exception as e:
+        log.error(e)
+        pass
+    res=tuple(router.api("/user/active/print"))
     return buildResponse(res,200)
 
 @app.route('/api/dev/sensors', methods = ['POST'])

py/api/api_logs.py

@@ -402,6 +402,8 @@ def dashboard_stats():
     # res['update_available']=True
     if username:
         res['username']=username
+    else:
+        res['username']=False
     res['blog']=[]
     noconnectiondata={
                 "content": "Unable to connect to mikrowizard.com! please check server connection",

py/api/api_user_tasks.py

@@ -112,7 +112,7 @@ def user_tasks_create():
         taskid=task.id
         crontab = CronTab(user=True)
         directory=Path(app.root_path).parent.absolute()
-        command = "python3 {}/task_run.py {}".format(directory,taskid)
+        command = "/usr/local/bin/python3 {}/task_run.py {}  >> /var/log/cron.log 2>&1".format(directory,taskid)
         comment = "MikroWizard task #" + "taskid:{};".format(taskid)
         jobs = crontab.find_comment(comment)
         if len(list(jobs)) > 0:

py/libs/firm_lib.py

@@ -363,6 +363,9 @@ def apply_firmware(packages,firm2,arch,dev,router,events,q):
     dev.failed_attempt=dev.failed_attempt+1
     if dev.failed_attempt > 3:
         db_events.firmware_event(dev.id,"updater","Update Failed","Critical",0,"Unable to Update device")
+        dev.save()
+        q.put({"id": dev.id})
+        return False
     dev.status="updating"
     dev.save()
     try:

py/libs/ping.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# ping.py: ping tool for MikroWizard
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+import asyncio
+import platform
+
+def ping_quality(time_ms):
+    if time_ms is None:
+        return "unreachable", "fa-solid fa-times-circle", "#dc3545"  # Red, times circle
+    if time_ms <= 50:
+        return "excellent", "fa-solid fa-check-circle", "#28a745"  # Green, check circle
+    elif time_ms <= 100:
+        return "good", "fa-solid fa-thumbs-up", "#80c29e"  # Light green, thumbs up
+    elif time_ms <= 200:
+        return "average", "fa-solid fa-exclamation-circle", "#ffc107"  # Yellow, exclamation circle
+    else:
+        return "poor", "fa-solid fa-times-circle", "#dc3545"  # Red, times circle
+
+async def ping_host(host, timeout=1):
+    system = platform.system()
+    cmd = ["ping", "-c", "1", "-W", str(timeout), host]
+
+    process = await asyncio.create_subprocess_exec(
+        *cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
+    )
+    
+    stdout, stderr = await process.communicate()
+    result = stdout.decode().strip()
+    error = stderr.decode().strip()
+
+    # Extract time from output
+    time_ms = None
+    if "time=" in result:
+        try:
+            time_part = result.split("time=")[-1].split()[0]
+            time_ms = float(time_part)
+        except ValueError:
+            pass
+
+    quality, icon, color = ping_quality(time_ms)
+    raw_response = result.split("\n")[0] if result else error.split("\n")[0]
+
+    return {
+        "host": host,
+        "status": "success" if time_ms is not None else "failed",
+        "time": time_ms if time_ms is not None else None,
+        "ping_quality": quality,
+        "icon": icon,
+        "color": color,
+        "raw_response": raw_response
+    }
+
+async def multi_ping_one_host(host, count=4, timeout=1):
+    tasks = [ping_host(host, timeout) for _ in range(count)]
+    results = await asyncio.gather(*tasks)
+
+    successful_pings = [r["time"] for r in results if r["status"] == "success"]
+    failed_pings = count - len(successful_pings)
+
+    average_ping_time = round(sum(successful_pings) / len(successful_pings), 2) if successful_pings else None
+
+    response = {
+        "host": host,
+        "count": count,
+        "successful_pings": len(successful_pings),
+        "failed_pings": failed_pings,
+        "average_ping_time": average_ping_time,
+        "results": results
+    }
+
+    return response
+
+def get_ping_results(host, count=4, timeout=1):
+    return asyncio.run(multi_ping_one_host(host, count, timeout))

py/libs/red.py

@@ -36,7 +36,6 @@ class RedisDB(object):
         self.r = redis.Redis(host='localhost', port=6379, db=0)
         self.delta = options.get('delta','')
 
-
     def create_sensor_rts(self,sensor):
         retention=self.retention
         if "rx" in sensor or "tx" in sensor:
@@ -137,3 +136,24 @@ class RedisDB(object):
                 pass
         return data
 
+
+    def store_data(self, device_id, key, command):
+        """
+        store data for specific key of specific command
+        """
+        redis_key = f"device:{device_id}:{key}"
+        
+        # Add the command to the list
+        self.r.rpush(redis_key, command.encode('utf-8'))
+        
+        # Trim the list to keep only the last 20 commands
+        # self.r.ltrim(redis_key, -20, -1)
+
+    def get_last_n_data(self, device_id, key, count=20):
+        """
+        Retrieves the last 'count' data executed for a specific device ID and key.
+        """
+        redis_key = f"device:{device_id}:{key}"
+        raw_commands = self.r.lrange(redis_key, -count, -1)
+        return [cmd.decode('utf-8') for cmd in raw_commands]
+        # return self.r.lrange(redis_key, -count, -1)

py/libs/util.py

@@ -416,7 +416,6 @@ def check_syslog_config(dev,router,apply=False):
         if len(confs)!=3:
             if apply:
                 ids=[item.get('.id') for item in results if 'mikrowizard' in item.get('prefix')]
-                log.error(ids)
                 if len(ids):
                     call.remove(*ids)
                 keys=['critical','error','info']
@@ -632,6 +631,8 @@ def run_snippet(dev, snippet):
                 result=ssh.exec_command(snippet)
                 if not result:
                     result="executed successfully"
+                if "no such item" in result:
+                    result=False
             except Exception as e:
                 log.error(e)
                 log_alert('ssh',dev,'During backup ssh error')

py/main.py

@@ -17,6 +17,7 @@ from api import api_backups
 from api import api_snippet
 try:
     from api import api_pro_api
+    from api import api_pro_api2
 except ImportError:
     pass
 

py/mules/syslog.py

@@ -5,15 +5,15 @@
 # MikroWizard.com , Mikrotik router management solution
 # Author: sepehr.ha@gmail.com
 
-from math import e
 import socketserver
-import re
+import asyncio
 import time
+import logging
+import re
 
 from libs.db import db_device
 import logging
 from libs.db import db_AA,db_events
-log = logging.getLogger("SYSLOG")
 from libs import util
 try:
     from libs import utilpro
@@ -22,27 +22,40 @@ except ImportError:
     ISPRO=False
     pass
 
+log = logging.getLogger("SYSLOG")
+
+
+# A global asyncio event loop
+event_loop = asyncio.new_event_loop()
+asyncio.set_event_loop(event_loop)
 
 class SyslogUDPHandler(socketserver.BaseRequestHandler):
-    def extract_data_from_regex(self,regex,line):
+    def extract_data_from_regex(self, regex, line):
         try:
             matches = re.finditer(regex, line, re.MULTILINE)
-            sgroups=[]   
+            sgroups = []
             for matchNum, match in enumerate(matches, start=1):
                 for groupNum in range(0, len(match.groups())):
                     groupNum = groupNum + 1
                     sgroups.append(match.group(groupNum))
             return sgroups
-        except:
+        except Exception as e:
+            log.error(f"Regex error: {e}")
             return None
+
     def handle(self):
+        # Run the coroutine in the global event loop
+        asyncio.run_coroutine_threadsafe(self.handle_log(), event_loop)
+        # Respond to the client (optional)
+
+    async def handle_log(self):
         data = bytes.decode(self.request[0].strip(), encoding="utf-8")
         message = str(data)
         #get current timestamp
         ts = int(time.time())
         socket = self.request[1]
         dev=db_device.query_device_by_ip(self.client_address[0])
-        regex=r'(.*),?(info.*|warning|critical) mikrowizard(\d+):.*'
+        regex=r'(.*),?(info.*|warning|critical|error) mikrowizard(\d+):.*'
         if dev:
             info=self.extract_data_from_regex(regex,message)
             opts=util.build_api_options(dev)
@@ -53,6 +66,7 @@ class SyslogUDPHandler(socketserver.BaseRequestHandler):
         except:
             log.error("**device id mismatch")
             log.error(message)
+            log.error(info)
             log.error(self.client_address[0])
             log.error("device id mismatch**")
             dev=False
@@ -96,6 +110,9 @@ class SyslogUDPHandler(socketserver.BaseRequestHandler):
                 regex= r"system,info mikrowizard\d+: (.*) (changed|added|removed|unscheduled) by (winbox-\d.{1,3}\d\/.*\(winbox\)|mac-msg\(winbox\)|tcp-msg\(winbox\)|ssh|telnet|api|api-ssl|.*\/web|ftp|www-ssl).*:(.*)@(.*) \((.*)\)"
                 #with new versions of mikrotik syslog is not sending the correct trace in message
                 buged_regex=r"system,info mikrowizard\d+: (.*) (changed|added|removed|unscheduled) by  \((.*)\)"
+                if ISPRO:
+                    # threading.Thread(target=utilpro.do_pro,args=()).start()
+                    utilpro.do_pro("syslog", False, dev, message)
                 if re.match(regex, message):
                     info=self.extract_data_from_regex(regex, message)
                     address=info[4].split('/')
@@ -138,15 +155,38 @@ class SyslogUDPHandler(socketserver.BaseRequestHandler):
                 elif "link up" in message:
                     info=self.extract_data_from_regex(link_regex,message)
                     util.check_or_fix_event(events,'state',"Link Down: " + info[0])
-            elif "dhcp,info mikrowizard" in message:
-                dhcp_regex=r'dhcp,info mikrowizard\d+: (dhcp-client|.*) (deassigned|assigned|.*) (\d+\.\d+\.\d+\.\d+|on.*address)\s*(from|to|$)\s*(.*)'
+            elif any(term in message for term in ["dhcp,info","dhcp,critical","dhcp,warning"]):
+                type='cleint'
+                # if (" dhcp-client on" in message):
+                #     dhcp_regex=r'dhcp,info mikrowizard\d+: dhcp-client on (.*) (got IP address|lost IP address) (\b(?:\d{1,3}\.){3}\d{1,3}\b|\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\b) ?-? ?(.*)?'
+                # else:
+                #     dhcp_regex=r'dhcp,info mikrowizard\d+: (.*) (assigned|deassigned) (\b(?:\d{1,3}\.){3}\d{1,3}\b|\b(?:[A-Fa-f0-9]{1,4}:){7}[A-Fa-f0-9]{1,4}\b) (for|to|from) (\b([A-Fa-f0-9]{2}[:-]){5}[A-Fa-f0-9]{2}\b)? ?(.*)?'
+                #     type='server'
+                if (not " dhcp-client on" in message):
+                    type='server'
+                # dhcp_regex=r'dhcp,info mikrowizard\d+: (dhcp-client|.*) (deassigned|assigned|.*) (\d+\.\d+\.\d+\.\d+|on.*address)\s*(from|to|for|- lease stopped locally|$)\s*(.*)'
+                dhcp_regex=r'dhcp,(?:info|warning|critical|error)(?:,info|,warning|,critical|,error)? mikrowizard\d+: (.*)'
                 info=self.extract_data_from_regex(dhcp_regex,message)
-                if info and "assigned" in message:
-                    db_events.state_event(dev.id, "syslog", "dhcp assigned","info",1,"server {} assigned {} to {}".format(info[0],info[2],info[4]))
-                elif info and "deassigned" in message:
-                    db_events.state_event(dev.id, "syslog", "dhcp deassigned","info",1,"server {} deassigned {} from {}".format(info[0],info[2],info[4]))
-                elif info and "dhcp-client" in message:
-                    db_events.state_event(dev.id, "syslog", "dhcp client","info",1,"{} {}".format(info[1],info[2]))
+                if "dhcp,info" in message:
+                    level="info"
+                elif "dhcp,warning" in message:
+                    level="warning"
+                elif "dhcp,critical" in message:
+                    level="critical"
+                else:
+                    level="error"
+                if type=='server':
+                    if info and "deassigned" in message:
+                        log.error("Logging deassigned")
+                        db_events.state_event(dev.id, "syslog", "dhcp deassigned",level,1,"{}".format(info[0]))
+                        # db_events.state_event(dev.id, "syslog", "dhcp assigned","info",1,"server {} assigned {} to {}".format(info[0],info[2],info[4]))
+                    elif info and "assigned" in message:
+                        log.error("Logging deassigned")
+                        db_events.state_event(dev.id, "syslog", "dhcp assigned",level,1,"{}".format(info[0]))
+                        # db_events.state_event(dev.id, "syslog", "dhcp deassigned","info",1,"server {} deassigned {} from {}".format(info[0],info[2],info[4]))
+                else:
+                    db_events.state_event(dev.id, "syslog", "dhcp client",level,1,"{}".format(info[0]))
+                    # db_events.state_event(dev.id, "syslog", "dhcp client","info",1,"{} {}".format(info[1],info[2]))
             elif "wireless,info mikrowizard" in message:
                 if ISPRO:
                     utilpro.wireless_syslog_event(dev ,message)
@@ -162,9 +202,24 @@ class SyslogUDPHandler(socketserver.BaseRequestHandler):
                         log.error(message)
             else:
                 log.error(message)
+
+def start_event_loop(loop):
+    """Run the event loop in a separate thread."""
+    asyncio.set_event_loop(loop)
+    loop.run_forever()
+
 if __name__ == "__main__":
     try:
-        server = socketserver.UDPServer(("0.0.0.0",5014), SyslogUDPHandler)
-        server.serve_forever(poll_interval=0.5)
+        # Start the asyncio event loop in a separate thread
+        import threading
+        thread = threading.Thread(target=start_event_loop, args=(event_loop,), daemon=True)
+        thread.start()
+
+        # Start the UDP server
+        server = socketserver.UDPServer(("0.0.0.0", 5014), SyslogUDPHandler)
+        server.serve_forever()
     except (IOError, SystemExit):
         raise
+    except KeyboardInterrupt:
+        log.info("Shutting down server")
+        event_loop.stop()

release-notes.md

@@ -1,5 +1,25 @@
-
 # Release Notes  
+
+## Version 1.0.8 Free / 1.1.0 Pro
+
+### New Features  
+- Router Ping Information: Added ping data to enhance connectivity monitoring.  
+- Active User Sessions: Device details page now displays current active users.  
+- Session Management: Introduced the ability to terminate active user sessions.  
+- Enhanced License Information: Dashboard now provides more detailed license-related insights.  
+- MikroTik Configuration Sync & Config Cloner (Pro): Introduced a new menu/page for configuration cloning/sync.  
+- DHCP Server & Lease History (Pro): DHCP server details along with historical lease information in device details.  
+
+### Improvements & Bug Fixes  
+- Async Syslog Server: The syslog server now utilizes asyncio for improved performance and efficiency.  
+- DHCP Log Handling: Enhanced processing of DHCP logs in the syslog system.  
+- Firmware Updater Fix: Resolved an issue where the firmware updater failed to retry properly.  
+---
+
+## Version 1.0.7 - Fast update
+- Firmware updater fix: Fix broken frimware update
+
+
 ## Version 1.0.6 - Firmware upgrade fix  
 
 ### Bugs Fixed