FIX BUGS:

--Firmware and packages download is failing for wrong arch name in code
--Fix CHR firmware updates
--Fix #5
New Feature:
--Check all packages and include all packages to also update with firmware

py/libs/firm_lib.py

@@ -8,24 +8,19 @@
 import pytz
 import datetime
 import time
-import uuid
 import socket
 import config
 from libs.db import db_sysconfig,db_firmware,db_tasks,db_events
-from cryptography.fernet import Fernet 
 from libs.check_routeros.routeros_check.resource import RouterOSCheckResource
 from libs.check_routeros.routeros_check.helper import  RouterOSVersion
 from typing import  Dict
 import re
 import json 
 import logging
-from libs.red import RedisDB
-from libs.ssh_helper import SSH_Helper
 import os
 from bs4 import BeautifulSoup
 import urllib.request
 import hashlib
-import netifaces
 log = logging.getLogger("util")
 from libs import util
 try:
@@ -85,6 +80,7 @@ def get_mikrotik_latest_firmware_link():
 
 def get_mikrotik_download_links(version,all_package=False):
     try:
+        log.info("Downloading firmwares from https://mikrotik.com/download/archive?v={}".format(version))
         html_page = urllib.request.urlopen("https://mikrotik.com/download/archive?v={}".format(version))
         soup = BeautifulSoup(html_page, "html.parser")
         firms={}
@@ -200,7 +196,7 @@ def extract_zip (file,path):
     except Exception as e:
         log.error(e)
         
-def download_firmware_to_repository(version,q,arch="all",all_package=False):
+def download_firmware_to_repository(version,q,arch="all",all_package=True):
     #repository='/app/firms/'
     repository=config.FIRM_DIR
     #create direcorty version in repository if not exist
@@ -222,32 +218,31 @@ def download_firmware_to_repository(version,q,arch="all",all_package=False):
                 if q:
                     q.put({"status":False})
                 return False
-            if all_package and arch+"-allpackage" == lnk:
+            if all_package and "-allpackage" in lnk == lnk:
-                arch_togo=lnk
+                arch_togo=lnk.split("-allpackage")[0]
                 link=links[lnk]["link"]
                 sha256=links[lnk]["sha"]
-                file=path+"all_packages-" + arch + ".zip"
+                file=path+"all_packages-" + arch_togo + ".zip"
-                log.error(link)
                 done=web2file(link, file, sha256=sha256)
                 files=extract_zip(file, path)
                 try:
                     if done and len(files)>0:
                         for f in files:
                             file=path+f
-                            log.error(file)
                             sha256=check_sha256(file)
-                            firm.insert(version=version, location=file, architecture=arch+"-"+f.split("-")[0], sha256=sha256).on_conflict(conflict_target=['version', 'architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
+                            firm.insert(version=version, location=file, architecture=arch_togo+"-"+f.split("-{}".format(version))[0], sha256=sha256).on_conflict(conflict_target=['version', 'architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
                 except Exception as e:
                     log.error(e)
                     pass
                 if q:
                     q.put({"status":True})
                 # return True
+                continue
             if arch!="all" and arch==lnk:
                 arch_togo=lnk
                 link=links[lnk]["link"]
                 sha256=links[lnk]["sha"]
-                file=path+"{}.npk".format(arch)
+                file=path+"{}.npk".format(arch_togo)
                 done=web2file(link, file,sha256=sha256)
                 try:
                     if done:
@@ -257,13 +252,14 @@ def download_firmware_to_repository(version,q,arch="all",all_package=False):
                     pass
                 if q:
                     q.put({"status":True})
+                continue
                 # return True
             if arch=="all":
                 #download file to path and check sha265 
                 arch_togo=lnk
                 link=links[lnk]["link"]
                 sha256=links[lnk]["sha"]
-                file=path+"{}.npk".format(arch)
+                file=path+"{}.npk".format(arch_togo)
                 done=web2file(link, file,sha256=sha256)
                 try:
                     if done:
@@ -302,6 +298,8 @@ def update_device(dev,q):
         q.put({"id": dev.id})
         return False
     arch=dev.arch
+    if "x86" in arch:
+        arch="x86"
     if not dev.firmware_to_install or RouterOSVersion(dev.firmware_to_install)!=ver_to_install:
         dev.firmware_to_install=ver_to_install
         dev.save()
@@ -321,55 +319,101 @@ def update_device(dev,q):
     #get correct firmware from db for updating
     firm=False
     if ISPRO:
-        firm=utilpro.safe_check(dev,_installed_version,ver_to_install)
+        firm,firm2=utilpro.safe_check(dev,_installed_version,ver_to_install)
     elif arch and arch!='':
+        log.warning(ver_to_install)
+        log.warning(arch)
         firm=db_firmware.get_frim_by_version(ver_to_install, arch)
     else:
         q.put({"id": dev.id})
-    if firm and firm.architecture == arch:
+    options=util.build_api_options(dev)
-        download_firmware_to_repository(str(ver_to_install), False,arch=arch,all_package=False)
+    #get /system package print 
-        dev.failed_attempt=dev.failed_attempt+1
+    router=RouterOSCheckResource(options) 
-        if dev.failed_attempt > 3:
+    try:
-            db_events.firmware_event(dev.id,"updater","Update Failed","Critical",0,"Unable to Update device")
+        call=router.api.path('/system/package')
-        dev.status="updating"
+        results = tuple(call)
-        dev.save()
+    except:
-        options=util.build_api_options(dev)
+        q.put({"id": dev.id})
-        try:
+        return False
+    packages=[]
+    if firm:
+        packages.append(firm)
+    else:
+        db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #2 :Please check firmware config in settings section")
+        log.error('No Firmware found for device {}({})'.format(dev.name,dev.ip))
+        q.put({"id": dev.id})
+        return False
+    
+    for res in results:
+        log.error(res['name'])
+        if res['name']!="routeros":
+            package=db_firmware.get_frim_by_version(ver_to_install, "{}-{}".format(arch,res['name']))
+            if package:
+                packages.append(package)
+    log.error(packages)
+    # q.put({"id": dev.id})
+    # return False
+
+    try:
+        apply_firmware(packages, firm2, arch, dev, router, events, q)
+    except:
+        q.put({"id": dev.id})
+
+def apply_firmware(packages,firm2,arch,dev,router,events,q):
+    dev.failed_attempt=dev.failed_attempt+1
+    if dev.failed_attempt > 3:
+        db_events.firmware_event(dev.id,"updater","Update Failed","Critical",0,"Unable to Update device")
+    dev.status="updating"
+    dev.save()
+    try:
+        url=dev.peer_ip
+        api = router._connect_api()
+        if not url:
             url=db_sysconfig.get_sysconfig('system_url')
-            url=url+"/api/firmware/get_firmware/{}".format(firm.id)
+        if not "http" in url:
-            router=RouterOSCheckResource(options)
+            url="http://"+url
-            api = router._connect_api()
+        if firm2:
-            params = {"url": url,"keep-result":"yes","dst-path":arch+".npk"}
+            url_firm2=url+"/api/firmware/get_firmware/{}".format(firm2.id)
+            params = {"url": url_firm2,"keep-result":"yes","dst-path":firm2.architecture+".npk"}
             cmd='/tool/fetch'
             call = api(cmd,**params)
             results = tuple(call)
             result: Dict[str, str] = results[-1]
-            if result['status'] == 'finished':
+            if result['status'] != 'finished':
-                util.check_or_fix_event(events,"firmware","Device storage")
-                cmd='/system/reboot'
-                call = api(cmd)
-                rebootresults = tuple(call)
-                if len(rebootresults)==0:
-                    util.check_or_fix_event(events,"firmware","Firmware repositpry")
-                    dev.status="updated"
-                    dev.save()
-                else:
-                    dev.status="failed"
-                    dev.save()
-            else:
-                db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"There is a problem with downloadin of Firmware in device")
                 dev.status="failed"
                 dev.save()
-        except Exception as e:
+                q.put({"id": dev.id})
-            dev.status="failed"
+                return False
-            dev.save()
+        for package in packages:
-            if 'no space left' in str(e):
+            url_package=url+"/api/firmware/get_firmware/{}".format(package.id)
-                db_events.firmware_event(dev.id,"updater","Device storage","Error",0,"There is not enogh space in device storage")
+            params = {"url": url_package,"keep-result":"yes","dst-path":package.architecture+".npk"}
-            if '404 Not Found' in str(e):
+            cmd='/tool/fetch'
-                db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #1 :Please check firmware config in settings section")
+            call = api(cmd, **params)
-            log.error(e)
+            results = tuple(call)
-            q.put({"id": dev.id})
+            log.warning(results)
-    else:
+            result: Dict[str, str] = results[-1]
-        db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #2 :Please check firmware config in settings section")
+            if result['status'] != 'finished':
-        log.error('No Firmware found for device {}({})'.format(dev.name,dev.ip))
+                log.error("There is a problem with downloading of Firmware in device")
-    q.put({"id": dev.id})
+                dev.status="failed"
+                dev.save()
+                db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"There is a problem with downloading of Firmware in device")
+                q.put({"id": dev.id})
+                return False
+        util.check_or_fix_event(events,"firmware","Device storage")
+        cmd='/system/reboot'
+        call = api(cmd)
+        rebootresults = tuple(call)
+        log.warning(rebootresults)
+        util.check_or_fix_event(events,"firmware","Firmware repositpry")
+        dev.status="updated"
+        dev.save()
+    except Exception as e:
+        dev.status="failed"
+        dev.save()
+        if 'no space left' in str(e):
+            db_events.firmware_event(dev.id,"updater","Device storage","Error",0,"There is not enogh space in device storage")
+        if '404 Not Found' in str(e):
+            db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #1 :Please check firmware config in settings section")
+        log.error(e)
+        q.put({"id": dev.id})
+    q.put({"id": dev.id})