Seprate firmware related functions

Code cleanup
Allow deletiong of firmwares

.gitignore

@@ -167,3 +167,7 @@ cython_debug/
 
 *pro_api*
 *pro.py
+*pro-reqs.txt
+*update_gen.sh
+*.zip
+back/

py/api/api_firmware.py

@@ -8,12 +8,13 @@
 from flask import request, jsonify,session,send_file
 from playhouse.shortcuts import model_to_dict
 import datetime
-from libs.db import db_tasks,db_sysconfig,db_device,db_firmware,db_syslog
-from libs import util
+from libs.db import db_tasks,db_sysconfig,db_firmware,db_syslog
+from libs import util,firm_lib
 from libs.webutil import app, login_required, get_myself,buildResponse,get_myself,get_ip,get_agent
 import bgtasks
 import re
 import logging
+
 log = logging.getLogger("api.firmware")
 import json
 
@@ -85,7 +86,7 @@ def get_firms():
 def get_downloadable_firms():
     """get list of availble Firmwares from Mikrotik Official webstire"""
     input = request.json or {}
-    versions=util.get_mikrotik_versions()
+    versions=firm_lib.get_mikrotik_versions()
     versions = sorted(versions, key=lambda x: [int(y) if y.isdigit() else int(re.sub(r'\D', '', y)) for y in x.split('.')])
 
     return buildResponse({"versions":versions}, 200)
@@ -105,6 +106,24 @@ def download_firmware_to_repository():
     else:
         return buildResponse({'status': status}, 200)
 
+@app.route('/api/firmware/delete_from_repository', methods = ['POST'])
+@login_required(role='admin',perm={'settings':'full'})
+def delete_from_repository():
+    """Delete File from Repo"""
+    input = request.json or {}
+    id = input.get('id',False)
+    firmware=db_firmware.get_firm(id)
+    log.error(firmware)
+    if firmware:
+        #Delete file from location
+        location=firmware.location
+        res=util.delete_file(location)
+        log.error(res)
+        if res:
+            firmware.delete_instance()
+            return buildResponse({'status': True}, 200)    
+    return buildResponse({'status': False},200,error="Firm Already deleted")
+
 @app.route('/api/firmware/update_firmware_settings', methods = ['POST'])
 @login_required(role='admin',perm={'settings':'write'})
 def update_firmware_settings():

py/bgtasks.py

@@ -7,7 +7,7 @@
 
 from uwsgidecorators import spool
 from playhouse.shortcuts import model_to_dict
-from libs import util
+from libs import util,firm_lib
 import time
 from libs.db import db_tasks,db_device,db_events,db_user_group_perm,db_device
 from threading import Thread
@@ -107,7 +107,7 @@ def update_device(*args, **kwargs):
                         dev.save()
                     if(not dev.update_availble):
                         continue
-                    t = Thread(target=util.update_device, args=(dev, q))
+                    t = Thread(target=firm_lib.update_device, args=(dev, q))
                     t.start()
                     threads.append(t)
                 for t in threads:
@@ -137,7 +137,7 @@ def download_firmware(*args, **kwargs):
                 num_threads = 1
                 q = queue.Queue()
                 threads = []
-                t = Thread(target=util.download_firmware_to_repository, args=(ver, q))
+                t = Thread(target=firm_lib.download_firmware_to_repository, args=(ver, q))
                 t.start()
                 threads.append(t)
                 for t in threads:

py/libs/firm_lib.py

@@ -0,0 +1,359 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# firm_lib.py: functions that we need :)
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+import pytz
+import datetime
+import time
+import uuid
+import socket
+import config
+from libs.db import db_sysconfig,db_firmware,db_backups,db_events
+from cryptography.fernet import Fernet 
+from libs.check_routeros.routeros_check.resource import RouterOSCheckResource
+from libs.check_routeros.routeros_check.helper import  RouterOSVersion
+from typing import  Dict
+import re
+import json 
+import logging
+from libs.red import RedisDB
+from libs.ssh_helper import SSH_Helper
+import os
+from bs4 import BeautifulSoup
+import urllib.request
+import hashlib
+import netifaces
+log = logging.getLogger("util")
+from libs import utilpro,util
+try:
+    from libs import utilpro
+    ISPRO=True
+except ImportError:
+    ISPRO=False
+    pass
+import zipfile
+
+def extract_from_link(link,all_package=False):
+    if all_package:
+        regex = r"https:\/\/download\.mikrotik\.com\/routeros\/(\d{1,3}.*)?\/all_packages-(.*)-(.*).zip"
+        matches = re.match(regex, link)
+        if not matches:
+            return False
+        res=matches.groups()
+        version=res[0]
+        arch = res[1]
+        return {"link":link, "arch":arch, "version":version, "all_package":True}
+    else:
+        regex = r"https:\/\/download\.mikrotik\.com\/routeros\/(\d{1,3}.*)?\/routeros-(.*).npk"
+        matches = re.match(regex,link)
+        res=matches.groups()
+        version=res[0]
+        arch = res[1].replace(version, "")
+        if arch == "":
+            arch = "x86"
+        else:
+            arch=arch.replace("-","")
+        return {"link":link,"arch":arch, "version":version}
+    
+
+def get_mikrotik_latest_firmware_link():
+    try:
+        html_page = urllib.request.urlopen("https://mikrotik.com/download/")
+        soup = BeautifulSoup(html_page, "html.parser")
+        firms={}
+        for link in soup.findAll('a'):
+            link=str(link.get('href'))
+            if ".npk" in link:
+                frimware=extract_from_link(link)
+                firms.setdefault(frimware["version"],{})
+                firms[frimware["version"]][frimware["arch"]]={"link":frimware["link"],"mark":"latest"}
+                # firms.append(link)
+        return firms
+    except Exception as e:
+        log.error(e)
+        return False
+
+def get_mikrotik_download_links(version,all_package=False):
+    try:
+        html_page = urllib.request.urlopen("https://mikrotik.com/download/archive?v={}".format(version))
+        soup = BeautifulSoup(html_page, "html.parser")
+        firms={}
+        for trs in soup.findAll('tr'):
+            link=trs.findAll('a')
+            if len(link):
+                lnk=str(link[0].get('href'))
+                sha=str(link[1].get('data-checksum-sha256'))
+                if ".npk" in lnk:
+                    frimware=extract_from_link(lnk)
+                    firms.setdefault(frimware["version"], {})
+                    firms[frimware["version"]][frimware["arch"]]={"link":frimware["link"],"sha":sha}
+                    # firms.append(link)
+                elif all_package and ".zip" in lnk:
+                    frimware=extract_from_link(lnk, all_package=all_package)
+                    if not frimware:
+                        continue
+                    firms.setdefault(frimware["version"], {})
+                    firms[frimware["version"]][frimware["arch"]+"-"+"allpackage"]={"link":frimware["link"],"sha":sha}
+        return firms
+    except Exception as e:
+        log.error(e)
+        return False    
+
+def get_mikrotik_versions():
+    try:
+        html_page = urllib.request.urlopen("https://mikrotik.com/download/archive")
+        soup = BeautifulSoup(html_page, "html.parser")
+        versions=[]
+        for link in soup.findAll('a'):
+            ver=link.find("strong")
+            if ver:
+                versions.append(ver.text)
+        try:
+            vers=list(get_mikrotik_latest_firmware_link().keys())
+            if versions and vers:
+                unique_elements = set(versions + vers)
+                versions = list(unique_elements)
+            elif not versions and vers:
+                if vers:
+                    versions = vers
+        except Exception as e:
+            log.error(e)
+            pass
+        return versions
+    except Exception as e:
+        log.error(e)
+        return False
+
+def check_sha256(path,sha256=False):
+    hash_obj = hashlib.sha256()
+    if not sha256 and os.path.exists(path):
+        with open(path, 'rb') as f:
+            hash_obj.update(f.read())
+        return hash_obj.hexdigest()
+    elif os.path.exists(path) and sha256:
+        with open(path, 'rb') as f:
+            hash_obj.update(f.read())
+        return hash_obj.hexdigest() == sha256
+    else:
+        return False 
+
+def web2file(url, filePath,sha256=False, tries=3, timeout=3, sleepBetween=1):
+    tempPath = filePath
+    status=False
+    if os.path.exists(tempPath) and sha256:
+        hash_obj = hashlib.sha256()
+        with open(tempPath, 'rb') as f:
+            hash_obj.update(f.read())
+        if hash_obj.hexdigest() == sha256:
+            log.error("File already exists : {}".format(filePath))
+            return True
+    failures = 0
+    while True:
+        tries=tries-1
+        if failures == tries:
+            try:
+                os.remove(tempPath)
+            except:
+                pass
+        try:
+            socket.setdefaulttimeout(timeout)
+            urllib.request.urlretrieve(url, tempPath)
+            if sha256:
+                hash_obj = hashlib.sha256()
+                with open(tempPath, 'rb') as f:
+                    hash_obj.update(f.read())
+                if hash_obj.hexdigest() == sha256:
+                    status=True
+                    break
+            else:
+                status=True
+                break
+        except urllib.error.HTTPError:
+            log.error("HTTP Error")
+        except urllib.error.URLError:
+            time.sleep(sleepBetween)
+        except TimeoutError:
+            pass
+        except socket.timeout:
+            pass
+    return status
+def extract_zip (file,path):
+    #extract and return file names from zip file
+    try:
+        with zipfile.ZipFile(file, 'r') as zip_ref:
+            zip_ref.extractall(path)
+            names=zip_ref.namelist()
+        return names
+    except Exception as e:
+        log.error(e)
+        
+def download_firmware_to_repository(version,q,arch="all",all_package=False):
+    #repository='/app/firms/'
+    repository=config.FIRM_DIR
+    #create direcorty version in repository if not exist
+    path=repository+version+"/"
+    os.makedirs(path, exist_ok=True)
+    # try:
+    if all_package:
+        #download all_packages
+        links=get_mikrotik_download_links(version,all_package=all_package)
+    else:
+        links=get_mikrotik_download_links(version)
+    if links:
+        links=links[version]
+        firm=db_firmware.Firmware()
+        for lnk in links:
+            if all_package and arch+"-allpackage" == lnk:
+                arch_togo=lnk
+                link=links[lnk]["link"]
+                sha256=links[lnk]["sha"]
+                file=path+"all_packages-" + arch + ".zip"
+                log.error(link)
+                done=web2file(link, file, sha256=sha256)
+                files=extract_zip(file, path)
+                try:
+                    if done and len(files)>0:
+                        for f in files:
+                            file=path+f
+                            log.error(file)
+                            sha256=check_sha256(file)
+                            firm.insert(version=version, location=file, architecture=arch+"-"+f.split("-")[0], sha256=sha256).on_conflict(conflict_target=['version', 'architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
+                except Exception as e:
+                    log.error(e)
+                    pass
+                if q:
+                    q.put({"status":True})
+                # return True
+            if arch!="all" and arch==lnk:
+                arch_togo=lnk
+                link=links[lnk]["link"]
+                sha256=links[lnk]["sha"]
+                file=path+"{}.npk".format(arch)
+                done=web2file(link, file,sha256=sha256)
+                try:
+                    if done:
+                        firm.insert(version=version, location=file, architecture=arch_togo, sha256=sha256).on_conflict(conflict_target=['version','architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
+                except Exception as e:
+                    log.error(e)
+                    pass
+                if q:
+                    q.put({"status":True})
+                # return True
+            if arch=="all":
+                #download file to path and check sha265 
+                arch_togo=lnk
+                link=links[lnk]["link"]
+                sha256=links[lnk]["sha"]
+                file=path+"{}.npk".format(arch)
+                done=web2file(link, file,sha256=sha256)
+                try:
+                    if done:
+                        firm.insert(version=version, location=file, architecture=arch_togo, sha256=sha256).on_conflict(conflict_target=['version','architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
+                except Exception as e:
+                    log.error(e)
+                    pass
+        if q:
+            q.put({"status":True})
+        return True
+    else:
+        if q:
+            q.put({"status":False})
+        return False
+    # except Exception as e:
+    #     log.error(e)
+    #     if q:
+    #         q.put({"status":True})
+    #     return False
+
+
+def update_device(dev,q):
+    events=list(db_events.get_events_by_src_and_status("updater", 0,dev.id).dicts())
+    ofa=db_sysconfig.get_firmware_action().value
+    _installed_version=RouterOSVersion(dev.current_firmware)
+    try:
+        if dev.firmware_to_install:
+            ver_to_install=dev.firmware_to_install
+        elif ofa=="keep" and _installed_version < RouterOSVersion('7.0.0'):
+            ver_to_install=db_sysconfig.get_firmware_old().value
+        else:
+            ver_to_install=db_sysconfig.get_firmware_latest().value
+        ver_to_install = RouterOSVersion(ver_to_install)
+    except Exception as e:
+        log.error(e)
+        q.put({"id": dev.id})
+        return False
+    arch=dev.arch
+    if not dev.firmware_to_install or RouterOSVersion(dev.firmware_to_install)!=ver_to_install:
+        dev.firmware_to_install=ver_to_install
+        dev.save()
+    try:
+        if _installed_version==ver_to_install:
+            util.check_or_fix_event(events,"firmware","Update Failed")
+            util.check_or_fix_event(events,"firmware","Firmware repositpry")
+            util.check_or_fix_event(events,"firmware","Device storage")
+            dev.failed_attempt=0
+            dev.firmware_to_install=None
+            dev.save()
+            q.put({"id": dev.id})
+            return True
+    except Exception as e:
+        log.error(e)
+        pass
+    #get correct firmware from db for updating
+    firm=False
+    if ISPRO:
+        firm=utilpro.safe_check(dev,_installed_version,ver_to_install)
+    elif arch and arch!='':
+        firm=db_firmware.get_frim_by_version(ver_to_install, arch)
+    else:
+        q.put({"id": dev.id})
+    if firm and firm.architecture == arch:
+        download_firmware_to_repository(str(ver_to_install), False,arch=arch,all_package=False)
+        dev.failed_attempt=dev.failed_attempt+1
+        if dev.failed_attempt > 3:
+            db_events.firmware_event(dev.id,"updater","Update Failed","Critical",0,"Unable to Update device")
+        dev.status="updating"
+        dev.save()
+        options=util.build_api_options(dev)
+        try:
+            url=db_sysconfig.get_sysconfig('system_url')
+            url=url+"/api/firmware/get_firmware/{}".format(firm.id)
+            router=RouterOSCheckResource(options)
+            api = router._connect_api()
+            params = {"url": url,"keep-result":"yes","dst-path":arch+".npk"}
+            cmd='/tool/fetch'
+            call = api(cmd,**params)
+            results = tuple(call)
+            result: Dict[str, str] = results[-1]
+            if result['status'] == 'finished':
+                util.check_or_fix_event(events,"firmware","Device storage")
+                cmd='/system/reboot'
+                call = api(cmd)
+                rebootresults = tuple(call)
+                if len(rebootresults)==0:
+                    util.check_or_fix_event(events,"firmware","Firmware repositpry")
+                    dev.status="updated"
+                    dev.save()
+                else:
+                    dev.status="failed"
+                    dev.save()
+            else:
+                db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"There is a problem with downloadin of Firmware in device")
+                dev.status="failed"
+                dev.save()
+        except Exception as e:
+            dev.status="failed"
+            dev.save()
+            if 'no space left' in str(e):
+                db_events.firmware_event(dev.id,"updater","Device storage","Error",0,"There is not enogh space in device storage")
+            if '404 Not Found' in str(e):
+                db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #1 :Please check firmware config in settings section")
+            log.error(e)
+            q.put({"id": dev.id})
+    else:
+        db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #2 :Please check firmware config in settings section")
+        log.error('No Firmware found for device {}({})'.format(dev.name,dev.ip))
+    q.put({"id": dev.id})

py/libs/util.py

@@ -27,6 +27,7 @@ import urllib.request
 import hashlib
 import netifaces
 log = logging.getLogger("util")
+from libs import utilpro
 try:
     from libs import utilpro
     ISPRO=True
@@ -51,24 +52,6 @@ def utc2local(utc_dt, tz=tz_hki):
     d = utc_dt.replace(tzinfo=tz_utc)
     return d.astimezone(tz)
 
-def local2utc(local_dt, tz=tz_hki):
-    """Convert local time into UTC."""
-
-    if not local_dt:
-        return local_dt
-
-    d = local_dt.replace(tzinfo=tz)
-    return d.astimezone(tz_utc)
-
-def utcnow():
-    """Return UTC now."""
-    return datetime.datetime.utcnow()
-
-def generate_token():
-    """Generate a random token
-    (an uuid like 8491997531e44d37ac3105b300774e08)"""
-    return uuid.uuid4().hex
-
 def check_port(ip,port):
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     sock.settimeout(0.5)
@@ -86,7 +69,6 @@ def crypt_data(text):
     encrypted_password = cipher_suite.encrypt(text.encode()).decode() 
     return encrypted_password
 
-
 def decrypt_data(text):
     # Encryption: Decrypting password using Fernet symmetric encryption 
     cipher_suite = Fernet(config.CRYPT_KEY) 
@@ -312,9 +294,6 @@ def grab_device_data(dev, q):
             q.put({"id": dev.id,"reason":"Could not health data from device","detail":"Get Health","done":False})
             return True
         check_or_fix_event(events,"connection","Get Health")
-        # ToDo remove keys without messurable value
-        # keys.remove('fan-switch')
-        # keys.remove('fan-on-threshold')
         try:
             # arch=result['architecture-name']
             try:
@@ -743,327 +722,6 @@ def store_config(dev,configs):
         log_alert('backup',dev,'Problem During backup when saving file')
         return False
 
-def extract_from_link(link,all_package=False):
-    if all_package:
-        regex = r"https:\/\/download\.mikrotik\.com\/routeros\/(\d{1,3}.*)?\/all_packages-(.*)-(.*).zip"
-        matches = re.match(regex, link)
-        if not matches:
-            return False
-        res=matches.groups()
-        version=res[0]
-        arch = res[1]
-        return {"link":link, "arch":arch, "version":version, "all_package":True}
-    else:
-        regex = r"https:\/\/download\.mikrotik\.com\/routeros\/(\d{1,3}.*)?\/routeros-(.*).npk"
-        matches = re.match(regex,link)
-        res=matches.groups()
-        version=res[0]
-        arch = res[1].replace(version, "")
-        if arch == "":
-            arch = "x86"
-        else:
-            arch=arch.replace("-","")
-        return {"link":link,"arch":arch, "version":version}
-    
-
-def get_mikrotik_latest_firmware_link():
-    try:
-        html_page = urllib.request.urlopen("https://mikrotik.com/download/")
-        soup = BeautifulSoup(html_page, "html.parser")
-        firms={}
-        for link in soup.findAll('a'):
-            link=str(link.get('href'))
-            if ".npk" in link:
-                frimware=extract_from_link(link)
-                firms.setdefault(frimware["version"],{})
-                firms[frimware["version"]][frimware["arch"]]={"link":frimware["link"],"mark":"latest"}
-                # firms.append(link)
-        return firms
-    except Exception as e:
-        log.error(e)
-        return False
-
-def get_mikrotik_download_links(version,all_package=False):
-    try:
-        html_page = urllib.request.urlopen("https://mikrotik.com/download/archive?v={}".format(version))
-        soup = BeautifulSoup(html_page, "html.parser")
-        firms={}
-        for trs in soup.findAll('tr'):
-            link=trs.findAll('a')
-            if len(link):
-                lnk=str(link[0].get('href'))
-                sha=str(link[1].get('data-checksum-sha256'))
-                if ".npk" in lnk:
-                    frimware=extract_from_link(lnk)
-                    firms.setdefault(frimware["version"], {})
-                    firms[frimware["version"]][frimware["arch"]]={"link":frimware["link"],"sha":sha}
-                    # firms.append(link)
-                elif all_package and ".zip" in lnk:
-                    frimware=extract_from_link(lnk, all_package=all_package)
-                    if not frimware:
-                        continue
-                    firms.setdefault(frimware["version"], {})
-                    firms[frimware["version"]][frimware["arch"]+"-"+"allpackage"]={"link":frimware["link"],"sha":sha}
-        return firms
-    except Exception as e:
-        log.error(e)
-        return False    
-
-def get_mikrotik_versions():
-    try:
-        html_page = urllib.request.urlopen("https://mikrotik.com/download/archive")
-        soup = BeautifulSoup(html_page, "html.parser")
-        versions=[]
-        for link in soup.findAll('a'):
-            ver=link.find("strong")
-            if ver:
-                versions.append(ver.text)
-        try:
-            vers=list(get_mikrotik_latest_firmware_link().keys())
-            if versions and vers:
-                unique_elements = set(versions + vers)
-                versions = list(unique_elements)
-            elif not versions and vers:
-                if vers:
-                    versions = vers
-        except Exception as e:
-            log.error(e)
-            pass
-        return versions
-    except Exception as e:
-        log.error(e)
-        return False
-
-def check_sha256(path,sha256=False):
-    hash_obj = hashlib.sha256()
-    if not sha256 and os.path.exists(path):
-        with open(path, 'rb') as f:
-            hash_obj.update(f.read())
-        return hash_obj.hexdigest()
-    elif os.path.exists(path) and sha256:
-        with open(path, 'rb') as f:
-            hash_obj.update(f.read())
-        return hash_obj.hexdigest() == sha256
-    else:
-        return False 
-
-def web2file(url, filePath,sha256=False, tries=3, timeout=3, sleepBetween=1):
-    tempPath = filePath
-    status=False
-    if os.path.exists(tempPath) and sha256:
-        hash_obj = hashlib.sha256()
-        with open(tempPath, 'rb') as f:
-            hash_obj.update(f.read())
-        if hash_obj.hexdigest() == sha256:
-            log.error("File already exists : {}".format(filePath))
-            return True
-    failures = 0
-    while True:
-        tries=tries-1
-        if failures == tries:
-            try:
-                os.remove(tempPath)
-            except:
-                pass
-        try:
-            socket.setdefaulttimeout(timeout)
-            urllib.request.urlretrieve(url, tempPath)
-            if sha256:
-                hash_obj = hashlib.sha256()
-                with open(tempPath, 'rb') as f:
-                    hash_obj.update(f.read())
-                if hash_obj.hexdigest() == sha256:
-                    status=True
-                    break
-            else:
-                status=True
-                break
-        except urllib.error.HTTPError:
-            log.error("HTTP Error")
-        except urllib.error.URLError:
-            time.sleep(sleepBetween)
-        except TimeoutError:
-            pass
-        except socket.timeout:
-            pass
-    return status
-def extract_zip (file,path):
-    #extract and return file names from zip file
-    try:
-        with zipfile.ZipFile(file, 'r') as zip_ref:
-            zip_ref.extractall(path)
-            names=zip_ref.namelist()
-        return names
-    except Exception as e:
-        log.error(e)
-        
-def download_firmware_to_repository(version,q,arch="all",all_package=False):
-    #repository='/app/firms/'
-    repository=config.FIRM_DIR
-    #create direcorty version in repository if not exist
-    path=repository+version+"/"
-    os.makedirs(path, exist_ok=True)
-    # try:
-    if all_package:
-        #download all_packages
-        links=get_mikrotik_download_links(version,all_package=all_package)
-    else:
-        links=get_mikrotik_download_links(version)
-    if links:
-        links=links[version]
-        firm=db_firmware.Firmware()
-        for lnk in links:
-            if all_package and arch+"-allpackage" == lnk:
-                arch_togo=lnk
-                link=links[lnk]["link"]
-                sha256=links[lnk]["sha"]
-                file=path+"all_packages-" + arch + ".zip"
-                log.error(link)
-                done=web2file(link, file, sha256=sha256)
-                files=extract_zip(file, path)
-                try:
-                    if done and len(files)>0:
-                        for f in files:
-                            file=path+f
-                            log.error(file)
-                            sha256=check_sha256(file)
-                            firm.insert(version=version, location=file, architecture=arch+"-"+f.split("-")[0], sha256=sha256).on_conflict(conflict_target=['version', 'architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
-                except Exception as e:
-                    log.error(e)
-                    pass
-                if q:
-                    q.put({"status":True})
-                # return True
-            if arch!="all" and arch==lnk:
-                arch_togo=lnk
-                link=links[lnk]["link"]
-                sha256=links[lnk]["sha"]
-                file=path+"{}.npk".format(arch)
-                done=web2file(link, file,sha256=sha256)
-                try:
-                    if done:
-                        firm.insert(version=version, location=file, architecture=arch_togo, sha256=sha256).on_conflict(conflict_target=['version','architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
-                except Exception as e:
-                    log.error(e)
-                    pass
-                if q:
-                    q.put({"status":True})
-                # return True
-            if arch=="all":
-                #download file to path and check sha265 
-                arch_togo=lnk
-                link=links[lnk]["link"]
-                sha256=links[lnk]["sha"]
-                file=path+"{}.npk".format(arch)
-                done=web2file(link, file,sha256=sha256)
-                try:
-                    if done:
-                        firm.insert(version=version, location=file, architecture=arch_togo, sha256=sha256).on_conflict(conflict_target=['version','architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() 
-                except Exception as e:
-                    log.error(e)
-                    pass
-        if q:
-            q.put({"status":True})
-        return True
-    else:
-        if q:
-            q.put({"status":False})
-        return False
-    # except Exception as e:
-    #     log.error(e)
-    #     if q:
-    #         q.put({"status":True})
-    #     return False
-
-
-def update_device(dev,q):
-    events=list(db_events.get_events_by_src_and_status("updater", 0,dev.id).dicts())
-    ofa=db_sysconfig.get_firmware_action().value
-    _installed_version=RouterOSVersion(dev.current_firmware)
-    try:
-        if dev.firmware_to_install:
-            ver_to_install=dev.firmware_to_install
-        elif ofa=="keep" and _installed_version < RouterOSVersion('7.0.0'):
-            ver_to_install=db_sysconfig.get_firmware_old().value
-        else:
-            ver_to_install=db_sysconfig.get_firmware_latest().value
-        ver_to_install = RouterOSVersion(ver_to_install)
-    except Exception as e:
-        log.error(e)
-        q.put({"id": dev.id})
-        return False
-    arch=dev.arch
-    if not dev.firmware_to_install or RouterOSVersion(dev.firmware_to_install)!=ver_to_install:
-        dev.firmware_to_install=ver_to_install
-        dev.save()
-    try:
-        if _installed_version==ver_to_install:
-            check_or_fix_event(events,"firmware","Update Failed")
-            check_or_fix_event(events,"firmware","Firmware repositpry")
-            check_or_fix_event(events,"firmware","Device storage")
-            dev.failed_attempt=0
-            dev.firmware_to_install=None
-            dev.save()
-            q.put({"id": dev.id})
-            return True
-    except Exception as e:
-        log.error(e)
-        pass
-    #get correct firmware from db for updating
-    firm=False
-    if ISPRO:
-        firm=utilpro.safe_check(dev,_installed_version,ver_to_install)
-    elif arch and arch!='':
-        firm=db_firmware.get_frim_by_version(ver_to_install, arch)
-    else:
-        q.put({"id": dev.id})
-    if firm and firm.architecture == arch:
-        download_firmware_to_repository(str(ver_to_install), False,arch=arch,all_package=False)
-        dev.failed_attempt=dev.failed_attempt+1
-        if dev.failed_attempt > 3:
-            db_events.firmware_event(dev.id,"updater","Update Failed","Critical",0,"Unable to Update device")
-        dev.status="updating"
-        dev.save()
-        options=build_api_options(dev)
-        try:
-            url=db_sysconfig.get_sysconfig('system_url')
-            url=url+"/api/firmware/get_firmware/{}".format(firm.id)
-            router=RouterOSCheckResource(options)
-            api = router._connect_api()
-            params = {"url": url,"keep-result":"yes","dst-path":arch+".npk"}
-            cmd='/tool/fetch'
-            call = api(cmd,**params)
-            results = tuple(call)
-            result: Dict[str, str] = results[-1]
-            if result['status'] == 'finished':
-                check_or_fix_event(events,"firmware","Device storage")
-                cmd='/system/reboot'
-                call = api(cmd)
-                rebootresults = tuple(call)
-                if len(rebootresults)==0:
-                    check_or_fix_event(events,"firmware","Firmware repositpry")
-                    dev.status="updated"
-                    dev.save()
-                else:
-                    dev.status="failed"
-                    dev.save()
-            else:
-                db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"There is a problem with downloadin of Firmware in device")
-                dev.status="failed"
-                dev.save()
-        except Exception as e:
-            dev.status="failed"
-            dev.save()
-            if 'no space left' in str(e):
-                db_events.firmware_event(dev.id,"updater","Device storage","Error",0,"There is not enogh space in device storage")
-            if '404 Not Found' in str(e):
-                db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #1 :Please check firmware config in settings section")
-            log.error(e)
-            q.put({"id": dev.id})
-    else:
-        db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #2 :Please check firmware config in settings section")
-        log.error('No Firmware found for device {}({})'.format(dev.name,dev.ip))
-    q.put({"id": dev.id})
 
 def get_ethernet_wifi_interfaces():
     interfaces = netifaces.interfaces()
@@ -1122,11 +780,19 @@ def get_local_users(opts,router=False,full=False):
         log.error(e)
         return False
 
+def delete_file(file):
+    try:
+        #check if file exist:
+        if not os.path.exists(file):
+            return True
+        os.remove(file)
+        return True
+    except Exception as e:
+        log.error(e)
+        return False
 def ispro():
     return ISPRO
 
 if __name__ == '__main__':
-
-    # quick adhoc tests
     logging.basicConfig(level=logging.DEBUG)
 

py/mules/firmware.py

@@ -6,7 +6,7 @@
 # Author: sepehr.ha@gmail.com
 
 import time
-from libs import util
+from libs import util,firm_lib
 from libs.db import db_tasks,db_device
 import logging
 import queue
@@ -34,7 +34,7 @@ def updater():
                 if ISPRO:
                     t = Thread(target=utilpro.update_device, args=(dev,{"version_to_install":dev.firmware_to_install},False, q))
                 else:
-                    t = Thread(target=util.update_device, args=(dev, q))
+                    t = Thread(target=firm_lib.update_device, args=(dev, q))
                 t.start()
                 threads.append(t)
             for t in threads:

py/mules/updater.py

@@ -33,7 +33,6 @@ def install_package(package):
 
 
 def set_get_install_date():
-    
     install_date=False
     try:
         install_date=db_sysconfig.get_sysconfig('install_date')
@@ -43,7 +42,6 @@ def set_get_install_date():
         install_date=datetime.datetime.now()
         db_sysconfig.set_sysconfig('install_date',install_date.strftime("%Y-%m-%d %H:%M:%S"))
         install_date=install_date.strftime("%Y-%m-%d %H:%M:%S")
-    
     return install_date
 
 # Example usage
@@ -124,8 +122,6 @@ def main():
             username = db_sysconfig.get_sysconfig('username')
         except:
             log.error("No username found")
-            time.sleep(300)
-            continue
         # util.send_mikrowizard_request(params)
         if not username or username.strip()=="":
             log.error("No username found")