From c669b762ae95bb334f42c189a92e8c389a928d71 Mon Sep 17 00:00:00 2001 From: sepehr Date: Fri, 10 Jan 2025 18:00:26 +0300 Subject: [PATCH] FIX BUGS: --Firmware and packages download is failing for wrong arch name in code --Fix CHR firmware updates --Fix #5 New Feature: --Check all packages and include all packages to also update with firmware --- py/libs/firm_lib.py | 154 ++++++++++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 55 deletions(-) diff --git a/py/libs/firm_lib.py b/py/libs/firm_lib.py index 34e60c6..f8c4cd5 100644 --- a/py/libs/firm_lib.py +++ b/py/libs/firm_lib.py @@ -8,24 +8,19 @@ import pytz import datetime import time -import uuid import socket import config from libs.db import db_sysconfig,db_firmware,db_tasks,db_events -from cryptography.fernet import Fernet from libs.check_routeros.routeros_check.resource import RouterOSCheckResource from libs.check_routeros.routeros_check.helper import RouterOSVersion from typing import Dict import re import json import logging -from libs.red import RedisDB -from libs.ssh_helper import SSH_Helper import os from bs4 import BeautifulSoup import urllib.request import hashlib -import netifaces log = logging.getLogger("util") from libs import util try: @@ -85,6 +80,7 @@ def get_mikrotik_latest_firmware_link(): def get_mikrotik_download_links(version,all_package=False): try: + log.info("Downloading firmwares from https://mikrotik.com/download/archive?v={}".format(version)) html_page = urllib.request.urlopen("https://mikrotik.com/download/archive?v={}".format(version)) soup = BeautifulSoup(html_page, "html.parser") firms={} @@ -200,7 +196,7 @@ def extract_zip (file,path): except Exception as e: log.error(e) -def download_firmware_to_repository(version,q,arch="all",all_package=False): +def download_firmware_to_repository(version,q,arch="all",all_package=True): #repository='/app/firms/' repository=config.FIRM_DIR #create direcorty version in repository if not exist @@ -222,32 +218,31 @@ def download_firmware_to_repository(version,q,arch="all",all_package=False): if q: q.put({"status":False}) return False - if all_package and arch+"-allpackage" == lnk: - arch_togo=lnk + if all_package and "-allpackage" in lnk == lnk: + arch_togo=lnk.split("-allpackage")[0] link=links[lnk]["link"] sha256=links[lnk]["sha"] - file=path+"all_packages-" + arch + ".zip" - log.error(link) + file=path+"all_packages-" + arch_togo + ".zip" done=web2file(link, file, sha256=sha256) files=extract_zip(file, path) try: if done and len(files)>0: for f in files: file=path+f - log.error(file) sha256=check_sha256(file) - firm.insert(version=version, location=file, architecture=arch+"-"+f.split("-")[0], sha256=sha256).on_conflict(conflict_target=['version', 'architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() + firm.insert(version=version, location=file, architecture=arch_togo+"-"+f.split("-{}".format(version))[0], sha256=sha256).on_conflict(conflict_target=['version', 'architecture'], preserve=['location', 'architecture', 'version'], update={'sha256':sha256}).execute() except Exception as e: log.error(e) pass if q: q.put({"status":True}) # return True + continue if arch!="all" and arch==lnk: arch_togo=lnk link=links[lnk]["link"] sha256=links[lnk]["sha"] - file=path+"{}.npk".format(arch) + file=path+"{}.npk".format(arch_togo) done=web2file(link, file,sha256=sha256) try: if done: @@ -257,13 +252,14 @@ def download_firmware_to_repository(version,q,arch="all",all_package=False): pass if q: q.put({"status":True}) + continue # return True if arch=="all": #download file to path and check sha265 arch_togo=lnk link=links[lnk]["link"] sha256=links[lnk]["sha"] - file=path+"{}.npk".format(arch) + file=path+"{}.npk".format(arch_togo) done=web2file(link, file,sha256=sha256) try: if done: @@ -302,6 +298,8 @@ def update_device(dev,q): q.put({"id": dev.id}) return False arch=dev.arch + if "x86" in arch: + arch="x86" if not dev.firmware_to_install or RouterOSVersion(dev.firmware_to_install)!=ver_to_install: dev.firmware_to_install=ver_to_install dev.save() @@ -321,55 +319,101 @@ def update_device(dev,q): #get correct firmware from db for updating firm=False if ISPRO: - firm=utilpro.safe_check(dev,_installed_version,ver_to_install) + firm,firm2=utilpro.safe_check(dev,_installed_version,ver_to_install) elif arch and arch!='': + log.warning(ver_to_install) + log.warning(arch) firm=db_firmware.get_frim_by_version(ver_to_install, arch) else: q.put({"id": dev.id}) - if firm and firm.architecture == arch: - download_firmware_to_repository(str(ver_to_install), False,arch=arch,all_package=False) - dev.failed_attempt=dev.failed_attempt+1 - if dev.failed_attempt > 3: - db_events.firmware_event(dev.id,"updater","Update Failed","Critical",0,"Unable to Update device") - dev.status="updating" - dev.save() - options=util.build_api_options(dev) - try: + options=util.build_api_options(dev) + #get /system package print + router=RouterOSCheckResource(options) + try: + call=router.api.path('/system/package') + results = tuple(call) + except: + q.put({"id": dev.id}) + return False + packages=[] + if firm: + packages.append(firm) + else: + db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #2 :Please check firmware config in settings section") + log.error('No Firmware found for device {}({})'.format(dev.name,dev.ip)) + q.put({"id": dev.id}) + return False + + for res in results: + log.error(res['name']) + if res['name']!="routeros": + package=db_firmware.get_frim_by_version(ver_to_install, "{}-{}".format(arch,res['name'])) + if package: + packages.append(package) + log.error(packages) + # q.put({"id": dev.id}) + # return False + + try: + apply_firmware(packages, firm2, arch, dev, router, events, q) + except: + q.put({"id": dev.id}) + +def apply_firmware(packages,firm2,arch,dev,router,events,q): + dev.failed_attempt=dev.failed_attempt+1 + if dev.failed_attempt > 3: + db_events.firmware_event(dev.id,"updater","Update Failed","Critical",0,"Unable to Update device") + dev.status="updating" + dev.save() + try: + url=dev.peer_ip + api = router._connect_api() + if not url: url=db_sysconfig.get_sysconfig('system_url') - url=url+"/api/firmware/get_firmware/{}".format(firm.id) - router=RouterOSCheckResource(options) - api = router._connect_api() - params = {"url": url,"keep-result":"yes","dst-path":arch+".npk"} + if not "http" in url: + url="http://"+url + if firm2: + url_firm2=url+"/api/firmware/get_firmware/{}".format(firm2.id) + params = {"url": url_firm2,"keep-result":"yes","dst-path":firm2.architecture+".npk"} cmd='/tool/fetch' call = api(cmd,**params) results = tuple(call) result: Dict[str, str] = results[-1] - if result['status'] == 'finished': - util.check_or_fix_event(events,"firmware","Device storage") - cmd='/system/reboot' - call = api(cmd) - rebootresults = tuple(call) - if len(rebootresults)==0: - util.check_or_fix_event(events,"firmware","Firmware repositpry") - dev.status="updated" - dev.save() - else: - dev.status="failed" - dev.save() - else: - db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"There is a problem with downloadin of Firmware in device") + if result['status'] != 'finished': dev.status="failed" dev.save() - except Exception as e: - dev.status="failed" - dev.save() - if 'no space left' in str(e): - db_events.firmware_event(dev.id,"updater","Device storage","Error",0,"There is not enogh space in device storage") - if '404 Not Found' in str(e): - db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #1 :Please check firmware config in settings section") - log.error(e) - q.put({"id": dev.id}) - else: - db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #2 :Please check firmware config in settings section") - log.error('No Firmware found for device {}({})'.format(dev.name,dev.ip)) - q.put({"id": dev.id}) \ No newline at end of file + q.put({"id": dev.id}) + return False + for package in packages: + url_package=url+"/api/firmware/get_firmware/{}".format(package.id) + params = {"url": url_package,"keep-result":"yes","dst-path":package.architecture+".npk"} + cmd='/tool/fetch' + call = api(cmd, **params) + results = tuple(call) + log.warning(results) + result: Dict[str, str] = results[-1] + if result['status'] != 'finished': + log.error("There is a problem with downloading of Firmware in device") + dev.status="failed" + dev.save() + db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"There is a problem with downloading of Firmware in device") + q.put({"id": dev.id}) + return False + util.check_or_fix_event(events,"firmware","Device storage") + cmd='/system/reboot' + call = api(cmd) + rebootresults = tuple(call) + log.warning(rebootresults) + util.check_or_fix_event(events,"firmware","Firmware repositpry") + dev.status="updated" + dev.save() + except Exception as e: + dev.status="failed" + dev.save() + if 'no space left' in str(e): + db_events.firmware_event(dev.id,"updater","Device storage","Error",0,"There is not enogh space in device storage") + if '404 Not Found' in str(e): + db_events.firmware_event(dev.id,"updater","Firmware repositpry","Error",0,"Firmware not found #1 :Please check firmware config in settings section") + log.error(e) + q.put({"id": dev.id}) + q.put({"id": dev.id})