Host monitoring

.gitignore

@@ -7,4 +7,5 @@
 routerfleet/production_settings.py
 .idea/
 db.sqlite3
-backups/
+backups/
+containers/*/.venv

containers/monitoring/monitoring.py

@@ -0,0 +1,98 @@
+import requests
+import time
+from datetime import datetime
+from subprocess import Popen, PIPE
+
+
+UPDATE_HOST_LIST_INTERVAL = 600  # How often to update the router list in seconds
+MONITOR_INTERVAL = 60  # How often to monitor each router in seconds
+MAX_NOTIFICATIONS_PER_MONITOR_INTERVAL = 50  # Throttle the number of notifications sent to the remote API
+HOST_LIST_URL = "http://127.0.0.1:8000/monitoring/export_router_list/"
+UPDATE_STATUS_URL = "http://127.0.0.1:8000/monitoring/update_router_status/"
+DEBUG = False
+
+# Global variables
+host_list = []
+host_list_update_timestamp = 0
+notification_count = 0
+
+
+def get_verbose_status(status):
+    return "online" if status else "offline"
+
+
+def fetch_host_list():
+    global host_list_update_timestamp
+    try:
+        print(f"{datetime.now()} - Fetching host list...")
+        response = requests.get(HOST_LIST_URL)
+        if response.status_code == 200:
+            host_list_update_timestamp = time.time()
+            return response.json()['router_list'], True
+        else:
+            print(f"{datetime.now()} - Error fetching host list: HTTP {response.status_code}")
+    except Exception as e:
+        print(f"{datetime.now()} - Exception fetching host list: {e}")
+    return [], False
+
+
+def update_host_status(uuid, status):
+    global notification_count
+    if notification_count >= MAX_NOTIFICATIONS_PER_MONITOR_INTERVAL:
+        print(f"{datetime.now()} - Notification limit reached. Skipping Remote API update for {host_list[uuid]['address']}")
+        return  # Skip if notification limit is reached
+    try:
+        response = requests.get(f"{UPDATE_STATUS_URL}?uuid={uuid}&status={get_verbose_status(status)}")
+        if response.status_code == 200:
+            print(f"{datetime.now()} - Remote API Status updated for {host_list[uuid]['address']} to {get_verbose_status(status)}")
+            notification_count += 1
+            host_list[uuid]['online'] = status
+        else:
+            print(f"{datetime.now()} - Error updating status for {host_list[uuid]['address']}: HTTP {response.status_code}")
+    except Exception as e:
+        print(f"{datetime.now()} - Exception updating status for {host_list[uuid]['address']}: {e}")
+
+
+def check_host_status(host_uuid):
+    command = ["fping", host_list[host_uuid]['address']]
+    process = Popen(command, stdout=PIPE, stderr=PIPE)
+    stdout, stderr = process.communicate()
+    current_online = True if process.returncode == 0 else False
+    if DEBUG:
+        print(f"{datetime.now()} - {host_list[host_uuid]['address']} is {get_verbose_status(current_online)}")
+    if current_online != host_list[host_uuid]['online']:
+        print(f"{datetime.now()} - Status changed for {host_list[host_uuid]['address']} to {get_verbose_status(current_online)}")
+        update_host_status(host_uuid, current_online)
+
+
+def update_and_monitor():
+    global host_list, host_list_update_timestamp, notification_count
+    while True:
+        current_time = time.time()
+        notification_count = 0
+
+        if current_time - host_list_update_timestamp > UPDATE_HOST_LIST_INTERVAL:
+            new_host_list, fetch_host_list_success = fetch_host_list()
+            if fetch_host_list_success:
+                host_list = new_host_list
+                print(f"{datetime.now()} - host list updated.")
+                if DEBUG:
+                    print(host_list)
+
+        if host_list:
+            if DEBUG:
+                print(f"{datetime.now()} - Monitoring host... Interval between each monitor: {MONITOR_INTERVAL / len(host_list)} seconds")
+            for host_uuid in host_list:
+                if DEBUG:
+                    print(host_list[host_uuid])
+                check_host_status(host_uuid)
+                time.sleep(MONITOR_INTERVAL / len(host_list))
+        else:
+            print(f"{datetime.now()} - No host to monitor.")
+            time.sleep(MONITOR_INTERVAL)
+
+
+if __name__ == "__main__":
+    update_and_monitor()
+
+

containers/monitoring/requirements.txt

@@ -0,0 +1,5 @@
+certifi==2024.2.2
+charset-normalizer==3.3.2
+idna==3.6
+requests==2.31.0
+urllib3==2.2.1

monitoring/views.py

@@ -1,3 +1,31 @@
 from django.shortcuts import render
+from router_manager.models import Router
+from django.http import JsonResponse
 
-# Create your views here.
+
+def view_export_router_list(request):
+    router_list = {}
+    for router in Router.objects.filter(enabled=True, monitoring=True):
+        router_list[str(router.uuid)] = {
+            'address': router.address,
+            'name': router.name,
+            'online': router.routerstatus.status_online,
+            'uuid': str(router.uuid),
+        }
+    data = {
+        'router_list': router_list
+    }
+    return JsonResponse(data)
+
+
+def view_update_router_status(request):
+    router = Router.objects.get(uuid=request.GET.get('uuid'))
+    new_status = request.GET.get('status')
+    if new_status not in ['online', 'offline']:
+        return JsonResponse({'status': 'error', 'error_message': 'Invalid status'}, status=400)
+    if new_status == 'online':
+        router.routerstatus.status_online = True
+    else:
+        router.routerstatus.status_online = False
+    router.routerstatus.save()
+    return JsonResponse({'status': 'success'})

requirements.txt

@@ -1,14 +1,19 @@
 asgiref==3.7.2
 bcrypt==4.1.2
+certifi==2024.2.2
 cffi==1.16.0
+charset-normalizer==3.3.2
 crispy-bootstrap4==2024.1
 crispy-bootstrap5==2024.2
 cryptography==42.0.5
 Django==5.0.3
 django-crispy-forms==2.1
+idna==3.6
 paramiko==3.4.0
 pycparser==2.21
 PyNaCl==1.5.0
+requests==2.31.0
 scp==0.14.5
 sqlparse==0.4.4
 typing_extensions==4.10.0
+urllib3==2.2.1

router_manager/forms.py

@@ -54,24 +54,16 @@ class RouterForm(forms.ModelForm):
         cleaned_data = super().clean()
         name = cleaned_data.get('name')
         ssh_key = cleaned_data.get('ssh_key')
+        username = cleaned_data.get('username')
         password = cleaned_data.get('password')
         address = cleaned_data.get('address')
+        router_type = cleaned_data.get('router_type')
+        backup_profile = cleaned_data.get('backup_profile')
 
         if name:
             name = name.strip()
             cleaned_data['name'] = name
 
-        if ssh_key and password:
-            raise forms.ValidationError('You must provide a password or an SSH Key, not both')
-        if not ssh_key and not password and not self.instance.password:
-            raise forms.ValidationError('You must provide a password or an SSH Key')
-
-        if not password and self.instance.password:
-            cleaned_data['password'] = self.instance.password
-
-        if ssh_key and not password:
-            cleaned_data['password'] = ''
-
         if address:
             address = address.lower()
             cleaned_data['address'] = address
@@ -83,9 +75,27 @@ class RouterForm(forms.ModelForm):
                     ipaddress.ip_address(address)
                 except ValueError:
                     raise forms.ValidationError('The address field must be a valid hostname or IP address.')
+
+        if router_type == 'monitoring':
+            cleaned_data['password'] = ''
+            cleaned_data['ssh_key'] = None
+            if backup_profile:
+                raise forms.ValidationError('Monitoring only routers cannot have a backup profile')
+            return cleaned_data
+
+        if ssh_key and password:
+            raise forms.ValidationError('You must provide a password or an SSH Key, not both')
+        if not ssh_key and not password and not self.instance.password:
+            raise forms.ValidationError('You must provide a password or an SSH Key')
+
+        if not password and self.instance.password:
+            cleaned_data['password'] = self.instance.password
+
+        if ssh_key and not password:
+            cleaned_data['password'] = ''
+
         test_authentication_success, test_authentication_message = test_authentication(
-            cleaned_data['router_type'], cleaned_data['address'], cleaned_data['username'], cleaned_data['password'],
-            cleaned_data['ssh_key']
+            router_type, cleaned_data['address'], username, cleaned_data['password'], ssh_key
         )
         if not test_authentication_success:
             if test_authentication_message:

router_manager/models.py

@@ -26,7 +26,7 @@ class Router(models.Model):
     monitoring = models.BooleanField(default=True)
     backup_profile = models.ForeignKey(BackupProfile, on_delete=models.SET_NULL, null=True, blank=True)
 
-    router_type = models.CharField(max_length=100, choices=(('routeros', 'Mikrotik (RouterOS)'), ('openwrt', 'OpenWRT')))
+    router_type = models.CharField(max_length=100, choices=(('monitoring', 'Monitoring Only'), ('routeros', 'Mikrotik (RouterOS)'), ('openwrt', 'OpenWRT')))
     enabled = models.BooleanField(default=True)
 
     updated = models.DateTimeField(auto_now=True)

router_manager/views.py

@@ -58,6 +58,7 @@ def view_manage_router(request):
     if form.is_valid():
         form.save()
         messages.success(request, 'Router saved successfully')
+        router_status, router_status_created = RouterStatus.objects.get_or_create(router=form.instance)
         return redirect('router_list')
 
     context = {

routerfleet/urls.py

@@ -5,6 +5,7 @@ from user_manager.views import view_manage_user, view_user_list
 from accounts.views import view_login, view_logout, view_create_first_user
 from router_manager.views import view_router_list, view_manage_router, view_router_group_list, view_ssh_key_list, view_manage_router_group, view_manage_sshkey, view_router_details
 from backup.views import view_backup_profile_list, view_manage_backup_profile, view_backup_list, view_backup_details, view_debug_run_backups, view_compare_backups
+from monitoring.views import view_export_router_list, view_update_router_status
 
 
 urlpatterns = [
@@ -29,5 +30,6 @@ urlpatterns = [
     path('backup/backup_list/', view_backup_list, name='backup_list'),
     path('backup/backup_details/', view_backup_details, name='backup_info'),
     path('backup/compare/', view_compare_backups, name='compare_backups'),
-
+    path('monitoring/export_router_list/', view_export_router_list, name='export_router_list'),
+    path('monitoring/update_router_status/', view_update_router_status, name='update_router_status'),
 ]

templates/router_manager/router_list.html

@@ -49,23 +49,27 @@
                                             {% endif %}
                                         </td>
                                         <td>
+                                            {% if router.router_type != 'monitoring' %}
                                             {% if router.backup_profile %}
                                                 {{ router.backup_profile }} {% if router.routerstatus.last_backup_failed %}<i class="fas fa-exclamation-triangle text-danger" title="Last backup failed to complete"></i>{% endif %}
                                             {% else %}
                                                 <i class="fas fa-exclamation-triangle text-warning" title="No backup profile selected"></i>
                                             {% endif %}
+                                            {% endif %}
                                         </td>
                                     
                                         <td>
                                             {{ router.routergroup_set.count }}
                                         </td>
                                         <td class="min-width">
-                                            {% if router.ssh_key %}
-                                                <i class="fas fa-key" title="SSH Key: {{ router.ssh_key }}"></i>
-                                            {% elif router.password %}
-                                                <i class="fas fa-keyboard" title="Password Authentication"></i>
-                                            {% else %}
-                                                <i class="fas fa-exclamation-triangle text-warning" title="Missing authentication"></i>
+                                            {% if router.router_type != 'monitoring' %}
+                                                {% if router.ssh_key %}
+                                                    <i class="fas fa-key" title="SSH Key: {{ router.ssh_key }}"></i>
+                                                {% elif router.password %}
+                                                    <i class="fas fa-keyboard" title="Password Authentication"></i>
+                                                {% else %}
+                                                    <i class="fas fa-exclamation-triangle text-warning" title="Missing authentication"></i>
+                                                {% endif %}
                                             {% endif %}
                                         </td>
                                         <td class="min-width">