From f366e2cd698ad7d22e685fc798dc64ceff0e36f9 Mon Sep 17 00:00:00 2001
From: Eduardo Silva <eduardo@eth0.com.br>
Date: Wed, 10 Jul 2024 10:18:29 -0300
Subject: [PATCH] import tool first implementation

---
 import_tool/__init__.py                       |   0
 import_tool/admin.py                          |   3 +
 import_tool/apps.py                           |   6 ++
 import_tool/forms.py                          | 100 ++++++++++++++++++
 import_tool/migrations/0001_initial.py        |  53 ++++++++++
 ...ame_created_at_csvdata_created_and_more.py |  33 ++++++
 import_tool/migrations/__init__.py            |   0
 import_tool/models.py                         |  44 ++++++++
 import_tool/tests.py                          |   3 +
 import_tool/views.py                          |  22 ++++
 routerfleet/settings.py                       |   1 +
 routerfleet/urls.py                           |   3 +
 templates/import_tool/import_tool_list.html   |  45 ++++++++
 templates/router_manager/router_list.html     |   1 +
 14 files changed, 314 insertions(+)
 create mode 100644 import_tool/__init__.py
 create mode 100644 import_tool/admin.py
 create mode 100644 import_tool/apps.py
 create mode 100644 import_tool/forms.py
 create mode 100644 import_tool/migrations/0001_initial.py
 create mode 100644 import_tool/migrations/0002_rename_created_at_csvdata_created_and_more.py
 create mode 100644 import_tool/migrations/__init__.py
 create mode 100644 import_tool/models.py
 create mode 100644 import_tool/tests.py
 create mode 100644 import_tool/views.py
 create mode 100644 templates/import_tool/import_tool_list.html

diff --git a/import_tool/__init__.py b/import_tool/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/import_tool/admin.py b/import_tool/admin.py
new file mode 100644
index 0000000..8c38f3f
--- /dev/null
+++ b/import_tool/admin.py
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
diff --git a/import_tool/apps.py b/import_tool/apps.py
new file mode 100644
index 0000000..f334127
--- /dev/null
+++ b/import_tool/apps.py
@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ImportToolConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'import_tool'
diff --git a/import_tool/forms.py b/import_tool/forms.py
new file mode 100644
index 0000000..9a0306a
--- /dev/null
+++ b/import_tool/forms.py
@@ -0,0 +1,100 @@
+from router_manager.models import SUPPORTED_ROUTER_TYPES
+
+from django import forms
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import Layout, Submit, Row, Column, HTML
+from .models import CsvData
+import csv
+import io
+import re
+from django.core.exceptions import ValidationError
+
+SUPPORTED_ROUTER_TYPES = [rt[0] for rt in SUPPORTED_ROUTER_TYPES]
+
+
+class CsvDataForm(forms.ModelForm):
+    class Meta:
+        model = CsvData
+        fields = ['raw_csv_data']
+
+    def __init__(self, *args, **kwargs):
+        super(CsvDataForm, self).__init__(*args, **kwargs)
+        self.helper = FormHelper()
+        self.helper.form_method = 'post'
+        self.helper.layout = Layout(
+            Row(
+                Column('raw_csv_data', css_class='form-group col-md-12 mb-0'),
+                css_class='form-row'
+            ),
+            Row(
+                Column(
+                    Submit('submit', 'Save', css_class='btn btn-success'),
+                    HTML(' <a class="btn btn-secondary" href="/router/import_tool/">Back</a> '),
+                    css_class='col-md-12'),
+                css_class='form-row'
+            )
+        )
+
+    def clean(self):
+        cleaned_data = super().clean()
+        raw_csv_data = cleaned_data.get('raw_csv_data')
+
+        expected_fields = [
+            'name', 'username', 'password', 'ssh_key', 'address', 'port', 'router_type',
+            'backup_profile', 'router_group', 'monitoring'
+        ]
+
+        if raw_csv_data:
+            csv_file = io.StringIO(raw_csv_data)
+            reader = csv.DictReader(csv_file)
+
+            if reader.fieldnames is None:
+                raise ValidationError("The input is not a valid CSV file or it's empty.")
+
+            if reader.fieldnames != expected_fields:
+                raise ValidationError(f"The CSV header does not match the expected format. Expected fields: {expected_fields}")
+
+            required_fields = {'name', 'username', 'address', 'port', 'router_type', 'monitoring'}
+            optional_fields = {'backup_profile', 'router_group'}
+            valid_fields = required_fields | optional_fields | {'password', 'ssh_key'}
+            seen_names = set()
+
+            for line_number, row in enumerate(reader, start=2):  # start=2 to account for the header
+                if not any(row.values()):
+                    continue  # Skip empty lines
+
+                missing_fields = required_fields - row.keys()
+                if missing_fields:
+                    raise ValidationError(f"Missing required fields: {missing_fields} on line {line_number}")
+
+                extra_fields = set(row.keys()) - valid_fields
+                if extra_fields:
+                    raise ValidationError(f"Unexpected fields: {extra_fields} on line {line_number}")
+
+                if not (row.get('password') or row.get('ssh_key')):
+                    raise ValidationError(f"Either 'password' or 'ssh_key' must be provided on line {line_number}")
+                if row.get('password') and row.get('ssh_key'):
+                    raise ValidationError(f"Both 'password' and 'ssh_key' cannot be provided together on line {line_number}")
+
+                if row['name'] in seen_names:
+                    raise ValidationError(f"Duplicate name '{row['name']}' found on line {line_number}")
+                seen_names.add(row['name'])
+
+                if row['router_type'] not in SUPPORTED_ROUTER_TYPES:
+                    raise ValidationError(f"Invalid router_type '{row['router_type']}' on line {line_number}")
+
+                if row['monitoring'].lower() not in {'true', 'false'}:
+                    raise ValidationError(f"Invalid value for 'monitoring' on line {line_number}. Must be 'true' or 'false'.")
+
+                if not self._is_valid_ip_or_hostname(row['address']):
+                    raise ValidationError(f"Invalid address '{row['address']}' on line {line_number}")
+
+                if not row['port'].isdigit() or not (1 <= int(row['port']) <= 65535):
+                    raise ValidationError(f"Invalid port '{row['port']}' on line {line_number}")
+
+        return cleaned_data
+
+    def _is_valid_ip_or_hostname(self, value):
+        ip_pattern = re.compile(r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$")
+        hostname_pattern = re.compile(r"^(?=.{1,253}$)(?:(?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,6}$")
+        return ip_pattern.match(value) or hostname_pattern.match(value)
diff --git a/import_tool/migrations/0001_initial.py b/import_tool/migrations/0001_initial.py
new file mode 100644
index 0000000..2a20eda
--- /dev/null
+++ b/import_tool/migrations/0001_initial.py
@@ -0,0 +1,53 @@
+# Generated by Django 5.0.7 on 2024-07-09 19:06
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('backup', '0008_alter_backupprofile_backup_interval'),
+        ('router_manager', '0016_router_port'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CsvData',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('raw_csv_data', models.TextField()),
+                ('import_data', models.JSONField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ImportTask',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+                ('username', models.CharField(max_length=100)),
+                ('password', models.CharField(blank=True, max_length=100, null=True)),
+                ('address', models.CharField(max_length=100)),
+                ('port', models.IntegerField(default=22)),
+                ('router_type', models.CharField(choices=[('monitoring', 'Monitoring Only'), ('routeros', 'Mikrotik (RouterOS)'), ('openwrt', 'OpenWRT')], max_length=100)),
+                ('monitoring', models.BooleanField(default=True)),
+                ('import_success', models.BooleanField(default=False)),
+                ('import_error', models.BooleanField(default=False)),
+                ('import_error_message', models.TextField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)),
+                ('backup_profile', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='backup.backupprofile')),
+                ('csv_data', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='import_tool.csvdata')),
+                ('router', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='router_manager.router')),
+                ('router_group', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='router_manager.routergroup')),
+                ('ssh_key', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='router_manager.sshkey')),
+            ],
+        ),
+    ]
diff --git a/import_tool/migrations/0002_rename_created_at_csvdata_created_and_more.py b/import_tool/migrations/0002_rename_created_at_csvdata_created_and_more.py
new file mode 100644
index 0000000..61fc28a
--- /dev/null
+++ b/import_tool/migrations/0002_rename_created_at_csvdata_created_and_more.py
@@ -0,0 +1,33 @@
+# Generated by Django 5.0.7 on 2024-07-09 19:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('import_tool', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='csvdata',
+            old_name='created_at',
+            new_name='created',
+        ),
+        migrations.RenameField(
+            model_name='csvdata',
+            old_name='updated_at',
+            new_name='updated',
+        ),
+        migrations.RenameField(
+            model_name='importtask',
+            old_name='created_at',
+            new_name='created',
+        ),
+        migrations.RenameField(
+            model_name='importtask',
+            old_name='updated_at',
+            new_name='updated',
+        ),
+    ]
diff --git a/import_tool/migrations/__init__.py b/import_tool/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/import_tool/models.py b/import_tool/models.py
new file mode 100644
index 0000000..bbc8fba
--- /dev/null
+++ b/import_tool/models.py
@@ -0,0 +1,44 @@
+from django.db import models
+
+from backup.models import BackupProfile
+from router_manager.models import Router, SSHKey, SUPPORTED_ROUTER_TYPES, RouterGroup
+import uuid
+
+
+class CsvData(models.Model):
+    raw_csv_data = models.TextField()
+    import_data = models.JSONField(blank=True, null=True)
+
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
+
+    def __str__(self):
+        return self.name
+
+
+class ImportTask(models.Model):
+    csv_data = models.ForeignKey(CsvData, on_delete=models.CASCADE)
+    router = models.ForeignKey(Router, on_delete=models.SET_NULL, blank=True, null=True)
+    name = models.CharField(max_length=100)
+    ssh_key = models.ForeignKey(SSHKey, on_delete=models.SET_NULL, blank=True, null=True)
+    username = models.CharField(max_length=100)
+    password = models.CharField(max_length=100, blank=True, null=True)
+    router_group = models.ForeignKey(RouterGroup, on_delete=models.SET_NULL, blank=True, null=True)
+    address = models.CharField(max_length=100)
+    port = models.IntegerField(default=22)
+    backup_profile = models.ForeignKey(BackupProfile, on_delete=models.SET_NULL, blank=True, null=True)
+    router_type = models.CharField(max_length=100, choices=SUPPORTED_ROUTER_TYPES)
+    monitoring = models.BooleanField(default=True)
+
+    import_success = models.BooleanField(default=False)
+    import_error = models.BooleanField(default=False)
+    import_error_message = models.TextField(blank=True, null=True)
+
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
+
+    def __str__(self):
+        return self.name
+
diff --git a/import_tool/tests.py b/import_tool/tests.py
new file mode 100644
index 0000000..7ce503c
--- /dev/null
+++ b/import_tool/tests.py
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
diff --git a/import_tool/views.py b/import_tool/views.py
new file mode 100644
index 0000000..3981b25
--- /dev/null
+++ b/import_tool/views.py
@@ -0,0 +1,22 @@
+from django.shortcuts import render
+from django.contrib.auth.decorators import login_required
+from .models import CsvData, ImportTask
+from .forms import CsvDataForm
+
+
+@login_required()
+def view_import_tool_list(request):
+    import_list = CsvData.objects.all().order_by('-created')
+    data = {
+        'import_list': import_list,
+        'page_title': 'CSV import List',
+    }
+    return render(request, 'import_tool/import_tool_list.html', context=data)
+
+
+@login_required()
+def view_import_csv_file(request):
+    form = CsvDataForm(request.POST or None)
+    data = {'form': form, 'page_title': 'Import CSV File'}
+
+    return render(request, 'generic_form.html', context=data)
diff --git a/routerfleet/settings.py b/routerfleet/settings.py
index 7d9f0d4..737e013 100644
--- a/routerfleet/settings.py
+++ b/routerfleet/settings.py
@@ -48,6 +48,7 @@ INSTALLED_APPS = [
     'integration_manager',
     'routerfleet_tools',
     'message_center',
+    'import_tool'
 ]
 
 MIDDLEWARE = [
diff --git a/routerfleet/urls.py b/routerfleet/urls.py
index 637420d..c3f9828 100644
--- a/routerfleet/urls.py
+++ b/routerfleet/urls.py
@@ -10,6 +10,7 @@ from monitoring.views import view_export_router_list, view_update_router_status,
 from backup_data.views import view_generate_backup_schedule, view_create_backup_tasks, view_perform_backup_tasks, view_housekeeping
 from routerfleet_tools.views import cron_check_updates
 from message_center.views import view_message_channel_list, view_manage_message_settings, view_manage_message_channel, view_debug_test_messages, view_cron_concatenate_notifications, view_cron_send_messages, view_cron_daily_reports, view_message_history
+from import_tool.views import view_import_tool_list, view_import_csv_file
 
 
 urlpatterns = [
@@ -32,6 +33,8 @@ urlpatterns = [
     path('router/manage_group/', view_manage_router_group, name='manage_router_group'),
     path('router/manage_sshkey/', view_manage_sshkey, name='manage_sshkey'),
     path('router/create_instant_backup/', view_create_instant_backup_task, name='create_instant_backup_task'),
+    path('router/import_tool/', view_import_tool_list, name='import_tool_list'),
+    path('router/import_tool/csv/', view_import_csv_file, name='import_csv_file'),
     path('backup/profile_list/', view_backup_profile_list, name='backup_profile_list'),
     path('backup/manage_profile/', view_manage_backup_profile, name='manage_backup_profile'),
     path('backup/backup_list/', view_backup_list, name='backup_list'),
diff --git a/templates/import_tool/import_tool_list.html b/templates/import_tool/import_tool_list.html
new file mode 100644
index 0000000..cbfe19a
--- /dev/null
+++ b/templates/import_tool/import_tool_list.html
@@ -0,0 +1,45 @@
+{% extends 'base.html' %}
+
+
+{% block content %}
+    <div class='row'>
+        <div class='col-lg-12'>
+            <div class="card card-primary card-outline">
+                {% if page_title %}
+                    <div class="card-header">
+                        <h3 class="card-title">{{ page_title }}</h3>
+                    </div>
+                {% endif %}
+                <div class="card-body">
+                    <div class="row">
+                        <div class="col-lg-12">
+                            <table class="table table-hover datatables-no-export">
+                                <thead>
+                                <tr>
+
+                                </tr>
+
+                                </thead>
+                                <tbody>
+
+                                </tbody>
+                            </table>
+                        </div>
+
+                    </div>
+                    <div class="row">
+                        <div class="col-lg-12">
+                            <a href="/router/import_tool/csv/" class="btn btn-primary">Import CSV</a>
+                        </div>
+
+                    </div>
+
+                </div>
+            </div>
+        </div>
+    </div>
+{% endblock %}
+
+{% block custom_page_scripts %}
+
+{% endblock %}
\ No newline at end of file
diff --git a/templates/router_manager/router_list.html b/templates/router_manager/router_list.html
index c913962..dc4c8be 100644
--- a/templates/router_manager/router_list.html
+++ b/templates/router_manager/router_list.html
@@ -84,6 +84,7 @@
                     <div class="row">
                         <div class="col-lg-12">
                             <a href="/router/manage/" class="btn btn-primary">Add Router</a>
+                            <a href="/router/import_tool/" class="btn btn-outline-primary">Import tool</a>
                         </div>
 
                     </div>