Automatic BackupSchedule management

This commit is contained in:
Eduardo Silva 2024-04-02 16:56:35 -03:00
parent 31b1c663f2
commit a12a126d38
13 changed files with 298 additions and 31 deletions

View file

@ -9,12 +9,12 @@ class BackupProfileForm(forms.ModelForm):
model = BackupProfile
fields = [
'name', 'daily_backup', 'weekly_backup', 'monthly_backup',
'daily_retenion', 'weekly_retention', 'monthly_retenion',
'daily_retention', 'weekly_retention', 'monthly_retention',
'retain_backups_on_error', 'daily_day_monday', 'daily_day_tuesday',
'daily_day_wednesday', 'daily_day_thursday', 'daily_day_friday',
'daily_day_saturday', 'daily_day_sunday', 'weekly_day',
'monthly_day', 'daily_hour', 'weekly_hour', 'monthly_hour',
'max_retry', 'retry_interval', 'backup_interval'
'max_retry', 'retry_interval', 'backup_interval', 'retrieve_interval', 'instant_retention'
]
# widgets = {
# 'weekly_day': forms.Select(),
@ -31,7 +31,7 @@ class BackupProfileForm(forms.ModelForm):
super(BackupProfileForm, self).__init__(*args, **kwargs)
self.helper = FormHelper()
self.helper.form_method = 'post'
if self.instance.pk:
if self.instance.pk and self.instance.name != 'default':
delete_html = "<a href='javascript:void(0)' class='btn btn-outline-danger' data-command='delete' onclick='openCommandDialog(this)'>Delete</a>"
else:
delete_html = ''
@ -46,9 +46,12 @@ class BackupProfileForm(forms.ModelForm):
self.fields['daily_backup'].label = 'Daily'
self.fields['weekly_backup'].label = 'Weekly'
self.fields['monthly_backup'].label = 'Monthly'
self.fields['daily_retenion'].label = 'Retention (days)'
self.fields['daily_retention'].label = 'Retention (days)'
self.fields['weekly_retention'].label = 'Retention (days)'
self.fields['monthly_retenion'].label = 'Retention (days)'
self.fields['monthly_retention'].label = 'Retention (days)'
self.fields['instant_retention'].label = 'Instant Retention (days)'
if self.instance.pk and self.instance.name == 'default':
self.fields['name'].widget.attrs['readonly'] = True
self.helper.layout = Layout(
Div(Div('name', css_class='col-md-12'), css_class='row'),
@ -60,9 +63,8 @@ class BackupProfileForm(forms.ModelForm):
Div(
Div(HTML('<hr><h4>Daily Backups</h4>'), css_class='col-md-12'),
Div('daily_hour', css_class='col-md-4'),
Div('daily_retenion', css_class='col-md-4'),
Div(css_class='col-md-4'),
Div('daily_hour', css_class='col-md-6'),
Div('daily_retention', css_class='col-md-6'),
Div('daily_day_monday', css_class='col-md-4'),
Div('daily_day_tuesday', css_class='col-md-4'),
Div('daily_day_wednesday', css_class='col-md-4'),
@ -75,25 +77,29 @@ class BackupProfileForm(forms.ModelForm):
Div(
Div(HTML('<hr><h4>Weekly Backups</h4>'), css_class='col-md-12'),
Div('weekly_hour', css_class='col-md-4'),
Div('weekly_retention', css_class='col-md-4'),
Div('weekly_day', css_class='col-md-4'),
Div('weekly_hour', css_class='col-md-6'),
Div('weekly_day', css_class='col-md-6'),
Div('weekly_retention', css_class='col-md-6'),
css_id='weekly_settings', css_class='row'
),
Div(
Div(HTML('<hr><h4>Monthly Backups</h4>'), css_class='col-md-12'),
Div('monthly_hour', css_class='col-md-4'),
Div('monthly_retenion', css_class='col-md-4'),
Div('monthly_day', css_class='col-md-4'),
Div('monthly_hour', css_class='col-md-6'),
Div('monthly_day', css_class='col-md-6'),
Div('monthly_retention', css_class='col-md-6'),
css_id='monthly_settings', css_class='row'
),
Div(
Div(HTML('<hr><h4>Backup Settings</h4>'), css_class='col-md-12'),
Div('max_retry', css_class='col-md-4'),
Div('retry_interval', css_class='col-md-4'),
Div('backup_interval', css_class='col-md-4'),
Div('max_retry', css_class='col-md-6'),
Div('retry_interval', css_class='col-md-6'),
Div('backup_interval', css_class='col-md-6'),
Div('retrieve_interval', css_class='col-md-6'),
Div('instant_retention', css_class='col-md-6'),
Div('retain_backups_on_error', css_class='col-md-12'),
css_id='misc_settings', css_class='row'
),
@ -120,6 +126,11 @@ class BackupProfileForm(forms.ModelForm):
daily_day_friday = cleaned_data.get('daily_day_friday')
daily_day_saturday = cleaned_data.get('daily_day_saturday')
daily_day_sunday = cleaned_data.get('daily_day_sunday')
name = cleaned_data.get('name')
if self.instance.pk:
if self.instance.name == 'default' and name != 'default':
raise forms.ValidationError('You cannot change the default profile name')
if daily_backup:
if not daily_day_monday and not daily_day_tuesday and not daily_day_wednesday and not daily_day_thursday and not daily_day_friday and not daily_day_saturday and not daily_day_sunday:

View file

@ -0,0 +1,23 @@
# Generated by Django 5.0.3 on 2024-04-01 15:41
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('backup', '0003_alter_backupprofile_retry_interval'),
]
operations = [
migrations.AddField(
model_name='backupprofile',
name='instant_retenion',
field=models.IntegerField(default=3650),
),
migrations.AddField(
model_name='backupprofile',
name='retrieve_interval',
field=models.IntegerField(choices=[(1, '1 Minute'), (15, '15 Minutes'), (30, '30 Minutes'), (60, '1 Hour')], default=1),
),
]

View file

@ -0,0 +1,28 @@
# Generated by Django 5.0.3 on 2024-04-01 15:48
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('backup', '0004_backupprofile_instant_retenion_and_more'),
]
operations = [
migrations.RenameField(
model_name='backupprofile',
old_name='daily_retenion',
new_name='daily_retention',
),
migrations.RenameField(
model_name='backupprofile',
old_name='instant_retenion',
new_name='instant_retention',
),
migrations.RenameField(
model_name='backupprofile',
old_name='monthly_retenion',
new_name='monthly_retention',
),
]

View file

@ -0,0 +1,18 @@
# Generated by Django 5.0.3 on 2024-04-01 16:33
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('backup', '0005_rename_daily_retenion_backupprofile_daily_retention_and_more'),
]
operations = [
migrations.AlterField(
model_name='backupprofile',
name='retrieve_interval',
field=models.IntegerField(choices=[(15, '15 Seconds'), (30, '30 Seconds'), (60, '1 Minute'), (900, '15 Minutes'), (1800, '30 Minutes'), (3600, '1 Hour')], default=60),
),
]

View file

@ -0,0 +1,18 @@
# Generated by Django 5.0.3 on 2024-04-02 19:25
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('backup', '0006_alter_backupprofile_retrieve_interval'),
]
operations = [
migrations.AddField(
model_name='backupprofile',
name='profile_error_information',
field=models.CharField(blank=True, max_length=100, null=True),
),
]

View file

@ -17,9 +17,10 @@ class BackupProfile(models.Model):
weekly_backup = models.BooleanField(default=False)
monthly_backup = models.BooleanField(default=False)
daily_retenion = models.IntegerField(default=7)
daily_retention = models.IntegerField(default=7)
weekly_retention = models.IntegerField(default=30)
monthly_retenion = models.IntegerField(default=365)
monthly_retention = models.IntegerField(default=365)
instant_retention = models.IntegerField(default=3650)
retain_backups_on_error = models.BooleanField(default=True)
daily_day_monday = models.BooleanField(default=True)
@ -39,8 +40,11 @@ class BackupProfile(models.Model):
max_retry = models.IntegerField(default=3, choices=((1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5')))
retry_interval = models.IntegerField(default=30, choices=((1, '1 Minute'), (15, '15 Minutes'), (30, '30 Minutes'), (60, '1 Hour')))
retrieve_interval = models.IntegerField(default=60, choices=((15, '15 Seconds'), (30, '30 Seconds'), (60, '1 Minute'), (900, '15 Minutes'), (1800, '30 Minutes'), (3600, '1 Hour')))
backup_interval = models.IntegerField(default=60, choices=((0, 'No interval'), (5, '5 seconds'), (60, '1 minute')))
profile_error_information = models.CharField(max_length=100, blank=True, null=True)
updated = models.DateTimeField(auto_now=True)
created = models.DateTimeField(auto_now_add=True)
uuid = models.UUIDField(unique=True, editable=False, default=uuid.uuid4)

View file

@ -6,7 +6,7 @@ from django.contrib import messages
from routerlib.backup_functions import perform_backup
from .models import BackupProfile
from .forms import BackupProfileForm
from router_manager.models import Router
from router_manager.models import Router, BackupSchedule
from backup_data.models import RouterBackup
import difflib
import unicodedata
@ -15,6 +15,7 @@ from routerlib.functions import gen_backup_name, get_router_backup_file_extensio
@login_required()
def view_backup_profile_list(request):
default_backup_profile, _ = BackupProfile.objects.get_or_create(name='default')
context = {
'backup_profile_list': BackupProfile.objects.all().order_by('name'),
'page_title': 'Backup Profiles'
@ -28,12 +29,16 @@ def view_manage_backup_profile(request):
backup_profile = get_object_or_404(BackupProfile, uuid=request.GET.get('uuid'))
if request.GET.get('action') == 'delete':
if request.GET.get('confirmation') == 'delete':
if Router.objects.filter(backup_profile=backup_profile).exists():
messages.warning(request, 'Backup profile in use|Backup profile is in use and cannot be deleted')
if backup_profile.name == 'default':
messages.warning(request, 'Backup profile not deleted|Default profile cannot be deleted')
return redirect('backup_profile_list')
else:
backup_profile.delete()
messages.success(request, 'Backup profile deleted successfully')
if Router.objects.filter(backup_profile=backup_profile).exists():
messages.warning(request, 'Backup profile in use|Backup profile is in use and cannot be deleted')
return redirect('backup_profile_list')
else:
backup_profile.delete()
messages.success(request, 'Backup profile deleted successfully')
return redirect('backup_profile_list')
else:
messages.warning(request, 'Backup profile not deleted|Invalid confirmation')
@ -43,7 +48,9 @@ def view_manage_backup_profile(request):
form = BackupProfileForm(request.POST or None, instance=backup_profile)
if form.is_valid():
form.instance.profile_error_information = ''
form.save()
BackupSchedule.objects.filter(router__backup_profile=form.instance).delete()
messages.success(request, 'Backup Profile saved successfully')
return redirect('backup_profile_list')

View file

@ -1,3 +1,140 @@
from django.shortcuts import render
from django.http import JsonResponse
from backup.models import BackupProfile
from backup_data.models import RouterBackup
from router_manager.models import Router, BackupSchedule
# Create your views here.
from datetime import datetime, timedelta
from django.utils import timezone
def next_weekday(now, weekday, hour):
days_ahead = weekday - now.weekday()
if days_ahead < 0 or (days_ahead == 0 and now.hour >= hour): # if backup date is for today and hour has passed, move to next week
days_ahead += 7
next_backup = now + timedelta(days=days_ahead)
return next_backup.replace(hour=hour, minute=0, second=0, microsecond=0)
def find_next_active_day(start_date, active_days, backup_hour):
for i in range(7): # Verifica os próximos 7 dias
potential_date = start_date + timedelta(days=i)
if active_days[potential_date.weekday()]:
next_active_date = potential_date.replace(hour=backup_hour, minute=0, second=0, microsecond=0)
if next_active_date > timezone.now():
return next_active_date
# Se já passou a hora no primeiro dia válido, procura no dia correspondente da próxima semana
if i == 0:
return potential_date + timedelta(days=7, hours=(backup_hour - potential_date.hour))
return None
def calculate_next_backup(backup_profile):
now = timezone.now()
if backup_profile.daily_backup:
weekdays_enabled = [
backup_profile.daily_day_monday,
backup_profile.daily_day_tuesday,
backup_profile.daily_day_wednesday,
backup_profile.daily_day_thursday,
backup_profile.daily_day_friday,
backup_profile.daily_day_saturday,
backup_profile.daily_day_sunday,
]
next_daily_backup = find_next_active_day(now, weekdays_enabled, backup_profile.daily_hour)
else:
next_daily_backup = None
if backup_profile.weekly_backup:
weekday = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'].index(backup_profile.weekly_day)
next_weekly_backup = next_weekday(now, weekday, backup_profile.weekly_hour)
else:
next_weekly_backup = None
if backup_profile.monthly_backup:
potential_monthly_backup = datetime(year=now.year, month=now.month, day=backup_profile.monthly_day,
hour=backup_profile.monthly_hour, tzinfo=timezone.get_current_timezone())
if potential_monthly_backup <= now:
# Se o dia do mês já passou, ou é hoje mas a hora já passou, agenda para o próximo mês
month_increment = 1 if now.month < 12 else -11
year_increment = 0 if now.month < 12 else 1
next_monthly_backup = potential_monthly_backup.replace(year=now.year + year_increment,
month=now.month + month_increment)
else:
next_monthly_backup = potential_monthly_backup
else:
next_monthly_backup = None
return next_daily_backup, next_weekly_backup, next_monthly_backup
def generate_backup_schedule(request):
data = {
'backup_schedule_created': 0,
'daily_backup_schedule_created': 0,
'weekly_backup_schedule_created': 0,
'monthly_backup_schedule_created': 0,
'daily_backup_schedule_removed': 0,
'weekly_backup_schedule_removed': 0,
'monthly_backup_schedule_removed': 0
}
for router in Router.objects.filter(backupschedule__isnull=True):
new_backup_schedule, _ = BackupSchedule.objects.get_or_create(router=router)
data['backup_schedule_created'] += 1
for backup_profile in BackupProfile.objects.all():
backup_schedule_list = BackupSchedule.objects.filter(router__backup_profile=backup_profile)
next_daily_backup, next_weekly_backup, next_monthly_backup = calculate_next_backup(backup_profile)
if backup_profile.daily_backup and not next_daily_backup:
backup_profile.profile_error_information = 'Error calculating next daily backup. Check profile settings'
backup_profile.save()
if backup_profile.weekly_backup and not next_weekly_backup:
backup_profile.profile_error_information = 'Error calculating next weekly backup. Check profile settings'
backup_profile.save()
if backup_profile.monthly_backup and not next_monthly_backup:
backup_profile.profile_error_information = 'Error calculating next monthly backup. Check profile settings'
backup_profile.save()
if backup_profile.daily_backup:
daily_schedule_list = backup_schedule_list.filter(next_daily_backup__isnull=True)
for schedule in daily_schedule_list:
schedule.next_daily_backup = next_daily_backup
schedule.save()
data['daily_backup_schedule_created'] += 1
else:
daily_schedule_list = backup_schedule_list.filter(next_daily_backup__isnull=False)
for schedule in daily_schedule_list:
schedule.next_daily_backup = None
schedule.save()
data['daily_backup_schedule_removed'] += 1
if backup_profile.weekly_backup:
weekly_schedule_list = backup_schedule_list.filter(next_weekly_backup__isnull=True)
for schedule in weekly_schedule_list:
schedule.next_weekly_backup = next_weekly_backup
schedule.save()
data['weekly_backup_schedule_created'] += 1
else:
weekly_schedule_list = backup_schedule_list.filter(next_weekly_backup__isnull=False)
for schedule in weekly_schedule_list:
schedule.next_weekly_backup = None
schedule.save()
data['weekly_backup_schedule_removed'] += 1
if backup_profile.monthly_backup:
monthly_schedule_list = backup_schedule_list.filter(next_monthly_backup__isnull=True)
for schedule in monthly_schedule_list:
schedule.next_monthly_backup = next_monthly_backup
schedule.save()
data['monthly_backup_schedule_created'] += 1
else:
monthly_schedule_list = backup_schedule_list.filter(next_monthly_backup__isnull=False)
for schedule in monthly_schedule_list:
schedule.next_monthly_backup = None
schedule.save()
data['monthly_backup_schedule_removed'] += 1
return JsonResponse(data)

View file

@ -1,5 +1,5 @@
from django.contrib import admin
from .models import Router, SSHKey, RouterStatus
from .models import Router, SSHKey, RouterStatus, BackupSchedule
class RouterAdmin(admin.ModelAdmin):
@ -27,3 +27,12 @@ class RouterStatusAdmin(admin.ModelAdmin):
admin.site.register(RouterStatus, RouterStatusAdmin)
class BackupScheduleAdmin(admin.ModelAdmin):
list_display = ('router', 'next_daily_backup', 'next_weekly_backup', 'next_monthly_backup')
search_fields = ('router', 'next_daily_backup', 'next_weekly_backup', 'next_monthly_backup')
list_filter = ('router', 'next_daily_backup', 'next_weekly_backup', 'next_monthly_backup')
admin.site.register(BackupSchedule, BackupScheduleAdmin)

View file

@ -1,7 +1,7 @@
from django.contrib import messages
from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.decorators import login_required
from .models import Router, RouterGroup, RouterStatus, SSHKey
from .models import Router, RouterGroup, RouterStatus, SSHKey, BackupSchedule
from .forms import RouterForm, RouterGroupForm, SSHKeyForm
@ -30,7 +30,7 @@ def view_router_list(request):
@login_required()
def view_router_details(request):
router = get_object_or_404(Router, uuid=request.GET.get('uuid'))
router_status, router_status_created = RouterStatus.objects.get_or_create(router=router)
router_status, _ = RouterStatus.objects.get_or_create(router=router)
context = {
'router': router,
'router_status': router_status,
@ -39,6 +39,7 @@ def view_router_details(request):
}
return render(request, 'router_manager/router_details.html', context=context)
@login_required()
def view_manage_router(request):
if request.GET.get('uuid'):
@ -58,7 +59,8 @@ def view_manage_router(request):
if form.is_valid():
form.save()
messages.success(request, 'Router saved successfully')
router_status, router_status_created = RouterStatus.objects.get_or_create(router=form.instance)
router_status, _ = RouterStatus.objects.get_or_create(router=form.instance)
BackupSchedule.objects.filter(router=form.instance).delete()
return redirect('router_list')
context = {

View file

@ -6,6 +6,7 @@ from accounts.views import view_login, view_logout, view_create_first_user
from router_manager.views import view_router_list, view_manage_router, view_router_group_list, view_ssh_key_list, view_manage_router_group, view_manage_sshkey, view_router_details
from backup.views import view_backup_profile_list, view_manage_backup_profile, view_backup_list, view_backup_details, view_debug_run_backups, view_compare_backups, view_backup_download, view_backup_delete
from monitoring.views import view_export_router_list, view_update_router_status
from backup_data.views import generate_backup_schedule
urlpatterns = [
@ -34,4 +35,5 @@ urlpatterns = [
path('backup/delete/', view_backup_delete, name='delete_backup'),
path('monitoring/export_router_list/', view_export_router_list, name='export_router_list'),
path('monitoring/update_router_status/', view_update_router_status, name='update_router_status'),
path('cron/generate_backup_schedule/', generate_backup_schedule, name='generate_backup_schedule'),
]

View file

@ -47,7 +47,7 @@ def perform_backup(router_backup: RouterBackup):
router_backup.backup_pending_retrieval = True
router_backup.error_message = ''
router_backup.retry_count = 0
router_backup.next_retry = timezone.now() + datetime.timedelta(minutes=router_backup.router.backup_profile.backup_interval)
router_backup.next_retry = timezone.now() + datetime.timedelta(seconds=router_backup.router.backup_profile.retrieve_interval)
router_backup.save()
else:
handle_backup_failure(router_backup, error_message)

View file

@ -23,6 +23,7 @@
<th>Weekly</th>
<th>Monthly</th>
<th></th>
<th></th>
</tr>
</thead>
@ -52,6 +53,13 @@
{% endif %}
</td>
<td>
{% if backup_profile.profile_error_information %}
<i class="fas fa-exclamation-triangle text-warning"></i> {{ backup_profile.profile_error_information }}
{% endif %}
</td>
<td class="min-width">
<a href="/backup/manage_profile/?uuid={{ backup_profile.uuid }}"><i class="fas fa-edit"></i></a>
</td>