global-functions: $SymbolByUnicodeName: allow to add more symbols...

... for example from a module. Add a script `mod/symbols-extra` with
something like:

    :global SymbolsExtra;

    :set ($SymbolsExtra->"rocket") "\F0\9F\9A\80";

CONTRIBUTIONS.md

@@ -35,6 +35,7 @@ Add yourself to the list,
 [donate with PayPal ↗️](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A4ZXBD6YS2W8J)!
 
 * Abdul Mannan Abbasi
+* Alex Maier
 * Andrea Ruffini Perico
 * Andrew Cox
 * Christoph Boss (@Kampfwurst)

accesslist-duplicates.capsman.rsc

@@ -22,7 +22,7 @@
   :foreach AccList in=[ /caps-man/access-list/find where mac-address!="00:00:00:00:00:00" ] do={
     :local Mac [ /caps-man/access-list/get $AccList mac-address ];
     :if ($Seen->$Mac = 1) do={
-      /caps-man/access-list/print where mac-address=$Mac;
+      /caps-man/access-list/print without-paging where mac-address=$Mac;
       :local Remove [ :tonum [ /terminal/ask prompt="\nNumeric id to remove, any key to skip!" ] ];
 
       :if ([ :typeof $Remove ] = "num") do={

accesslist-duplicates.local.rsc

@@ -22,7 +22,7 @@
   :foreach AccList in=[ /interface/wireless/access-list/find where mac-address!="00:00:00:00:00:00" ] do={
     :local Mac [ /interface/wireless/access-list/get $AccList mac-address ];
     :if ($Seen->$Mac = 1) do={
-      /interface/wireless/access-list/print where mac-address=$Mac;
+      /interface/wireless/access-list/print without-paging where mac-address=$Mac;
       :local Remove [ :tonum [ /terminal/ask prompt="\nNumeric id to remove, any key to skip!" ] ];
 
       :if ([ :typeof $Remove ] = "num") do={

accesslist-duplicates.template.rsc

@@ -27,9 +27,9 @@
     :local Mac [ /interface/wifi/access-list/get $AccList mac-address ];
     :local Mac [ /interface/wireless/access-list/get $AccList mac-address ];
     :if ($Seen->$Mac = 1) do={
-      /caps-man/access-list/print where mac-address=$Mac;
-      /interface/wifi/access-list/print where mac-address=$Mac;
-      /interface/wireless/access-list/print where mac-address=$Mac;
+      /caps-man/access-list/print without-paging where mac-address=$Mac;
+      /interface/wifi/access-list/print without-paging where mac-address=$Mac;
+      /interface/wireless/access-list/print without-paging where mac-address=$Mac;
       :local Remove [ :tonum [ /terminal/ask prompt="\nNumeric id to remove, any key to skip!" ] ];
 
       :if ([ :typeof $Remove ] = "num") do={

accesslist-duplicates.wifi.rsc

@@ -22,7 +22,7 @@
   :foreach AccList in=[ /interface/wifi/access-list/find where mac-address!="00:00:00:00:00:00" ] do={
     :local Mac [ /interface/wifi/access-list/get $AccList mac-address ];
     :if ($Seen->$Mac = 1) do={
-      /interface/wifi/access-list/print where mac-address=$Mac;
+      /interface/wifi/access-list/print without-paging where mac-address=$Mac;
       :local Remove [ :tonum [ /terminal/ask prompt="\nNumeric id to remove, any key to skip!" ] ];
 
       :if ([ :typeof $Remove ] = "num") do={

backup-email.rsc

@@ -27,6 +27,7 @@
 
   :global CleanName;
   :global DeviceInfo;
+  :global FileExists;
   :global FormatLine;
   :global LogPrint;
   :global MkDir;
@@ -124,17 +125,19 @@
     attach=$Attach; remove-attach=true });
 
   # wait for the mail to be sent
-  :local I 0;
-  :while ([ :len [ /file/find where name ~ ($FilePath . "\\.(backup|rsc)\$") ] ] > 0) do={
-    :if ($I >= 120) do={
-      $LogPrint warning $ScriptName ("Files are still available, sending e-mail failed.");
-      :set PackagesUpdateBackupFailure true;
-      :set ExitOK true;
-      :error false;
-    }
-    :delay 1s;
-    :set I ($I + 1);
+  :do {
+    :retry {
+      :if ([ $FileExists ($FilePath . ".conf") ".conf file" ] = true || \
+           [ $FileExists ($FilePath . ".backup") "backup" ] = true || \
+           [ $FileExists ($FilePath . ".rsc") "script" ] = true) do={
+        :error "Files are still available.";
+      }
+    } delay=1s max=120;
+  } on-error={
+    $LogPrint warning $ScriptName ("Files are still available, sending e-mail failed.");
+    :set PackagesUpdateBackupFailure true;
   }
+  # do not remove the files here, as the mail is still queued!
 } do={
   :global ExitError; $ExitError $ExitOK [ :jobname ] $Err;
 }

capsman-download-packages.capsman.rsc

@@ -20,6 +20,7 @@
 
   :global CleanFilePath;
   :global DownloadPackage;
+  :global FileGet;
   :global LogPrint;
   :global MkDir;
   :global RmFile;
@@ -42,7 +43,7 @@
     :error false;
   }
 
-  :if ([ :len [ /file/find where name=$PackagePath type="directory" ] ] = 0) do={
+  :if ([ $FileGet $PackagePath ] = false) do={
     :if ([ $MkDir $PackagePath ] = false) do={
       $LogPrint warning $ScriptName ("Creating directory at CAPsMAN package path (" . \
         $PackagePath . ") failed!");
@@ -53,7 +54,7 @@
       "). Please place your packages!");
   }
 
-  :foreach Package in=[ /file/find where type=package \
+  :foreach Package in=[ /file/find where type="package" \
         package-version!=$InstalledVersion name~("^" . $PackagePath) ] do={
     :local File [ /file/get $Package ];
     :if ($File->"package-architecture" = "mips") do={
@@ -66,7 +67,7 @@
     }
   }
 
-  :if ([ :len [ /file/find where type=package name~("^" . $PackagePath) ] ] = 0) do={
+  :if ([ :len [ /file/find where type="package" name~("^" . $PackagePath) ] ] = 0) do={
     $LogPrint info $ScriptName ("No packages available, downloading default set.");
     :foreach Arch in={ "arm"; "mipsbe" } do={
       :foreach Package in={ "routeros"; "wireless" } do={

capsman-download-packages.template.rsc

@@ -21,6 +21,7 @@
 
   :global CleanFilePath;
   :global DownloadPackage;
+  :global FileGet;
   :global LogPrint;
   :global MkDir;
   :global RmFile;
@@ -44,7 +45,7 @@
     :error false;
   }
 
-  :if ([ :len [ /file/find where name=$PackagePath type="directory" ] ] = 0) do={
+  :if ([ $FileGet $PackagePath ] = false) do={
     :if ([ $MkDir $PackagePath ] = false) do={
       $LogPrint warning $ScriptName ("Creating directory at CAPsMAN package path (" . \
         $PackagePath . ") failed!");
@@ -55,7 +56,7 @@
       "). Please place your packages!");
   }
 
-  :foreach Package in=[ /file/find where type=package \
+  :foreach Package in=[ /file/find where type="package" \
         package-version!=$InstalledVersion name~("^" . $PackagePath) ] do={
     :local File [ /file/get $Package ];
     :if ($File->"package-architecture" = "mips") do={
@@ -68,7 +69,7 @@
     }
   }
 
-  :if ([ :len [ /file/find where type=package name~("^" . $PackagePath) ] ] = 0) do={
+  :if ([ :len [ /file/find where type="package" name~("^" . $PackagePath) ] ] = 0) do={
     $LogPrint info $ScriptName ("No packages available, downloading default set.");
 # NOT /interface/wifi/ #
     :foreach Arch in={ "arm"; "mipsbe" } do={

capsman-download-packages.wifi.rsc

@@ -20,6 +20,7 @@
 
   :global CleanFilePath;
   :global DownloadPackage;
+  :global FileGet;
   :global LogPrint;
   :global MkDir;
   :global RmFile;
@@ -42,7 +43,7 @@
     :error false;
   }
 
-  :if ([ :len [ /file/find where name=$PackagePath type="directory" ] ] = 0) do={
+  :if ([ $FileGet $PackagePath ] = false) do={
     :if ([ $MkDir $PackagePath ] = false) do={
       $LogPrint warning $ScriptName ("Creating directory at CAPsMAN package path (" . \
         $PackagePath . ") failed!");
@@ -53,7 +54,7 @@
       "). Please place your packages!");
   }
 
-  :foreach Package in=[ /file/find where type=package \
+  :foreach Package in=[ /file/find where type="package" \
         package-version!=$InstalledVersion name~("^" . $PackagePath) ] do={
     :local File [ /file/get $Package ];
     :if ($File->"package-architecture" = "mips") do={
@@ -66,7 +67,7 @@
     }
   }
 
-  :if ([ :len [ /file/find where type=package name~("^" . $PackagePath) ] ] = 0) do={
+  :if ([ :len [ /file/find where type="package" name~("^" . $PackagePath) ] ] = 0) do={
     $LogPrint info $ScriptName ("No packages available, downloading default set.");
     :foreach Arch in={ "arm"; "arm64" } do={
       :local Packages { "arm"={ "routeros"; "wifi-qcom"; "wifi-qcom-ac" };

check-routeros-update.rsc

@@ -28,6 +28,7 @@
   :global EscapeForRegEx;
   :global FetchUserAgentStr;
   :global LogPrint;
+  :global RebootForUpdate;
   :global ScriptFromTerminal;
   :global ScriptLock;
   :global SendNotification2;
@@ -62,9 +63,14 @@
   $WaitFullyConnected;
 
   :if ([ :len [ /system/scheduler/find where name="_RebootForUpdate" ] ] > 0) do={
-    $LogPrint info $ScriptName ("A reboot for update is already scheduled.");
-    :set ExitOK true;
-    :error false;
+    :if ([ :typeof $RebootForUpdate ] = "nothing") do={
+      $LogPrint info $ScriptName ("Found a stale scheduler for reboot, removing.");
+      /system/scheduler/remove "_RebootForUpdate";
+    } else={
+      $LogPrint info $ScriptName ("A reboot for update is already scheduled.");
+      :set ExitOK true;
+      :error false;
+    }
   }
 
   $LogPrint debug $ScriptName ("Checking for updates...");

doc/fw-addr-lists.md

@@ -32,7 +32,10 @@ certificate is checked.
 
 > ⚠️ **Warning**: The script does not limit the size of a list, but keep in
 > mind that huge lists can exhaust your device's resources (RAM and CPU),
-> and may take a long time to process.
+> and may take a long time to process.  
+> Even crashes for the complete scripting (and CLI) subsystem are possible.
+> This should be logged accordingly with warnings when global functions are
+> reloaded from scheduler.
 
 Requirements and installation
 -----------------------------

doc/packages-update.md

@@ -46,8 +46,8 @@ Configuration
 
 The configuration goes to `global-config-overlay`, this is the only parameter:
 
-* `PackagesUpdateDeferReboot`: defer the reboot for night (between 3 AM
-  and 5 AM)
+* `PackagesUpdateDeferReboot`: defer the reboot for night (between 3 AM and
+  5 AM), use a numerical value in days suffixed with a `d` to defer further
 
 By modifying the scheduler's `start-time` you can force the reboot at
 different time.

fw-addr-lists.rsc

@@ -48,6 +48,12 @@
   }
   $WaitFullyConnected;
 
+  :if ([ :len [ /log/find where topics=({"script"; "warning"}) \
+      message=("\$LogPrintOnce: The message is already in log, scripting subsystem may have crashed before!") ] ] > 0) do={
+    $LogPrintOnce warning $ScriptName ("Scripting subsystem may have crashed, possibly caused by us. Delaying!");
+    :delay 5m;
+  }
+
   :local ListComment ("managed by " . $ScriptName);
 
   :foreach FwListName,FwList in=$FwAddrLists do={

global-functions.rsc

@@ -38,6 +38,7 @@
 :global ExitError;
 :global FetchHuge;
 :global FetchUserAgentStr;
+:global FileExists;
 :global FileGet;
 :global FormatLine;
 :global FormatMultiLines;
@@ -364,6 +365,7 @@
 
   :global CertificateAvailable;
   :global CleanFilePath;
+  :global FileExists;
   :global LogPrint;
   :global MkDir;
   :global RmFile;
@@ -384,7 +386,7 @@
     :return false;
   }
 
-  :if ([ :len [ /file/find where name=$PkgDest type="package" ] ] > 0) do={
+  :if ([ $FileExists $PkgDest "package" ] = true) do={
     $LogPrint info $0 ("Package file " . $PkgName . " already exists.");
     :return true;
   }
@@ -406,7 +408,7 @@
     :return false;
   }
 
-  :if ([ /file/get [ find where name=$PkgDest ] type ] != "package") do={
+  :if ([ $FileExists $PkgDest "package" ] = false) do={
     $LogPrint warning $0 ("Downloaded file is not a package, removing.");
     $RmFile $PkgDest;
     :return false;
@@ -530,10 +532,39 @@
     $Resource->"architecture-name" . " " . $Caller . "/Fetch (https://rsc.eworm.de/)");
 }
 
+# check for existence of file, optionally with type
+:set FileExists do={
+  :local FileName [ :tostr $1 ];
+  :local Type     [ :tostr $2 ];
+
+  :global FileGet;
+
+  :local FileVal [ $FileGet $FileName ];
+  :if ($FileVal = false) do={
+    :return false;
+  }
+
+  :if ([ :len ($FileVal->"size") ] = 0) do={
+    :return false;
+  }
+
+  :if ([ :len $Type ] = 0 || $FileVal->"type" = $Type) do={
+    :return true;
+  }
+
+  :return false;
+}
+
 # get file properties in array, or false on error
 :set FileGet do={
   :local FileName [ :tostr $1 ];
 
+  :global WaitForFile;
+
+  :if ([ $WaitForFile $FileName 0s ] = false) do={
+    :return false;
+  }
+
   :local FileVal false;
   :do {
     :set FileVal [ /file/get $FileName ];
@@ -1577,7 +1608,9 @@
 
   :global LogPrintOnce;
 
-  :local Symbols {
+  :global SymbolsExtra;
+
+  :local Symbols ({
     "abacus"="\F0\9F\A7\AE";
     "alarm-clock"="\E2\8F\B0";
     "arrow-down"="\E2\AC\87";
@@ -1612,7 +1645,7 @@
     "star"="\E2\AD\90";
     "warning-sign"="\E2\9A\A0";
     "white-heavy-check-mark"="\E2\9C\85"
-  }
+  }, $SymbolsExtra);
 
   :if ([ :len ($Symbols->$Name) ] = 0) do={
     $LogPrintOnce warning $0 ("No symbol available for name '" . $Name . "'!");
@@ -1746,15 +1779,25 @@
   :global MAX;
 
   :set FileName [ $CleanFilePath $FileName ];
-  :local I 1;
-  :local Delay ([ $MAX [ $EitherOr $WaitTime 2s ] 100ms ] / 10);
+  :local Delay ([ $MAX [ $EitherOr $WaitTime 2s ] 100ms ] / 9);
 
-  :while ([ :len [ /file/find where name=$FileName ] ] = 0) do={
-    :if ($I >= 10) do={
-      :return false;
-    }
+  :do {
+    :retry { 
+      :if ([ :len [ /file/find where name=$FileName ] ] = 0) do={
+        :error false;
+      }
+    } delay=$Delay max=10;
+  } on-error={
+    :return false;
+  }
+
+  :while ([ :len [ /file/find where name=$FileName ] ] > 0) do={
+    :do {
+      /file/get $FileName;
+      :return true;
+    } on-error={ }
     :delay $Delay;
-    :set I ($I + 1);
+    :set Delay ($Delay * 3 / 2);
   }
 
   :return false;

mod/notification-email.rsc

@@ -40,9 +40,11 @@
 
   :global EitherOr;
   :global EMailGenerateFrom;
+  :global FileExists;
   :global IsDNSResolving;
   :global IsTimeSync;
   :global LogPrint;
+  :global RmFile;
 
   :local AllDone true;
   :local QueueLen [ :len $EmailQueue ];
@@ -93,7 +95,7 @@
       :onerror Err {
         :local Attach ({});
         :foreach File in=[ :toarray [ $EitherOr ($Message->"attach") "" ] ] do={
-          :if ([ :len [ /file/find where name=$File ] ] = 1) do={
+          :if ([ $FileExists $File ] = true) do={
             :set Attach ($Attach, $File);
           } else={
             $LogPrint warning $0 ("File '" . $File . "' does not exist, can not attach.");
@@ -110,7 +112,7 @@
             :set Wait false;
             :if (($Message->"remove-attach") = true) do={
               :foreach File in=$Attach do={
-                /file/remove $File;
+                $RmFile $File;
               }
             }
           }

mod/ssh-keys-import.rsc

@@ -75,6 +75,7 @@
   :local User     [ :tostr $2 ];
 
   :global EitherOr;
+  :global FileExists;
   :global LogPrint;
   :global ParseKeyValueStore;
   :global SSHKeysImport;
@@ -84,8 +85,7 @@
     :return false;
   }
 
-  :local File [ /file/find where name=$FileName ];
-  :if ([ :len $File ] = 0) do={
+  :if ([ $FileExists $FileName ] = true) do={
     $LogPrint warning $0 ("File '" . $FileName . "' does not exist.");
     :return false;
   }

netwatch-dns.rsc

@@ -119,11 +119,13 @@
 
     :local Data false;
     :onerror Err {
-      :set Data ([ /tool/fetch check-certificate=yes-without-crl output=user \
-        http-header-field=({ "accept: application/dns-message" }) \
-        url=(($DohServer->"doh-url") . "?dns=" . [ :convert to=base64 ([ :rndstr length=2 ] . \
-        "\01\00" . "\00\01" . "\00\00" . "\00\00" . "\00\00" . "\09doh-check\05eworm\02de\00" . \
-        "\00\10" . "\00\01") ]) as-value ]->"data");
+      :retry {
+        :set Data ([ /tool/fetch check-certificate=yes-without-crl output=user \
+          http-header-field=({ "accept: application/dns-message" }) \
+          url=(($DohServer->"doh-url") . "?dns=" . [ :convert to=base64 ([ :rndstr length=2 ] . \
+          "\01\00" . "\00\01" . "\00\00" . "\00\00" . "\00\00" . "\09doh-check\05eworm\02de\00" . \
+          "\00\10" . "\00\01") ]) as-value ]->"data");
+      } delay=1s max=3;
     } do={
       $LogPrint warning $ScriptName ("Request to DoH server " . ($DohServer->"doh-url") . \
           " failed: " . $Err);

netwatch-notify.rsc

@@ -62,14 +62,14 @@
 
     :local FwAddrList ($ScriptName . "-" . [ $GetRandom20CharAlNum ]);
     :if ([ :typeof [ :toip $Expected ] ] = "ip") do={
-      /ip/firewall/address-list/add address=$Name list=$FwAddrList dynamic=yes timeout=10s;
+      /ip/firewall/address-list/add address=$Name list=$FwAddrList dynamic=yes timeout=30s;
       :delay 20ms;
       :if ([ :len [ /ip/firewall/address-list/find where list=$FwAddrList address=$Expected ] ] > 0) do={
         :return true;
       }
     }
     :if ([ :typeof [ :toip6 $Expected ] ] = "ip6") do={
-      /ipv6/firewall/address-list/add address=$Name list=$FwAddrList dynamic=yes timeout=10s;
+      /ipv6/firewall/address-list/add address=$Name list=$FwAddrList dynamic=yes timeout=30s;
       :delay 20ms;
       :if ([ :len [ /ipv6/firewall/address-list/find where list=$FwAddrList address=$Expected ] ] > 0) do={
         :return true;
@@ -127,7 +127,7 @@
               $LogPrint [ $IfThenElse ($HostInfo->"no-resolve-fail" != true) warning debug ] \
                   $ScriptName ("Resolving name '" . $HostInfo->"resolve" . [ $IfThenElse \
                   ($HostInfo->"resolve" != $HostInfo->"name") ("' for " . $Type . " '" . \
-                  $HostInfo->"name") "" ] . "' failed: " . $Err);
+                  $HostInfo->"name") "" ] . "' failed third time: " . $Err);
             }
           }
         }

packages-update.rsc

@@ -31,19 +31,25 @@
   :local Schedule do={
     :local ScriptName [ :tostr $1 ];
 
+    :global PackagesUpdateDeferReboot;
+
     :global GetRandomNumber;
+    :global IfThenElse;
     :global LogPrint;
 
     :global RebootForUpdate do={
       /system/reboot;
     }
 
+    :local Interval [ $IfThenElse ([ :totime $PackagesUpdateDeferReboot ] >= 1d) \
+        $PackagesUpdateDeferReboot 1d ];
     :local StartTime [ :tostr [ :totime (10800 + [ $GetRandomNumber 7200 ]) ] ];
-    /system/scheduler/add name="_RebootForUpdate" start-time=$StartTime interval=1d \
+    /system/scheduler/add name="_RebootForUpdate" start-time=$StartTime interval=$Interval \
         on-event=("/system/scheduler/remove \"_RebootForUpdate\"; " . \
         ":global RebootForUpdate; \$RebootForUpdate;");
     $LogPrint info $ScriptName ("Scheduled reboot for update at " . $StartTime . \
-        " local time (" . [ /system/clock/get time-zone-name ] . ").");
+        " local time (" . [ /system/clock/get time-zone-name ] . ")" . \
+        [ $IfThenElse ($Interval > 1d) (" deferred by " . $Interval) ] . ".");
     :return true;
   }
 
@@ -153,7 +159,7 @@
       :error true;
     }
   } else={
-    :if ($PackagesUpdateDeferReboot = true) do={
+    :if ($PackagesUpdateDeferReboot = true || [ :totime $PackagesUpdateDeferReboot ] >= 1d) do={
       $Schedule $ScriptName;
       :set ExitOK true;
       :error true;

telegram-chat.rsc

@@ -30,6 +30,7 @@
   :global CertificateAvailable;
   :global EitherOr;
   :global EscapeForRegEx;
+  :global FileExists;
   :global GetRandom20CharAlNum;
   :global IfThenElse;
   :global LogPrint;
@@ -154,7 +155,7 @@
             :if ([ $WaitForFile ($File . ".done") [ $EitherOr $TelegramChatRunTime 20s ] ] = false) do={
               :set State ([ $SymbolForNotification "warning-sign" ] . "The command did not finish, still running in background.\n\n");
             }
-            :if ([ :len [ /file/find where name=($File . ".failed") ] ] > 0) do={
+            :if ([ $FileExists ($File . ".failed") ] = true) do={
               :set State ([ $SymbolForNotification "cross-mark" ] . "The command failed with an error!\n\n");
             }
             :local Content ([ /file/read chunk-size=32768 file=$File as-value ]->"data");