Allow to emulate natural sorting on SQLite databases

.env

@@ -23,6 +23,10 @@ DATABASE_MYSQL_USE_SSL_CA=0
 # Only do this, if you know what you are doing!
 DATABASE_MYSQL_SSL_VERIFY_CERT=1
 
+# Emulate natural sorting of strings even on databases that do not support it (like SQLite, MySQL or MariaDB < 10.7)
+# This can be slow on big databases
+DATABASE_EMULATE_NATURAL_SORT=0
+
 ###################################################################################
 # General settings
 ###################################################################################

config/parameters.yaml

@@ -16,6 +16,8 @@ parameters:
 
   partdb.default_uri: '%env(string:DEFAULT_URI)%'             # The default URI to use for the Part-DB instance (e.g. https://part-db.example.com/). This is used for generating links in emails
 
+  partdb.db.emulate_natural_sort: '%env(bool:DATABASE_EMULATE_NATURAL_SORT)%' # If this is set to true, natural sorting is emulated on platforms that do not support it natively. This can be slow on large datasets.
+
   ######################################################################################################################
   # Users and Privacy
   ######################################################################################################################
@@ -145,3 +147,5 @@ parameters:
   env(HISTORY_SAVE_NEW_DATA): 1
 
   env(EDA_KICAD_CATEGORY_DEPTH): 0
+
+  env(DATABASE_EMULATE_NATURAL_SORT): 0

src/Doctrine/Functions/Natsort.php

@@ -27,6 +27,7 @@ use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Driver\AbstractPostgreSQLDriver;
 use Doctrine\DBAL\Platforms\MariaDBPlatform;
 use Doctrine\DBAL\Platforms\PostgreSQLPlatform;
+use Doctrine\DBAL\Platforms\SQLitePlatform;
 use Doctrine\ORM\Query\AST\Functions\FunctionNode;
 use Doctrine\ORM\Query\AST\Node;
 use Doctrine\ORM\Query\Parser;
@@ -39,6 +40,19 @@ class Natsort extends FunctionNode
 
     private static ?bool $supportsNaturalSort = null;
 
+    private static bool $allowSlowNaturalSort = false;
+
+    /**
+     * As we can not inject parameters into the function, we use an event listener, to call the value on the static function.
+     * This is the only way to inject the value into the function.
+     * @param  bool  $allow
+     * @return void
+     */
+    public static function allowSlowNaturalSort(bool $allow = true): void
+    {
+        self::$allowSlowNaturalSort = $allow;
+    }
+
     /**
      * Check if the MariaDB version which is connected to supports the natural sort (meaning it has a version of 10.7.0 or higher)
      * The result is cached in memory.
@@ -84,6 +98,14 @@ class Natsort extends FunctionNode
             return 'NATURAL_SORT_KEY(' . $this->field->dispatch($sqlWalker) . ')';
         }
 
+        //Do the following operations only if we allow slow natural sort
+        if (self::$allowSlowNaturalSort) {
+
+            if ($platform instanceof SQLitePlatform) {
+                return $this->field->dispatch($sqlWalker).' COLLATE NATURAL_CMP';
+            }
+        }
+
          //For every other platform, return the field as is
         return $this->field->dispatch($sqlWalker);
     }

src/Doctrine/Middleware/SQLiteRegexExtensionMiddlewareDriver.php

@@ -48,6 +48,11 @@ class SQLiteRegexExtensionMiddlewareDriver extends AbstractDriverMiddleware
                 $native_connection->sqliteCreateFunction('REGEXP', self::regexp(...), 2, \PDO::SQLITE_DETERMINISTIC);
                 $native_connection->sqliteCreateFunction('FIELD', self::field(...), -1, \PDO::SQLITE_DETERMINISTIC);
                 $native_connection->sqliteCreateFunction('FIELD2', self::field2(...), 2, \PDO::SQLITE_DETERMINISTIC);
+
+                //Create a new collation for natural sorting
+                if (method_exists($native_connection, 'sqliteCreateCollation')) {
+                    $native_connection->sqliteCreateCollation('NATURAL_CMP', strnatcmp(...));
+                }
             }
         }
 

src/EventListener/AllowSlowNaturalSortListener.php

@@ -0,0 +1,49 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2024 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+
+namespace App\EventListener;
+
+use App\Doctrine\Functions\Natsort;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+use Symfony\Component\EventDispatcher\Attribute\AsEventListener;
+use Symfony\Component\HttpKernel\Event\RequestEvent;
+
+/**
+ * This is a workaround to the fact that we can not inject parameters into doctrine custom functions.
+ * Therefore we use this event listener to call the static function on the custom function, to inject the value, before
+ * any NATSORT function is called.
+ */
+#[AsEventListener]
+class AllowSlowNaturalSortListener
+{
+    public function __construct(
+        #[Autowire(param: 'partdb.db.emulate_natural_sort')]
+        private readonly bool $allowNaturalSort)
+    {
+    }
+
+    public function __invoke(RequestEvent $event)
+    {
+        Natsort::allowSlowNaturalSort($this->allowNaturalSort);
+    }
+}