Restrict the depth of the category tree shown inside KiCAD to improve performance

The depth can be controlled via the EDA_KICAD_CATEGORY_DEPTH env

.docker/symfony.conf

@@ -42,6 +42,7 @@
         PassEnv PROVIDER_TME_KEY PROVIDER_TME_SECRET PROVIDER_TME_CURRENCY PROVIDER_TME_LANGUAGE PROVIDER_TME_COUNTRY PROVIDER_TME_GET_GROSS_PRICES
         PassEnv PROVIDER_OCTOPART_CLIENT_ID PROVIDER_OCTOPART_SECRET PROVIDER_OCTOPART_CURRENCY PROVIDER_OCTOPART_COUNTRY PROVIDER_OCTOPART_SEARCH_LIMIT PROVIDER_OCTOPART_ONLY_AUTHORIZED_SELLERS
         PassEnv PROVIDER_MOUSER_KEY PROVIDER_MOUSER_SEARCH_OPTION PROVIDER_MOUSER_SEARCH_LIMIT PROVIDER_MOUSER_SEARCH_WITH_SIGNUP_LANGUAGE
+        PassEnv EDA_KICAD_CATEGORY_DEPTH
 
         # For most configuration files from conf-available/, which are
         # enabled or disabled at a global level, it is possible to

.env

@@ -159,6 +159,14 @@ PROVIDER_MOUSER_SEARCH_LIMIT=50
 # Used when searching for keywords in the language specified when you signed up for Search API.
 PROVIDER_MOUSER_SEARCH_WITH_SIGNUP_LANGUAGE='true'
 
+##################################################################################
+# EDA integration related settings
+##################################################################################
+
+# This value determines the depth of the category tree, that is visible inside KiCad
+# 0 means that only the top level categories are visible. Set to a value > 0 to show more levels.
+EDA_KICAD_CATEGORY_DEPTH=0
+
 ###################################################################################
 # SAML Single sign on-settings
 ###################################################################################

config/parameters.yaml

@@ -142,3 +142,4 @@ parameters:
   env(HISTORY_SAVE_REMOVED_DATA): 1
   env(HISTORY_SAVE_NEW_DATA): 1
 
+  env(EDA_KICAD_CATEGORY_DEPTH): 0

config/services.yaml

@@ -316,6 +316,13 @@ services:
             $global_locale: '%partdb.locale%'
             $global_timezone: '%partdb.timezone%'
 
+    ####################################################################################################################
+    # EDA system
+    ####################################################################################################################
+    App\Services\EDA\KiCadHelper:
+        arguments:
+            $category_depth: '%env(int:EDA_KICAD_CATEGORY_DEPTH)%'
+
     ####################################################################################################################
     # Symfony overrides
     ####################################################################################################################

src/Controller/KiCadApiController.php

@@ -25,7 +25,7 @@ namespace App\Controller;
 
 use App\Entity\Parts\Category;
 use App\Entity\Parts\Part;
-use App\Services\EDAIntegration\KiCADHelper;
+use App\Services\EDA\KiCadHelper;
 use App\Services\Trees\NodesListBuilder;
 use Doctrine\ORM\EntityManagerInterface;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
@@ -36,7 +36,7 @@ use Symfony\Component\Routing\Annotation\Route;
 class KiCadApiController extends AbstractController
 {
     public function __construct(
-        private readonly KiCADHelper $kiCADHelper,
+        private readonly KiCadHelper $kiCADHelper,
     )
     {
     }

src/Repository/AbstractPartsContainingRepository.php

@@ -64,6 +64,11 @@ abstract class AbstractPartsContainingRepository extends StructuralDBElementRepo
         return $this->getPartsCountRecursiveWithDepthN($element, self::RECURSION_LIMIT);
     }
 
+    public function getPartsRecursive(AbstractPartsContainingDBElement $element): array
+    {
+        return $this->getPartsRecursiveWithDepthN($element, self::RECURSION_LIMIT);
+    }
+
     /**
      * The implementation of the recursive function to get the parts count.
      * This function is used to limit the recursion depth (remaining_depth is decreased on each call).
@@ -91,6 +96,23 @@ abstract class AbstractPartsContainingRepository extends StructuralDBElementRepo
         return $count;
     }
 
+    protected function getPartsRecursiveWithDepthN(AbstractPartsContainingDBElement $element, int $remaining_depth): array
+    {
+        if ($remaining_depth <= 0) {
+            throw new \RuntimeException('Recursion limit reached!');
+        }
+
+        //Add direct parts
+        $parts = $this->getParts($element);
+
+        //Then iterate over all children and add their parts
+        foreach ($element->getChildren() as $child) {
+            $parts = array_merge($parts, $this->getPartsRecursiveWithDepthN($child, $remaining_depth - 1));
+        }
+
+        return $parts;
+    }
+
     protected function getPartsByField(object $element, array $order_by, string $field_name): array
     {
         if (!$element instanceof AbstractPartsContainingDBElement) {

src/Services/EDA/KiCadHelper.php

@@ -21,7 +21,7 @@
 declare(strict_types=1);
 
 
-namespace App\Services\EDAIntegration;
+namespace App\Services\EDA;
 
 use App\Entity\Parts\Category;
 use App\Entity\Parts\Part;
@@ -33,7 +33,7 @@ use Symfony\Contracts\Cache\ItemInterface;
 use Symfony\Contracts\Cache\TagAwareCacheInterface;
 use Symfony\Contracts\Translation\TranslatorInterface;
 
-class KiCADHelper
+class KiCadHelper
 {
 
     public function __construct(
@@ -43,7 +43,11 @@ class KiCADHelper
         private readonly ElementCacheTagGenerator $tagGenerator,
         private readonly UrlGeneratorInterface $urlGenerator,
         private readonly TranslatorInterface $translator,
+        private readonly int $category_depth,
     ) {
+        if ($this->category_depth < 0) {
+            throw new \InvalidArgumentException('The category depth must be greater than or equal to 0');
+        }
     }
 
     /**
@@ -55,7 +59,7 @@ class KiCADHelper
      */
     public function getCategories(): array
     {
-        return $this->kicadCache->get('kicad_categories', function (ItemInterface $item) {
+        return $this->kicadCache->get('kicad_categories_' . $this->category_depth, function (ItemInterface $item) {
             //Invalidate the cache on category changes
             $secure_class_name = $this->tagGenerator->getElementTypeCacheTag(Category::class);
             $item->tag($secure_class_name);
@@ -69,9 +73,17 @@ class KiCADHelper
                     continue;
                 }
 
+                //Skip categories with a depth greater than the configured one
+                if ($category->getLevel() > $this->category_depth) {
+                    continue;
+                }
+
                 /** @var $category Category */
                 //Ensure that the category contains parts
-                if ($repo->getPartsCount($category) < 1) {
+                //For the last level, we need to use a recursive query, otherwise we can use a simple query
+                $parts_count = $category->getLevel() >= $this->category_depth ? $repo->getPartsCountRecursive($category) : $repo->getPartsCount($category);
+
+                if ($parts_count < 1) {
                     continue;
                 }
 
@@ -94,7 +106,7 @@ class KiCADHelper
      */
     public function getCategoryParts(Category $category): array
     {
-        return $this->kicadCache->get('kicad_category_parts_'.$category->getID(),
+        return $this->kicadCache->get('kicad_category_parts_'.$category->getID() . '_' . $this->category_depth,
             function (ItemInterface $item) use ($category) {
                 $item->tag([
                     $this->tagGenerator->getElementTypeCacheTag(Category::class),
@@ -102,7 +114,13 @@ class KiCADHelper
                 ]);
 
                 $category_repo = $this->em->getRepository(Category::class);
-                $parts = $category_repo->getParts($category);
+                if ($category->getLevel() >= $this->category_depth) {
+                    //Get all parts for the category and its children
+                    $parts = $category_repo->getPartsRecursive($category);
+                } else {
+                    //Get only direct parts for the category (without children), as the category is not collapsed
+                    $parts = $category_repo->getParts($category);
+                }
 
                 $result = [];
                 foreach ($parts as $part) {