Do not cache entities directly in NodesListBuilder but cache only the IDs instead

Otherwise the doctrine proxies break, and we get issues with loading the preview_images in structural Elements.

src/Entity/Attachments/AttachmentContainingDBElement.php

@@ -26,6 +26,7 @@ use App\Entity\Base\AbstractNamedDBElement;
 use App\Entity\Base\MasterAttachmentTrait;
 use App\Entity\Contracts\HasAttachmentsInterface;
 use App\Entity\Contracts\HasMasterAttachmentInterface;
+use App\Repository\AttachmentContainingDBElementRepository;
 use Doctrine\Common\Collections\ArrayCollection;
 use Doctrine\Common\Collections\Collection;
 use Doctrine\ORM\Mapping as ORM;
@@ -34,7 +35,7 @@ use Symfony\Component\Serializer\Annotation\Groups;
 /**
  * @template-covariant AT of Attachment
  */
-#[ORM\MappedSuperclass]
+#[ORM\MappedSuperclass(repositoryClass: AttachmentContainingDBElementRepository::class)]
 abstract class AttachmentContainingDBElement extends AbstractNamedDBElement implements HasMasterAttachmentInterface, HasAttachmentsInterface
 {
     use MasterAttachmentTrait;

src/Entity/Base/MasterAttachmentTrait.php

@@ -35,6 +35,8 @@ trait MasterAttachmentTrait
      * @var Attachment|null
      * Mapping is done in the subclasses (e.g. Part), like with the attachments.
      * If this is done here (which is possible in theory), the attachment is not lazy loaded anymore, which causes unnecessary overhead.
+     *
+     * !!! If you change this name, you have to change it in the fetchHint in the AttachmentContainingDBElementRepository (getElementsAndPreviewAttachmentByIDs()) too !!!
      */
     #[Assert\Expression('value == null or value.isPicture()', message: 'part.master_attachment.must_be_picture')]
     protected ?Attachment $master_picture_attachment = null;

src/Repository/AttachmentContainingDBElementRepository.php

@@ -0,0 +1,72 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2023 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+
+namespace App\Repository;
+
+use App\Entity\Attachments\AttachmentContainingDBElement;
+use Doctrine\ORM\Mapping\ClassMetadataInfo;
+
+/**
+ * @template TEntityClass of AttachmentContainingDBElement
+ * @extends NamedDBElementRepository<TEntityClass>
+ */
+class AttachmentContainingDBElementRepository extends NamedDBElementRepository
+{
+    /**
+     * @var array This array is used to cache the results of getElementsAndPreviewAttachmentByIDs function.
+     */
+    private array $elementsAndPreviewAttachmentCache = [];
+
+    /**
+     * Similar to the findByIDInMatchingOrder function, but it also hints to doctrine that the master picture attachment should be fetched eagerly.
+     * @param  array  $ids
+     * @return array
+     */
+    public function getElementsAndPreviewAttachmentByIDs(array $ids): array
+    {
+        //Convert the ids to a string
+        $cache_key = implode(',', $ids);
+
+        //Check if the result is already cached
+        if (isset($this->elementsAndPreviewAttachmentCache[$cache_key])) {
+            return $this->elementsAndPreviewAttachmentCache[$cache_key];
+        }
+
+        $qb = $this->createQueryBuilder('element');
+        $q = $qb->select('element')
+            ->where('element.id IN (?1)')
+            ->setParameter(1, $ids)
+            ->getQuery();
+
+        $q->setFetchMode($this->getEntityName(), 'master_picture_attachment', ClassMetadataInfo::FETCH_EAGER);
+
+        $result = $q->getResult();
+        $result = array_combine($ids, $result);
+        $result = array_map(fn ($id) => $result[$id], $ids);
+
+        //Cache the result
+        $this->elementsAndPreviewAttachmentCache[$cache_key] = $result;
+
+        return $result;
+    }
+}

src/Repository/DBElementRepository.php

@@ -51,6 +51,8 @@ use ReflectionClass;
  */
 class DBElementRepository extends EntityRepository
 {
+    private array $find_elements_by_id_cache = [];
+
     /**
      * Changes the ID of the given element to a new value.
      * You should only use it to undelete former existing elements, everything else is most likely a bad idea!
@@ -91,6 +93,38 @@ class DBElementRepository extends EntityRepository
         return $q->getResult();
     }
 
+    /**
+     * Returns the elements with the given IDs in the same order, as they were given in the input array.
+     *
+     * @param  array  $ids
+     * @return array
+     */
+    public function findByIDInMatchingOrder(array $ids): array
+    {
+        $cache_key = implode(',', $ids);
+
+        //Check if the result is already cached
+        if (isset($this->find_elements_by_id_cache[$cache_key])) {
+            return $this->find_elements_by_id_cache[$cache_key];
+        }
+
+        //Otherwise do the query
+        $qb = $this->createQueryBuilder('element');
+        $q = $qb->select('element')
+            ->where('element.id IN (?1)')
+            ->setParameter(1, $ids)
+            ->getQuery();
+
+        $result = $q->getResult();
+        $result = array_combine($ids, $result);
+        $result = array_map(fn ($id) => $result[$id], $ids);
+
+        //Cache the result
+        $this->find_elements_by_id_cache[$cache_key] = $result;
+
+        return $result;
+    }
+
     protected function setField(AbstractDBElement $element, string $field, int $new_value): void
     {
         $reflection = new ReflectionClass($element::class);

src/Repository/NamedDBElementRepository.php

@@ -65,11 +65,11 @@ class NamedDBElementRepository extends DBElementRepository
     }
 
     /**
-     * Returns the list of all nodes to use in a select box.
+     * Returns a flattened list of all nodes.
      * @return AbstractNamedDBElement[]
      * @phpstan-return array<int, AbstractNamedDBElement>
      */
-    public function toNodesList(): array
+    public function getFlatList(): array
     {
         //All nodes are sorted by name
         return $this->findBy([], ['name' => 'ASC']);

src/Repository/StructuralDBElementRepository.php

@@ -32,7 +32,7 @@ use RecursiveIteratorIterator;
  * @template TEntityClass of AbstractStructuralDBElement
  * @extends NamedDBElementRepository<TEntityClass>
  */
-class StructuralDBElementRepository extends NamedDBElementRepository
+class StructuralDBElementRepository extends AttachmentContainingDBElementRepository
 {
     /**
      * @var array An array containing all new entities created by getNewEntityByPath.
@@ -85,7 +85,7 @@ class StructuralDBElementRepository extends NamedDBElementRepository
      * @return AbstractStructuralDBElement[] a flattened list containing the tree elements
      * @phpstan-return array<int, TEntityClass>
      */
-    public function toNodesList(?AbstractStructuralDBElement $parent = null): array
+    public function getFlatList(?AbstractStructuralDBElement $parent = null): array
     {
         $result = [];
 

src/Services/Trees/NodesListBuilder.php

@@ -22,7 +22,12 @@ declare(strict_types=1);
 
 namespace App\Services\Trees;
 
+use App\Entity\Attachments\AttachmentContainingDBElement;
+use App\Entity\Base\AbstractDBElement;
+use App\Entity\Base\AbstractNamedDBElement;
 use App\Entity\Base\AbstractStructuralDBElement;
+use App\Repository\AttachmentContainingDBElementRepository;
+use App\Repository\DBElementRepository;
 use App\Repository\StructuralDBElementRepository;
 use App\Services\UserSystem\UserCacheKeyGenerator;
 use Doctrine\ORM\EntityManagerInterface;
@@ -49,6 +54,31 @@ class NodesListBuilder
      * @return AbstractStructuralDBElement[] a flattened list containing the tree elements
      */
     public function typeToNodesList(string $class_name, ?AbstractStructuralDBElement $parent = null): array
+    {
+        /**
+         * We can not cache the entities directly, because loading them from cache will break the doctrine proxies.
+         */
+        //Retrieve the IDs of the elements
+        $ids = $this->getFlattenedIDs($class_name, $parent);
+
+        //Retrieve the elements from the IDs, the order is the same as in the $ids array
+        /** @var DBElementRepository $repo */
+        $repo = $this->em->getRepository($class_name);
+
+        if ($repo instanceof AttachmentContainingDBElementRepository) {
+            return $repo->getElementsAndPreviewAttachmentByIDs($ids);
+        }
+
+        return $repo->getElementsFromIDArray($ids);
+    }
+
+    /**
+     * This functions returns the (cached) list of the IDs of the elements for the flattened tree.
+     * @param  string  $class_name
+     * @param  AbstractStructuralDBElement|null  $parent
+     * @return array
+     */
+    private function getFlattenedIDs(string $class_name, ?AbstractStructuralDBElement $parent = null): array
     {
         $parent_id = $parent instanceof AbstractStructuralDBElement ? $parent->getID() : '0';
         // Backslashes are not allowed in cache keys
@@ -58,10 +88,11 @@ class NodesListBuilder
         return $this->cache->get($key, function (ItemInterface $item) use ($class_name, $parent, $secure_class_name) {
             // Invalidate when groups, an element with the class or the user changes
             $item->tag(['groups', 'tree_list', $this->keyGenerator->generateKey(), $secure_class_name]);
+
             /** @var StructuralDBElementRepository $repo */
             $repo = $this->em->getRepository($class_name);
 
-            return $repo->toNodesList($parent);
+            return array_map(fn(AbstractDBElement $element) => $element->getID(), $repo->getFlatList($parent));
         });
     }