Merge branch 'reichelt_provider'

.docker/symfony.conf

@@ -44,6 +44,7 @@
         PassEnv PROVIDER_MOUSER_KEY PROVIDER_MOUSER_SEARCH_OPTION PROVIDER_MOUSER_SEARCH_LIMIT PROVIDER_MOUSER_SEARCH_WITH_SIGNUP_LANGUAGE
         PassEnv PROVIDER_LCSC_ENABLED PROVIDER_LCSC_CURRENCY
         PassEnv PROVIDER_OEMSECRETS_KEY PROVIDER_OEMSECRETS_COUNTRY_CODE PROVIDER_OEMSECRETS_CURRENCY PROVIDER_OEMSECRETS_ZERO_PRICE PROVIDER_OEMSECRETS_SET_PARAM PROVIDER_OEMSECRETS_SORT_CRITERIA
+        PassEnv PROVIDER_REICHELT_ENABLED PROVIDER_REICHELT_CURRENCY PROVIDER_REICHELT_COUNTRY PROVIDER_REICHELT_LANGUAGE PROVIDER_REICHELT_INCLUDE_VAT
         PassEnv EDA_KICAD_CATEGORY_DEPTH
 
         # For most configuration files from conf-available/, which are

.env

@@ -216,6 +216,22 @@ PROVIDER_OEMSECRETS_SET_PARAM=1
 #If unset or set to any other value, no sorting is performed.
 PROVIDER_OEMSECRETS_SORT_CRITERIA=C
 
+
+# Reichelt provider:
+# Reichelt.com offers no official API, so this info provider webscrapes the website to extract info
+# It could break at any time, use it at your own risk
+
+# We dont require an API key for Reichelt, just set this to 1 to enable Reichelt support
+PROVIDER_REICHELT_ENABLED=0
+# The country to get prices for
+PROVIDER_REICHELT_COUNTRY=DE
+# The language to get results in (en, de, fr, nl, pl, it, es)
+PROVIDER_REICHELT_LANGUAGE=en
+# Include VAT in prices (set to 1 to include VAT, 0 to exclude VAT)
+PROVIDER_REICHELT_INCLUDE_VAT=1
+# The currency to get prices in (only for countries with countries other than EUR)
+PROVIDER_REICHELT_CURRENCY=EUR
+
 ##################################################################################
 # EDA integration related settings
 ##################################################################################

composer.json

@@ -54,6 +54,8 @@
         "symfony/apache-pack": "^1.0",
         "symfony/asset": "6.4.*",
         "symfony/console": "6.4.*",
+        "symfony/css-selector": "6.4.*",
+        "symfony/dom-crawler": "6.4.*",
         "symfony/dotenv": "6.4.*",
         "symfony/expression-language": "6.4.*",
         "symfony/flex": "^v2.3.1",
@@ -104,7 +106,6 @@
         "rector/rector": "^2.0.4",
         "roave/security-advisories": "dev-latest",
         "symfony/browser-kit": "6.4.*",
-        "symfony/css-selector": "6.4.*",
         "symfony/debug-bundle": "6.4.*",
         "symfony/maker-bundle": "^1.13",
         "symfony/phpunit-bridge": "6.4.*",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "ca8701d95e24bae5d28ccdcfe242e8e4",
+    "content-hash": "75643d42e05fce4684644d375bff2d0a",
     "packages": [
         {
             "name": "amphp/amp",
@@ -8984,6 +8984,73 @@
             ],
             "time": "2025-01-25T08:04:58+00:00"
         },
+        {
+            "name": "symfony/dom-crawler",
+            "version": "v6.4.18",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/dom-crawler.git",
+                "reference": "fd07959d3e8992795029bdab3605c2e8e895034e"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/fd07959d3e8992795029bdab3605c2e8e895034e",
+                "reference": "fd07959d3e8992795029bdab3605c2e8e895034e",
+                "shasum": ""
+            },
+            "require": {
+                "masterminds/html5": "^2.6",
+                "php": ">=8.1",
+                "symfony/polyfill-ctype": "~1.8",
+                "symfony/polyfill-mbstring": "~1.0"
+            },
+            "require-dev": {
+                "symfony/css-selector": "^5.4|^6.0|^7.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Symfony\\Component\\DomCrawler\\": ""
+                },
+                "exclude-from-classmap": [
+                    "/Tests/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Fabien Potencier",
+                    "email": "fabien@symfony.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Eases DOM navigation for HTML and XML documents",
+            "homepage": "https://symfony.com",
+            "support": {
+                "source": "https://github.com/symfony/dom-crawler/tree/v6.4.18"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2025-01-09T15:35:00+00:00"
+        },
         {
             "name": "symfony/dotenv",
             "version": "v6.4.16",
@@ -18497,73 +18564,6 @@
             ],
             "time": "2024-09-25T14:18:03+00:00"
         },
-        {
-            "name": "symfony/dom-crawler",
-            "version": "v6.4.18",
-            "source": {
-                "type": "git",
-                "url": "https://github.com/symfony/dom-crawler.git",
-                "reference": "fd07959d3e8992795029bdab3605c2e8e895034e"
-            },
-            "dist": {
-                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/fd07959d3e8992795029bdab3605c2e8e895034e",
-                "reference": "fd07959d3e8992795029bdab3605c2e8e895034e",
-                "shasum": ""
-            },
-            "require": {
-                "masterminds/html5": "^2.6",
-                "php": ">=8.1",
-                "symfony/polyfill-ctype": "~1.8",
-                "symfony/polyfill-mbstring": "~1.0"
-            },
-            "require-dev": {
-                "symfony/css-selector": "^5.4|^6.0|^7.0"
-            },
-            "type": "library",
-            "autoload": {
-                "psr-4": {
-                    "Symfony\\Component\\DomCrawler\\": ""
-                },
-                "exclude-from-classmap": [
-                    "/Tests/"
-                ]
-            },
-            "notification-url": "https://packagist.org/downloads/",
-            "license": [
-                "MIT"
-            ],
-            "authors": [
-                {
-                    "name": "Fabien Potencier",
-                    "email": "fabien@symfony.com"
-                },
-                {
-                    "name": "Symfony Community",
-                    "homepage": "https://symfony.com/contributors"
-                }
-            ],
-            "description": "Eases DOM navigation for HTML and XML documents",
-            "homepage": "https://symfony.com",
-            "support": {
-                "source": "https://github.com/symfony/dom-crawler/tree/v6.4.18"
-            },
-            "funding": [
-                {
-                    "url": "https://symfony.com/sponsor",
-                    "type": "custom"
-                },
-                {
-                    "url": "https://github.com/fabpot",
-                    "type": "github"
-                },
-                {
-                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
-                    "type": "tidelift"
-                }
-            ],
-            "time": "2025-01-09T15:35:00+00:00"
-        },
         {
             "name": "symfony/maker-bundle",
             "version": "v1.62.1",

src/Services/InfoProviderSystem/DTOs/ParameterDTO.php

@@ -72,9 +72,9 @@ class ParameterDTO
                 group: $group);
         }
 
-        //If the attribute contains "..." or a tilde we assume it is a range
+        //If the attribute contains ".." or "..." or a tilde we assume it is a range
-        if (preg_match('/(\.{3}|~)/', $value) === 1) {
+        if (preg_match('/(\.{2,3}|~)/', $value) === 1) {
-            $parts = preg_split('/\s*(\.{3}|~)\s*/', $value);
+            $parts = preg_split('/\s*(\.{2,3}|~)\s*/', $value);
             if (count($parts) === 2) {
                 //Try to extract number and unit from value (allow leading +)
                 if ($unit === null || trim($unit) === '') {

src/Services/InfoProviderSystem/PartInfoRetriever.php

@@ -33,8 +33,8 @@ use Symfony\Contracts\Cache\ItemInterface;
 final class PartInfoRetriever
 {
 
-    private const CACHE_DETAIL_EXPIRATION = 60 * 60 * 24 * 4; // 4 days
+    private const CACHE_DETAIL_EXPIRATION = 5; // 4 days
-    private const CACHE_RESULT_EXPIRATION = 60 * 60 * 24 * 7; // 7 days
+    private const CACHE_RESULT_EXPIRATION = 5; // 7 days
 
     public function __construct(private readonly ProviderRegistry $provider_registry,
         private readonly DTOtoEntityConverter $dto_to_entity_converter, private readonly CacheInterface $partInfoCache)

src/Services/InfoProviderSystem/Providers/LCSCProvider.php

@@ -76,7 +76,7 @@ class LCSCProvider implements InfoProviderInterface
                 'Cookie' => new Cookie('currencyCode', $this->currency)
             ],
             'query' => [
-                'productCode' => $id,
+                'prductCode' => $id,
             ],
         ]);
 

src/Services/InfoProviderSystem/Providers/ReicheltProvider.php

@@ -0,0 +1,285 @@
+<?php
+/*
+ * This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
+ *
+ *  Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published
+ *  by the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+declare(strict_types=1);
+
+
+namespace App\Services\InfoProviderSystem\Providers;
+
+use App\Services\InfoProviderSystem\DTOs\FileDTO;
+use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
+use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
+use App\Services\InfoProviderSystem\DTOs\PriceDTO;
+use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO;
+use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
+use Symfony\Component\DependencyInjection\Attribute\Autowire;
+use Symfony\Component\DomCrawler\Crawler;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+class ReicheltProvider implements InfoProviderInterface
+{
+
+    public const DISTRIBUTOR_NAME = "Reichelt";
+
+    public function __construct(private readonly HttpClientInterface $client,
+        #[Autowire(env: "bool:PROVIDER_REICHELT_ENABLED")]
+        private readonly bool $enabled = true,
+        #[Autowire(env: "PROVIDER_REICHELT_LANGUAGE")]
+        private readonly string $language = "en",
+        #[Autowire(env: "PROVIDER_REICHELT_COUNTRY")]
+        private readonly string $country = "DE",
+        #[Autowire(env: "PROVIDER_REICHELT_INCLUDE_VAT")]
+        private readonly bool $includeVAT = false,
+        #[Autowire(env: "PROVIDER_REICHELT_CURRENCY")]
+        private readonly string $currency = "EUR",
+    )
+    {
+    }
+
+    public function getProviderInfo(): array
+    {
+        return [
+            'name' => 'Reichelt',
+            'description' => 'Webscrapping from reichelt.com to get part information',
+            'url' => 'https://www.reichelt.com/',
+            'disabled_help' => 'TODO'
+        ];
+    }
+
+    public function getProviderKey(): string
+    {
+        return 'reichelt';
+    }
+
+    public function isActive(): bool
+    {
+        return $this->enabled;
+    }
+
+    public function searchByKeyword(string $keyword): array
+    {
+        $response = $this->client->request('GET', sprintf($this->getBaseURL() . '/shop/search/%s', $keyword));
+        $html = $response->getContent();
+
+        //Parse the HTML and return the results
+        $dom = new Crawler($html);
+        //Iterate over all div.al_gallery_article elements
+        $results = [];
+        $dom->filter('div.al_gallery_article')->each(function (Crawler $element) use (&$results) {
+
+            //Extract product id from data-product attribute
+            $artId = json_decode($element->attr('data-product'), true, 2, JSON_THROW_ON_ERROR)['artid'];
+
+            $productID = $element->filter('meta[itemprop="productID"]')->attr('content');
+            $name = $element->filter('meta[itemprop="name"]')->attr('content');
+            $sku = $element->filter('meta[itemprop="sku"]')->attr('content');
+
+            //Try to extract a picture URL:
+            $pictureURL = $element->filter("div.al_artlogo img")->attr('src');
+
+            $results[] = new SearchResultDTO(
+                provider_key: $this->getProviderKey(),
+                provider_id: $artId,
+                name: $productID,
+                description: $name,
+                category: null,
+                manufacturer: $sku,
+                preview_image_url: $pictureURL,
+                provider_url: $element->filter('a.al_artinfo_link')->attr('href')
+                );
+        });
+
+        return $results;
+    }
+
+    public function getDetails(string $id): PartDetailDTO
+    {
+        //Check that the ID is a number
+        if (!is_numeric($id)) {
+            throw new \InvalidArgumentException("Invalid ID");
+        }
+
+        //Use this endpoint to resolve the artID to a product page
+        $response = $this->client->request('GET',
+            sprintf(
+                'https://www.reichelt.com/?ACTION=514&id=74&article=%s&LANGUAGE=%s&CCOUNTRY=%s',
+                $id,
+                strtoupper($this->language),
+                strtoupper($this->country)
+            )
+        );
+        $json = $response->toArray();
+
+        //Retrieve the product page from the response
+        $productPage = $this->getBaseURL() . '/shop/product' .  $json[0]['article_path'];
+
+
+        $response = $this->client->request('GET', $productPage, [
+            'query' => [
+                'CCTYPE' => $this->includeVAT ? 'private' : 'business',
+                'currency' => $this->currency,
+            ],
+        ]);
+        $html = $response->getContent();
+        $dom = new Crawler($html);
+
+        //Extract the product notes
+        $notes = $dom->filter('p[itemprop="description"]')->html();
+
+        //Extract datasheets
+        $datasheets = [];
+        $dom->filter('div.articleDatasheet a')->each(function (Crawler $element) use (&$datasheets) {
+            $datasheets[] = new FileDTO($element->attr('href'), $element->filter('span')->text());
+        });
+
+        //Determine price for one unit
+        $priceString = $dom->filter('meta[itemprop="price"]')->attr('content');
+        $currency = $dom->filter('meta[itemprop="priceCurrency"]')->attr('content', 'EUR');
+
+        //Create purchase info
+        $purchaseInfo = new PurchaseInfoDTO(
+            distributor_name: self::DISTRIBUTOR_NAME,
+            order_number: $json[0]['article_artnr'],
+            prices: array_merge(
+                [new PriceDTO(1.0, $priceString, $currency, $this->includeVAT)]
+            , $this->parseBatchPrices($dom, $currency)),
+            product_url: $productPage
+        );
+
+        //Create part object
+        return new PartDetailDTO(
+            provider_key: $this->getProviderKey(),
+            provider_id: $id,
+            name: $json[0]['article_artnr'],
+            description: $json[0]['article_besch'],
+            category: $this->parseCategory($dom),
+            manufacturer: $json[0]['manufacturer_name'],
+            mpn: $this->parseMPN($dom),
+            preview_image_url: $json[0]['article_picture'],
+            provider_url: $productPage,
+            notes: $notes,
+            datasheets: $datasheets,
+            parameters: $this->parseParameters($dom),
+            vendor_infos: [$purchaseInfo]
+        );
+
+    }
+
+    private function parseMPN(Crawler $dom): ?string
+    {
+        //Find the small element directly after meta[itemprop="url"] element
+        $element = $dom->filter('meta[itemprop="url"] + small');
+        //If the text contains GTIN text, take the small element afterwards
+        if (str_contains($element->text(), 'GTIN')) {
+            $element = $dom->filter('meta[itemprop="url"] + small + small');
+        }
+
+        //The MPN is contained in the span inside the element
+        return $element->filter('span')->text();
+    }
+
+    private function parseBatchPrices(Crawler $dom, string $currency): array
+    {
+        //Iterate over each a.inline-block element in div.discountValue
+        $prices = [];
+        $dom->filter('div.discountValue a.inline-block')->each(function (Crawler $element) use (&$prices, $currency) {
+            //The minimum amount is the number in the span.block element
+            $minAmountText = $element->filter('span.block')->text();
+
+            //Extract a integer from the text
+            $matches = [];
+            if (!preg_match('/\d+/', $minAmountText, $matches)) {
+                return;
+            }
+
+            $minAmount = (int) $matches[0];
+
+            //The price is the text of the p.productPrice element
+            $priceString = $element->filter('p.productPrice')->text();
+            //Replace comma with dot
+            $priceString = str_replace(',', '.', $priceString);
+            //Strip any non-numeric characters
+            $priceString = preg_replace('/[^0-9.]/', '', $priceString);
+
+            $prices[] = new PriceDTO($minAmount, $priceString, $currency, $this->includeVAT);
+        });
+
+        return $prices;
+    }
+
+
+    private function parseCategory(Crawler $dom): string
+    {
+        // Look for ol.breadcrumb and iterate over the li elements
+        $category = '';
+        $dom->filter('ol.breadcrumb li.triangle-left')->each(function (Crawler $element) use (&$category) {
+            //Do not include the .breadcrumb-showmore element
+            if ($element->attr('id') === 'breadcrumb-showmore') {
+                return;
+            }
+
+            $category .= $element->text() . ' -> ';
+        });
+        //Remove the trailing ' -> '
+        $category = substr($category, 0, -4);
+
+        return $category;
+    }
+
+    /**
+     * @param  Crawler  $dom
+     * @return ParameterDTO[]
+     */
+    private function parseParameters(Crawler $dom): array
+    {
+        $parameters = [];
+        //Iterate over each ul.articleTechnicalData which contains the specifications of each group
+        $dom->filter('ul.articleTechnicalData')->each(function (Crawler $groupElement) use (&$parameters) {
+            $groupName = $groupElement->filter('li.articleTechnicalHeadline')->text();
+
+            //Iterate over each second li in ul.articleAttribute, which contains the specifications
+            $groupElement->filter('ul.articleAttribute li:nth-child(2n)')->each(function (Crawler $specElement) use (&$parameters, $groupName) {
+                $parameters[] = ParameterDTO::parseValueField(
+                    name: $specElement->previousAll()->text(),
+                    value: $specElement->text(),
+                    group: $groupName
+                );
+            });
+        });
+
+        return $parameters;
+    }
+
+    private function getBaseURL(): string
+    {
+        //Without the trailing slash
+        return 'https://www.reichelt.com/' . strtolower($this->country) . '/' . strtolower($this->language);
+    }
+
+    public function getCapabilities(): array
+    {
+        return [
+            ProviderCapabilities::BASIC,
+            ProviderCapabilities::PICTURE,
+            ProviderCapabilities::DATASHEET,
+            ProviderCapabilities::PRICE,
+        ];
+    }
+}

tests/Services/InfoProviderSystem/DTOs/ParameterDTOTest.php

@@ -70,6 +70,16 @@ class ParameterDTOTest extends TestCase
             'test'
         ];
 
+        //Test ranges
+        yield [
+            new ParameterDTO('test', value_min: 1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'),
+            'test',
+            '1.0..2.0',
+            'kg',
+            'm',
+            'test'
+        ];
+
         //Test ranges with tilde
         yield [
             new ParameterDTO('test', value_min: -1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'),