diff --git a/.docker/symfony.conf b/.docker/symfony.conf index 5788a630..e87e37b0 100644 --- a/.docker/symfony.conf +++ b/.docker/symfony.conf @@ -44,6 +44,7 @@ PassEnv PROVIDER_MOUSER_KEY PROVIDER_MOUSER_SEARCH_OPTION PROVIDER_MOUSER_SEARCH_LIMIT PROVIDER_MOUSER_SEARCH_WITH_SIGNUP_LANGUAGE PassEnv PROVIDER_LCSC_ENABLED PROVIDER_LCSC_CURRENCY PassEnv PROVIDER_OEMSECRETS_KEY PROVIDER_OEMSECRETS_COUNTRY_CODE PROVIDER_OEMSECRETS_CURRENCY PROVIDER_OEMSECRETS_ZERO_PRICE PROVIDER_OEMSECRETS_SET_PARAM PROVIDER_OEMSECRETS_SORT_CRITERIA + PassEnv PROVIDER_REICHELT_ENABLED PROVIDER_REICHELT_CURRENCY PROVIDER_REICHELT_COUNTRY PROVIDER_REICHELT_LANGUAGE PROVIDER_REICHELT_INCLUDE_VAT PassEnv EDA_KICAD_CATEGORY_DEPTH # For most configuration files from conf-available/, which are diff --git a/.env b/.env index f0fbd306..c67d6fdc 100644 --- a/.env +++ b/.env @@ -216,6 +216,22 @@ PROVIDER_OEMSECRETS_SET_PARAM=1 #If unset or set to any other value, no sorting is performed. PROVIDER_OEMSECRETS_SORT_CRITERIA=C + +# Reichelt provider: +# Reichelt.com offers no official API, so this info provider webscrapes the website to extract info +# It could break at any time, use it at your own risk + +# We dont require an API key for Reichelt, just set this to 1 to enable Reichelt support +PROVIDER_REICHELT_ENABLED=0 +# The country to get prices for +PROVIDER_REICHELT_COUNTRY=DE +# The language to get results in (en, de, fr, nl, pl, it, es) +PROVIDER_REICHELT_LANGUAGE=en +# Include VAT in prices (set to 1 to include VAT, 0 to exclude VAT) +PROVIDER_REICHELT_INCLUDE_VAT=1 +# The currency to get prices in (only for countries with countries other than EUR) +PROVIDER_REICHELT_CURRENCY=EUR + ################################################################################## # EDA integration related settings ################################################################################## diff --git a/composer.json b/composer.json index 11026fa2..3b2904d6 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,8 @@ "symfony/apache-pack": "^1.0", "symfony/asset": "6.4.*", "symfony/console": "6.4.*", + "symfony/css-selector": "6.4.*", + "symfony/dom-crawler": "6.4.*", "symfony/dotenv": "6.4.*", "symfony/expression-language": "6.4.*", "symfony/flex": "^v2.3.1", @@ -104,7 +106,6 @@ "rector/rector": "^2.0.4", "roave/security-advisories": "dev-latest", "symfony/browser-kit": "6.4.*", - "symfony/css-selector": "6.4.*", "symfony/debug-bundle": "6.4.*", "symfony/maker-bundle": "^1.13", "symfony/phpunit-bridge": "6.4.*", diff --git a/composer.lock b/composer.lock index 892ae0c7..498132b2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ca8701d95e24bae5d28ccdcfe242e8e4", + "content-hash": "75643d42e05fce4684644d375bff2d0a", "packages": [ { "name": "amphp/amp", @@ -8984,6 +8984,73 @@ ], "time": "2025-01-25T08:04:58+00:00" }, + { + "name": "symfony/dom-crawler", + "version": "v6.4.18", + "source": { + "type": "git", + "url": "https://github.com/symfony/dom-crawler.git", + "reference": "fd07959d3e8992795029bdab3605c2e8e895034e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/fd07959d3e8992795029bdab3605c2e8e895034e", + "reference": "fd07959d3e8992795029bdab3605c2e8e895034e", + "shasum": "" + }, + "require": { + "masterminds/html5": "^2.6", + "php": ">=8.1", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-mbstring": "~1.0" + }, + "require-dev": { + "symfony/css-selector": "^5.4|^6.0|^7.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\DomCrawler\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Eases DOM navigation for HTML and XML documents", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/dom-crawler/tree/v6.4.18" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-01-09T15:35:00+00:00" + }, { "name": "symfony/dotenv", "version": "v6.4.16", @@ -18497,73 +18564,6 @@ ], "time": "2024-09-25T14:18:03+00:00" }, - { - "name": "symfony/dom-crawler", - "version": "v6.4.18", - "source": { - "type": "git", - "url": "https://github.com/symfony/dom-crawler.git", - "reference": "fd07959d3e8992795029bdab3605c2e8e895034e" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/fd07959d3e8992795029bdab3605c2e8e895034e", - "reference": "fd07959d3e8992795029bdab3605c2e8e895034e", - "shasum": "" - }, - "require": { - "masterminds/html5": "^2.6", - "php": ">=8.1", - "symfony/polyfill-ctype": "~1.8", - "symfony/polyfill-mbstring": "~1.0" - }, - "require-dev": { - "symfony/css-selector": "^5.4|^6.0|^7.0" - }, - "type": "library", - "autoload": { - "psr-4": { - "Symfony\\Component\\DomCrawler\\": "" - }, - "exclude-from-classmap": [ - "/Tests/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Eases DOM navigation for HTML and XML documents", - "homepage": "https://symfony.com", - "support": { - "source": "https://github.com/symfony/dom-crawler/tree/v6.4.18" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2025-01-09T15:35:00+00:00" - }, { "name": "symfony/maker-bundle", "version": "v1.62.1", diff --git a/src/Services/InfoProviderSystem/DTOs/ParameterDTO.php b/src/Services/InfoProviderSystem/DTOs/ParameterDTO.php index 3332700b..0b54d1a9 100644 --- a/src/Services/InfoProviderSystem/DTOs/ParameterDTO.php +++ b/src/Services/InfoProviderSystem/DTOs/ParameterDTO.php @@ -72,9 +72,9 @@ class ParameterDTO group: $group); } - //If the attribute contains "..." or a tilde we assume it is a range - if (preg_match('/(\.{3}|~)/', $value) === 1) { - $parts = preg_split('/\s*(\.{3}|~)\s*/', $value); + //If the attribute contains ".." or "..." or a tilde we assume it is a range + if (preg_match('/(\.{2,3}|~)/', $value) === 1) { + $parts = preg_split('/\s*(\.{2,3}|~)\s*/', $value); if (count($parts) === 2) { //Try to extract number and unit from value (allow leading +) if ($unit === null || trim($unit) === '') { diff --git a/src/Services/InfoProviderSystem/PartInfoRetriever.php b/src/Services/InfoProviderSystem/PartInfoRetriever.php index 1a31b197..1db36eee 100644 --- a/src/Services/InfoProviderSystem/PartInfoRetriever.php +++ b/src/Services/InfoProviderSystem/PartInfoRetriever.php @@ -33,8 +33,8 @@ use Symfony\Contracts\Cache\ItemInterface; final class PartInfoRetriever { - private const CACHE_DETAIL_EXPIRATION = 60 * 60 * 24 * 4; // 4 days - private const CACHE_RESULT_EXPIRATION = 60 * 60 * 24 * 7; // 7 days + private const CACHE_DETAIL_EXPIRATION = 5; // 4 days + private const CACHE_RESULT_EXPIRATION = 5; // 7 days public function __construct(private readonly ProviderRegistry $provider_registry, private readonly DTOtoEntityConverter $dto_to_entity_converter, private readonly CacheInterface $partInfoCache) diff --git a/src/Services/InfoProviderSystem/Providers/LCSCProvider.php b/src/Services/InfoProviderSystem/Providers/LCSCProvider.php index d903a8dd..375c6f4d 100755 --- a/src/Services/InfoProviderSystem/Providers/LCSCProvider.php +++ b/src/Services/InfoProviderSystem/Providers/LCSCProvider.php @@ -76,7 +76,7 @@ class LCSCProvider implements InfoProviderInterface 'Cookie' => new Cookie('currencyCode', $this->currency) ], 'query' => [ - 'productCode' => $id, + 'prductCode' => $id, ], ]); diff --git a/src/Services/InfoProviderSystem/Providers/ReicheltProvider.php b/src/Services/InfoProviderSystem/Providers/ReicheltProvider.php new file mode 100644 index 00000000..b0d704d3 --- /dev/null +++ b/src/Services/InfoProviderSystem/Providers/ReicheltProvider.php @@ -0,0 +1,285 @@ +. + */ + +declare(strict_types=1); + + +namespace App\Services\InfoProviderSystem\Providers; + +use App\Services\InfoProviderSystem\DTOs\FileDTO; +use App\Services\InfoProviderSystem\DTOs\ParameterDTO; +use App\Services\InfoProviderSystem\DTOs\PartDetailDTO; +use App\Services\InfoProviderSystem\DTOs\PriceDTO; +use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO; +use App\Services\InfoProviderSystem\DTOs\SearchResultDTO; +use Symfony\Component\DependencyInjection\Attribute\Autowire; +use Symfony\Component\DomCrawler\Crawler; +use Symfony\Contracts\HttpClient\HttpClientInterface; + +class ReicheltProvider implements InfoProviderInterface +{ + + public const DISTRIBUTOR_NAME = "Reichelt"; + + public function __construct(private readonly HttpClientInterface $client, + #[Autowire(env: "bool:PROVIDER_REICHELT_ENABLED")] + private readonly bool $enabled = true, + #[Autowire(env: "PROVIDER_REICHELT_LANGUAGE")] + private readonly string $language = "en", + #[Autowire(env: "PROVIDER_REICHELT_COUNTRY")] + private readonly string $country = "DE", + #[Autowire(env: "PROVIDER_REICHELT_INCLUDE_VAT")] + private readonly bool $includeVAT = false, + #[Autowire(env: "PROVIDER_REICHELT_CURRENCY")] + private readonly string $currency = "EUR", + ) + { + } + + public function getProviderInfo(): array + { + return [ + 'name' => 'Reichelt', + 'description' => 'Webscrapping from reichelt.com to get part information', + 'url' => 'https://www.reichelt.com/', + 'disabled_help' => 'TODO' + ]; + } + + public function getProviderKey(): string + { + return 'reichelt'; + } + + public function isActive(): bool + { + return $this->enabled; + } + + public function searchByKeyword(string $keyword): array + { + $response = $this->client->request('GET', sprintf($this->getBaseURL() . '/shop/search/%s', $keyword)); + $html = $response->getContent(); + + //Parse the HTML and return the results + $dom = new Crawler($html); + //Iterate over all div.al_gallery_article elements + $results = []; + $dom->filter('div.al_gallery_article')->each(function (Crawler $element) use (&$results) { + + //Extract product id from data-product attribute + $artId = json_decode($element->attr('data-product'), true, 2, JSON_THROW_ON_ERROR)['artid']; + + $productID = $element->filter('meta[itemprop="productID"]')->attr('content'); + $name = $element->filter('meta[itemprop="name"]')->attr('content'); + $sku = $element->filter('meta[itemprop="sku"]')->attr('content'); + + //Try to extract a picture URL: + $pictureURL = $element->filter("div.al_artlogo img")->attr('src'); + + $results[] = new SearchResultDTO( + provider_key: $this->getProviderKey(), + provider_id: $artId, + name: $productID, + description: $name, + category: null, + manufacturer: $sku, + preview_image_url: $pictureURL, + provider_url: $element->filter('a.al_artinfo_link')->attr('href') + ); + }); + + return $results; + } + + public function getDetails(string $id): PartDetailDTO + { + //Check that the ID is a number + if (!is_numeric($id)) { + throw new \InvalidArgumentException("Invalid ID"); + } + + //Use this endpoint to resolve the artID to a product page + $response = $this->client->request('GET', + sprintf( + 'https://www.reichelt.com/?ACTION=514&id=74&article=%s&LANGUAGE=%s&CCOUNTRY=%s', + $id, + strtoupper($this->language), + strtoupper($this->country) + ) + ); + $json = $response->toArray(); + + //Retrieve the product page from the response + $productPage = $this->getBaseURL() . '/shop/product' . $json[0]['article_path']; + + + $response = $this->client->request('GET', $productPage, [ + 'query' => [ + 'CCTYPE' => $this->includeVAT ? 'private' : 'business', + 'currency' => $this->currency, + ], + ]); + $html = $response->getContent(); + $dom = new Crawler($html); + + //Extract the product notes + $notes = $dom->filter('p[itemprop="description"]')->html(); + + //Extract datasheets + $datasheets = []; + $dom->filter('div.articleDatasheet a')->each(function (Crawler $element) use (&$datasheets) { + $datasheets[] = new FileDTO($element->attr('href'), $element->filter('span')->text()); + }); + + //Determine price for one unit + $priceString = $dom->filter('meta[itemprop="price"]')->attr('content'); + $currency = $dom->filter('meta[itemprop="priceCurrency"]')->attr('content', 'EUR'); + + //Create purchase info + $purchaseInfo = new PurchaseInfoDTO( + distributor_name: self::DISTRIBUTOR_NAME, + order_number: $json[0]['article_artnr'], + prices: array_merge( + [new PriceDTO(1.0, $priceString, $currency, $this->includeVAT)] + , $this->parseBatchPrices($dom, $currency)), + product_url: $productPage + ); + + //Create part object + return new PartDetailDTO( + provider_key: $this->getProviderKey(), + provider_id: $id, + name: $json[0]['article_artnr'], + description: $json[0]['article_besch'], + category: $this->parseCategory($dom), + manufacturer: $json[0]['manufacturer_name'], + mpn: $this->parseMPN($dom), + preview_image_url: $json[0]['article_picture'], + provider_url: $productPage, + notes: $notes, + datasheets: $datasheets, + parameters: $this->parseParameters($dom), + vendor_infos: [$purchaseInfo] + ); + + } + + private function parseMPN(Crawler $dom): ?string + { + //Find the small element directly after meta[itemprop="url"] element + $element = $dom->filter('meta[itemprop="url"] + small'); + //If the text contains GTIN text, take the small element afterwards + if (str_contains($element->text(), 'GTIN')) { + $element = $dom->filter('meta[itemprop="url"] + small + small'); + } + + //The MPN is contained in the span inside the element + return $element->filter('span')->text(); + } + + private function parseBatchPrices(Crawler $dom, string $currency): array + { + //Iterate over each a.inline-block element in div.discountValue + $prices = []; + $dom->filter('div.discountValue a.inline-block')->each(function (Crawler $element) use (&$prices, $currency) { + //The minimum amount is the number in the span.block element + $minAmountText = $element->filter('span.block')->text(); + + //Extract a integer from the text + $matches = []; + if (!preg_match('/\d+/', $minAmountText, $matches)) { + return; + } + + $minAmount = (int) $matches[0]; + + //The price is the text of the p.productPrice element + $priceString = $element->filter('p.productPrice')->text(); + //Replace comma with dot + $priceString = str_replace(',', '.', $priceString); + //Strip any non-numeric characters + $priceString = preg_replace('/[^0-9.]/', '', $priceString); + + $prices[] = new PriceDTO($minAmount, $priceString, $currency, $this->includeVAT); + }); + + return $prices; + } + + + private function parseCategory(Crawler $dom): string + { + // Look for ol.breadcrumb and iterate over the li elements + $category = ''; + $dom->filter('ol.breadcrumb li.triangle-left')->each(function (Crawler $element) use (&$category) { + //Do not include the .breadcrumb-showmore element + if ($element->attr('id') === 'breadcrumb-showmore') { + return; + } + + $category .= $element->text() . ' -> '; + }); + //Remove the trailing ' -> ' + $category = substr($category, 0, -4); + + return $category; + } + + /** + * @param Crawler $dom + * @return ParameterDTO[] + */ + private function parseParameters(Crawler $dom): array + { + $parameters = []; + //Iterate over each ul.articleTechnicalData which contains the specifications of each group + $dom->filter('ul.articleTechnicalData')->each(function (Crawler $groupElement) use (&$parameters) { + $groupName = $groupElement->filter('li.articleTechnicalHeadline')->text(); + + //Iterate over each second li in ul.articleAttribute, which contains the specifications + $groupElement->filter('ul.articleAttribute li:nth-child(2n)')->each(function (Crawler $specElement) use (&$parameters, $groupName) { + $parameters[] = ParameterDTO::parseValueField( + name: $specElement->previousAll()->text(), + value: $specElement->text(), + group: $groupName + ); + }); + }); + + return $parameters; + } + + private function getBaseURL(): string + { + //Without the trailing slash + return 'https://www.reichelt.com/' . strtolower($this->country) . '/' . strtolower($this->language); + } + + public function getCapabilities(): array + { + return [ + ProviderCapabilities::BASIC, + ProviderCapabilities::PICTURE, + ProviderCapabilities::DATASHEET, + ProviderCapabilities::PRICE, + ]; + } +} \ No newline at end of file diff --git a/tests/Services/InfoProviderSystem/DTOs/ParameterDTOTest.php b/tests/Services/InfoProviderSystem/DTOs/ParameterDTOTest.php index 7bbebf0b..ee4ca39a 100644 --- a/tests/Services/InfoProviderSystem/DTOs/ParameterDTOTest.php +++ b/tests/Services/InfoProviderSystem/DTOs/ParameterDTOTest.php @@ -70,6 +70,16 @@ class ParameterDTOTest extends TestCase 'test' ]; + //Test ranges + yield [ + new ParameterDTO('test', value_min: 1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'), + 'test', + '1.0..2.0', + 'kg', + 'm', + 'test' + ]; + //Test ranges with tilde yield [ new ParameterDTO('test', value_min: -1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'),