Merge branch 'reichelt_provider'

This commit is contained in:
Jan Böhmer 2025-02-20 16:46:58 +01:00
commit d176b68fd2
9 changed files with 388 additions and 75 deletions

View file

@ -44,6 +44,7 @@
PassEnv PROVIDER_MOUSER_KEY PROVIDER_MOUSER_SEARCH_OPTION PROVIDER_MOUSER_SEARCH_LIMIT PROVIDER_MOUSER_SEARCH_WITH_SIGNUP_LANGUAGE
PassEnv PROVIDER_LCSC_ENABLED PROVIDER_LCSC_CURRENCY
PassEnv PROVIDER_OEMSECRETS_KEY PROVIDER_OEMSECRETS_COUNTRY_CODE PROVIDER_OEMSECRETS_CURRENCY PROVIDER_OEMSECRETS_ZERO_PRICE PROVIDER_OEMSECRETS_SET_PARAM PROVIDER_OEMSECRETS_SORT_CRITERIA
PassEnv PROVIDER_REICHELT_ENABLED PROVIDER_REICHELT_CURRENCY PROVIDER_REICHELT_COUNTRY PROVIDER_REICHELT_LANGUAGE PROVIDER_REICHELT_INCLUDE_VAT
PassEnv EDA_KICAD_CATEGORY_DEPTH
# For most configuration files from conf-available/, which are

16
.env
View file

@ -216,6 +216,22 @@ PROVIDER_OEMSECRETS_SET_PARAM=1
#If unset or set to any other value, no sorting is performed.
PROVIDER_OEMSECRETS_SORT_CRITERIA=C
# Reichelt provider:
# Reichelt.com offers no official API, so this info provider webscrapes the website to extract info
# It could break at any time, use it at your own risk
# We dont require an API key for Reichelt, just set this to 1 to enable Reichelt support
PROVIDER_REICHELT_ENABLED=0
# The country to get prices for
PROVIDER_REICHELT_COUNTRY=DE
# The language to get results in (en, de, fr, nl, pl, it, es)
PROVIDER_REICHELT_LANGUAGE=en
# Include VAT in prices (set to 1 to include VAT, 0 to exclude VAT)
PROVIDER_REICHELT_INCLUDE_VAT=1
# The currency to get prices in (only for countries with countries other than EUR)
PROVIDER_REICHELT_CURRENCY=EUR
##################################################################################
# EDA integration related settings
##################################################################################

View file

@ -54,6 +54,8 @@
"symfony/apache-pack": "^1.0",
"symfony/asset": "6.4.*",
"symfony/console": "6.4.*",
"symfony/css-selector": "6.4.*",
"symfony/dom-crawler": "6.4.*",
"symfony/dotenv": "6.4.*",
"symfony/expression-language": "6.4.*",
"symfony/flex": "^v2.3.1",
@ -104,7 +106,6 @@
"rector/rector": "^2.0.4",
"roave/security-advisories": "dev-latest",
"symfony/browser-kit": "6.4.*",
"symfony/css-selector": "6.4.*",
"symfony/debug-bundle": "6.4.*",
"symfony/maker-bundle": "^1.13",
"symfony/phpunit-bridge": "6.4.*",

136
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "ca8701d95e24bae5d28ccdcfe242e8e4",
"content-hash": "75643d42e05fce4684644d375bff2d0a",
"packages": [
{
"name": "amphp/amp",
@ -8984,6 +8984,73 @@
],
"time": "2025-01-25T08:04:58+00:00"
},
{
"name": "symfony/dom-crawler",
"version": "v6.4.18",
"source": {
"type": "git",
"url": "https://github.com/symfony/dom-crawler.git",
"reference": "fd07959d3e8992795029bdab3605c2e8e895034e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/dom-crawler/zipball/fd07959d3e8992795029bdab3605c2e8e895034e",
"reference": "fd07959d3e8992795029bdab3605c2e8e895034e",
"shasum": ""
},
"require": {
"masterminds/html5": "^2.6",
"php": ">=8.1",
"symfony/polyfill-ctype": "~1.8",
"symfony/polyfill-mbstring": "~1.0"
},
"require-dev": {
"symfony/css-selector": "^5.4|^6.0|^7.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\DomCrawler\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Eases DOM navigation for HTML and XML documents",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/dom-crawler/tree/v6.4.18"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2025-01-09T15:35:00+00:00"
},
{
"name": "symfony/dotenv",
"version": "v6.4.16",
@ -18497,73 +18564,6 @@
],
"time": "2024-09-25T14:18:03+00:00"
},
{
"name": "symfony/dom-crawler",
"version": "v6.4.18",
"source": {
"type": "git",
"url": "https://github.com/symfony/dom-crawler.git",
"reference": "fd07959d3e8992795029bdab3605c2e8e895034e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/dom-crawler/zipball/fd07959d3e8992795029bdab3605c2e8e895034e",
"reference": "fd07959d3e8992795029bdab3605c2e8e895034e",
"shasum": ""
},
"require": {
"masterminds/html5": "^2.6",
"php": ">=8.1",
"symfony/polyfill-ctype": "~1.8",
"symfony/polyfill-mbstring": "~1.0"
},
"require-dev": {
"symfony/css-selector": "^5.4|^6.0|^7.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\DomCrawler\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Eases DOM navigation for HTML and XML documents",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/dom-crawler/tree/v6.4.18"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2025-01-09T15:35:00+00:00"
},
{
"name": "symfony/maker-bundle",
"version": "v1.62.1",

View file

@ -72,9 +72,9 @@ class ParameterDTO
group: $group);
}
//If the attribute contains "..." or a tilde we assume it is a range
if (preg_match('/(\.{3}|~)/', $value) === 1) {
$parts = preg_split('/\s*(\.{3}|~)\s*/', $value);
//If the attribute contains ".." or "..." or a tilde we assume it is a range
if (preg_match('/(\.{2,3}|~)/', $value) === 1) {
$parts = preg_split('/\s*(\.{2,3}|~)\s*/', $value);
if (count($parts) === 2) {
//Try to extract number and unit from value (allow leading +)
if ($unit === null || trim($unit) === '') {

View file

@ -33,8 +33,8 @@ use Symfony\Contracts\Cache\ItemInterface;
final class PartInfoRetriever
{
private const CACHE_DETAIL_EXPIRATION = 60 * 60 * 24 * 4; // 4 days
private const CACHE_RESULT_EXPIRATION = 60 * 60 * 24 * 7; // 7 days
private const CACHE_DETAIL_EXPIRATION = 5; // 4 days
private const CACHE_RESULT_EXPIRATION = 5; // 7 days
public function __construct(private readonly ProviderRegistry $provider_registry,
private readonly DTOtoEntityConverter $dto_to_entity_converter, private readonly CacheInterface $partInfoCache)

View file

@ -76,7 +76,7 @@ class LCSCProvider implements InfoProviderInterface
'Cookie' => new Cookie('currencyCode', $this->currency)
],
'query' => [
'productCode' => $id,
'prductCode' => $id,
],
]);

View file

@ -0,0 +1,285 @@
<?php
/*
* This file is part of Part-DB (https://github.com/Part-DB/Part-DB-symfony).
*
* Copyright (C) 2019 - 2025 Jan Böhmer (https://github.com/jbtronics)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
declare(strict_types=1);
namespace App\Services\InfoProviderSystem\Providers;
use App\Services\InfoProviderSystem\DTOs\FileDTO;
use App\Services\InfoProviderSystem\DTOs\ParameterDTO;
use App\Services\InfoProviderSystem\DTOs\PartDetailDTO;
use App\Services\InfoProviderSystem\DTOs\PriceDTO;
use App\Services\InfoProviderSystem\DTOs\PurchaseInfoDTO;
use App\Services\InfoProviderSystem\DTOs\SearchResultDTO;
use Symfony\Component\DependencyInjection\Attribute\Autowire;
use Symfony\Component\DomCrawler\Crawler;
use Symfony\Contracts\HttpClient\HttpClientInterface;
class ReicheltProvider implements InfoProviderInterface
{
public const DISTRIBUTOR_NAME = "Reichelt";
public function __construct(private readonly HttpClientInterface $client,
#[Autowire(env: "bool:PROVIDER_REICHELT_ENABLED")]
private readonly bool $enabled = true,
#[Autowire(env: "PROVIDER_REICHELT_LANGUAGE")]
private readonly string $language = "en",
#[Autowire(env: "PROVIDER_REICHELT_COUNTRY")]
private readonly string $country = "DE",
#[Autowire(env: "PROVIDER_REICHELT_INCLUDE_VAT")]
private readonly bool $includeVAT = false,
#[Autowire(env: "PROVIDER_REICHELT_CURRENCY")]
private readonly string $currency = "EUR",
)
{
}
public function getProviderInfo(): array
{
return [
'name' => 'Reichelt',
'description' => 'Webscrapping from reichelt.com to get part information',
'url' => 'https://www.reichelt.com/',
'disabled_help' => 'TODO'
];
}
public function getProviderKey(): string
{
return 'reichelt';
}
public function isActive(): bool
{
return $this->enabled;
}
public function searchByKeyword(string $keyword): array
{
$response = $this->client->request('GET', sprintf($this->getBaseURL() . '/shop/search/%s', $keyword));
$html = $response->getContent();
//Parse the HTML and return the results
$dom = new Crawler($html);
//Iterate over all div.al_gallery_article elements
$results = [];
$dom->filter('div.al_gallery_article')->each(function (Crawler $element) use (&$results) {
//Extract product id from data-product attribute
$artId = json_decode($element->attr('data-product'), true, 2, JSON_THROW_ON_ERROR)['artid'];
$productID = $element->filter('meta[itemprop="productID"]')->attr('content');
$name = $element->filter('meta[itemprop="name"]')->attr('content');
$sku = $element->filter('meta[itemprop="sku"]')->attr('content');
//Try to extract a picture URL:
$pictureURL = $element->filter("div.al_artlogo img")->attr('src');
$results[] = new SearchResultDTO(
provider_key: $this->getProviderKey(),
provider_id: $artId,
name: $productID,
description: $name,
category: null,
manufacturer: $sku,
preview_image_url: $pictureURL,
provider_url: $element->filter('a.al_artinfo_link')->attr('href')
);
});
return $results;
}
public function getDetails(string $id): PartDetailDTO
{
//Check that the ID is a number
if (!is_numeric($id)) {
throw new \InvalidArgumentException("Invalid ID");
}
//Use this endpoint to resolve the artID to a product page
$response = $this->client->request('GET',
sprintf(
'https://www.reichelt.com/?ACTION=514&id=74&article=%s&LANGUAGE=%s&CCOUNTRY=%s',
$id,
strtoupper($this->language),
strtoupper($this->country)
)
);
$json = $response->toArray();
//Retrieve the product page from the response
$productPage = $this->getBaseURL() . '/shop/product' . $json[0]['article_path'];
$response = $this->client->request('GET', $productPage, [
'query' => [
'CCTYPE' => $this->includeVAT ? 'private' : 'business',
'currency' => $this->currency,
],
]);
$html = $response->getContent();
$dom = new Crawler($html);
//Extract the product notes
$notes = $dom->filter('p[itemprop="description"]')->html();
//Extract datasheets
$datasheets = [];
$dom->filter('div.articleDatasheet a')->each(function (Crawler $element) use (&$datasheets) {
$datasheets[] = new FileDTO($element->attr('href'), $element->filter('span')->text());
});
//Determine price for one unit
$priceString = $dom->filter('meta[itemprop="price"]')->attr('content');
$currency = $dom->filter('meta[itemprop="priceCurrency"]')->attr('content', 'EUR');
//Create purchase info
$purchaseInfo = new PurchaseInfoDTO(
distributor_name: self::DISTRIBUTOR_NAME,
order_number: $json[0]['article_artnr'],
prices: array_merge(
[new PriceDTO(1.0, $priceString, $currency, $this->includeVAT)]
, $this->parseBatchPrices($dom, $currency)),
product_url: $productPage
);
//Create part object
return new PartDetailDTO(
provider_key: $this->getProviderKey(),
provider_id: $id,
name: $json[0]['article_artnr'],
description: $json[0]['article_besch'],
category: $this->parseCategory($dom),
manufacturer: $json[0]['manufacturer_name'],
mpn: $this->parseMPN($dom),
preview_image_url: $json[0]['article_picture'],
provider_url: $productPage,
notes: $notes,
datasheets: $datasheets,
parameters: $this->parseParameters($dom),
vendor_infos: [$purchaseInfo]
);
}
private function parseMPN(Crawler $dom): ?string
{
//Find the small element directly after meta[itemprop="url"] element
$element = $dom->filter('meta[itemprop="url"] + small');
//If the text contains GTIN text, take the small element afterwards
if (str_contains($element->text(), 'GTIN')) {
$element = $dom->filter('meta[itemprop="url"] + small + small');
}
//The MPN is contained in the span inside the element
return $element->filter('span')->text();
}
private function parseBatchPrices(Crawler $dom, string $currency): array
{
//Iterate over each a.inline-block element in div.discountValue
$prices = [];
$dom->filter('div.discountValue a.inline-block')->each(function (Crawler $element) use (&$prices, $currency) {
//The minimum amount is the number in the span.block element
$minAmountText = $element->filter('span.block')->text();
//Extract a integer from the text
$matches = [];
if (!preg_match('/\d+/', $minAmountText, $matches)) {
return;
}
$minAmount = (int) $matches[0];
//The price is the text of the p.productPrice element
$priceString = $element->filter('p.productPrice')->text();
//Replace comma with dot
$priceString = str_replace(',', '.', $priceString);
//Strip any non-numeric characters
$priceString = preg_replace('/[^0-9.]/', '', $priceString);
$prices[] = new PriceDTO($minAmount, $priceString, $currency, $this->includeVAT);
});
return $prices;
}
private function parseCategory(Crawler $dom): string
{
// Look for ol.breadcrumb and iterate over the li elements
$category = '';
$dom->filter('ol.breadcrumb li.triangle-left')->each(function (Crawler $element) use (&$category) {
//Do not include the .breadcrumb-showmore element
if ($element->attr('id') === 'breadcrumb-showmore') {
return;
}
$category .= $element->text() . ' -> ';
});
//Remove the trailing ' -> '
$category = substr($category, 0, -4);
return $category;
}
/**
* @param Crawler $dom
* @return ParameterDTO[]
*/
private function parseParameters(Crawler $dom): array
{
$parameters = [];
//Iterate over each ul.articleTechnicalData which contains the specifications of each group
$dom->filter('ul.articleTechnicalData')->each(function (Crawler $groupElement) use (&$parameters) {
$groupName = $groupElement->filter('li.articleTechnicalHeadline')->text();
//Iterate over each second li in ul.articleAttribute, which contains the specifications
$groupElement->filter('ul.articleAttribute li:nth-child(2n)')->each(function (Crawler $specElement) use (&$parameters, $groupName) {
$parameters[] = ParameterDTO::parseValueField(
name: $specElement->previousAll()->text(),
value: $specElement->text(),
group: $groupName
);
});
});
return $parameters;
}
private function getBaseURL(): string
{
//Without the trailing slash
return 'https://www.reichelt.com/' . strtolower($this->country) . '/' . strtolower($this->language);
}
public function getCapabilities(): array
{
return [
ProviderCapabilities::BASIC,
ProviderCapabilities::PICTURE,
ProviderCapabilities::DATASHEET,
ProviderCapabilities::PRICE,
];
}
}

View file

@ -70,6 +70,16 @@ class ParameterDTOTest extends TestCase
'test'
];
//Test ranges
yield [
new ParameterDTO('test', value_min: 1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'),
'test',
'1.0..2.0',
'kg',
'm',
'test'
];
//Test ranges with tilde
yield [
new ParameterDTO('test', value_min: -1.0, value_max: 2.0, unit: 'kg', symbol: 'm', group: 'test'),