From a14508d79b45f8d589051d845f016c7d63540238 Mon Sep 17 00:00:00 2001 From: "ljf (zamentur)" Date: Sat, 3 Feb 2024 12:58:36 +0100 Subject: [PATCH 001/423] Add sans-nuage instance (#3947) --- docs/01_General/06_Public_Hosts.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index 4aa905da..1d59d918 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -5,6 +5,7 @@ | ![](https://iplookup.flagfox.net/images/h16/GB.png) | https://rss-bridge.org/bridge01 | ![](https://img.shields.io/website/https/rss-bridge.org/bridge01.svg) | [@dvikan](https://github.com/dvikan) | London, Digital Ocean| | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rssbridge.flossboxin.org.in/ | ![](https://img.shields.io/badge/website-up-brightgreen) | [@vdbhb59](https://github.com/vdbhb59) | Hosted with OVH SAS (Maintained in India) | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rss-bridge.cheredeprince.net/ | ![](https://img.shields.io/website/https/rss-bridge.cheredeprince.net) | [@La_Bécasse](https://cheredeprince.net/contact) | Self-Hosted at home in France | +| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rss-bridge.sans-nuage.fr | ![](https://img.shields.io/website/https/rss-bridge.sans-nuage.fr) | [@Alsace Réseau Neutre](https://arn-fai.net/contact) | Hosted in Alsace, France | | ![](https://iplookup.flagfox.net/images/h16/GB.png) | https://rss-bridge.lewd.tech | ![](https://img.shields.io/website/https/rss-bridge.lewd.tech.svg) | [@Erisa](https://github.com/Erisa) | Hosted in London, protected by Cloudflare Rate Limiting | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://bridge.easter.fr | ![](https://img.shields.io/website/https/bridge.easter.fr.svg) | [@chatainsim](https://github.com/chatainsim) | Hosted in Isère, France | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://wtf.roflcopter.fr/rss-bridge/ | ![](https://img.shields.io/website/https/wtf.roflcopter.fr/rss-bridge.svg) | [roflcopter.fr](https://wtf.roflcopter.fr/) | Hosted in France | From 7c89712837fd35f6ec12d4652201d54b443ce73c Mon Sep 17 00:00:00 2001 From: Dag Date: Sat, 3 Feb 2024 13:56:56 +0100 Subject: [PATCH 002/423] ci: fix broken docs build (#3948) --- docs/10_Bridge_Specific/FacebookBridge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/10_Bridge_Specific/FacebookBridge.md b/docs/10_Bridge_Specific/FacebookBridge.md index f24f8aa8..665b802f 100644 --- a/docs/10_Bridge_Specific/FacebookBridge.md +++ b/docs/10_Bridge_Specific/FacebookBridge.md @@ -4,7 +4,7 @@ State of this bridge: - Facebook Groups (and probably other sections too) do not work at all - No maintainer - Needs cookie consent support for public pages -- Needs login support (see [this example]([url](https://github.com/RSS-Bridge/rss-bridge/issues/1891)) for Instagram) for private groups +- Needs login support see [this example](https://github.com/RSS-Bridge/rss-bridge/issues/1891) for Instagram) for private groups Due to the 2020 [Facebook redesign](https://engineering.fb.com/2020/05/08/web/facebook-redesign/) and the requirement to [accept cookies](https://www.facebook.com/business/help/348535683460989) From d175bab58e3afd63140791446b01dd36837fe2d8 Mon Sep 17 00:00:00 2001 From: Tostiman <18124323+t0stiman@users.noreply.github.com> Date: Sun, 4 Feb 2024 18:28:12 +0100 Subject: [PATCH 003/423] Fix car throttle bridge (#3925) --- bridges/CarThrottleBridge.php | 118 +++++++++++++++++++++++++------ bridges/HardwareInfoBridge.php | 1 + bridges/RaceDepartmentBridge.php | 1 + 3 files changed, 99 insertions(+), 21 deletions(-) diff --git a/bridges/CarThrottleBridge.php b/bridges/CarThrottleBridge.php index 70d7b54e..0a05ee60 100644 --- a/bridges/CarThrottleBridge.php +++ b/bridges/CarThrottleBridge.php @@ -6,46 +6,122 @@ class CarThrottleBridge extends BridgeAbstract const URI = 'https://www.carthrottle.com/'; const DESCRIPTION = 'Get the latest car-related news from Car Throttle.'; const MAINTAINER = 't0stiman'; + const DONATION_URI = 'https://ko-fi.com/tostiman'; + + const PARAMETERS = [ + 'Show articles from these categories:' => [ + 'news' => [ + 'name' => 'news', + 'type' => 'checkbox' + ], + 'reviews' => [ + 'name' => 'reviews', + 'type' => 'checkbox' + ], + 'features' => [ + 'name' => 'features', + 'type' => 'checkbox' + ], + 'videos' => [ + 'name' => 'videos', + 'type' => 'checkbox' + ], + 'gaming' => [ + 'name' => 'gaming', + 'type' => 'checkbox' + ] + ] + ]; public function collectData() { - $news = getSimpleHTMLDOMCached(self::URI . 'news'); + $this->items = []; - $this->items[] = []; + $this->handleCategory('news'); + $this->handleCategory('reviews'); + $this->handleCategory('features'); + $this->handleCategory2('videos', 'video'); + $this->handleCategory('gaming'); + } + + private function handleCategory($category) + { + if ($this->getInput($category)) { + $this->getArticles($category); + } + } + + private function handleCategory2($categoryParameter, $categoryURLname) + { + if ($this->getInput($categoryParameter)) { + $this->getArticles($categoryURLname); + } + } + + private function getArticles($category) + { + $categoryPage = getSimpleHTMLDOMCached(self::URI . $category); //for each post - foreach ($news->find('div.cmg-card') as $post) { + foreach ($categoryPage->find('div.cmg-card') as $post) { $item = []; - $titleElement = $post->find('div.title a.cmg-link')[0]; - $item['uri'] = self::URI . $titleElement->getAttribute('href'); + $titleElement = $post->find('div.title a')[0]; + $post_uri = self::URI . $titleElement->getAttribute('href'); + + if (!isset($post_uri) || $post_uri == '') { + continue; + } + + $item['uri'] = $post_uri; $item['title'] = $titleElement->innertext; $articlePage = getSimpleHTMLDOMCached($item['uri']); - $authorDiv = $articlePage->find('div.author div'); - if ($authorDiv) { - $item['author'] = $authorDiv[1]->innertext; - } + $item['author'] = $this->parseAuthor($articlePage); + + $articleElement = $articlePage->find('article')[0]; - $dinges = $articlePage->find('div.main-body')[0] ?? null; //remove ads - if ($dinges) { - foreach ($dinges->find('aside') as $ad) { - $ad->outertext = ''; - $dinges->save(); - } + foreach ($articleElement->find('aside') as $ad) { + $ad->outertext = ''; } - $var = $articlePage->find('div.summary')[0] ?? ''; - $var1 = $articlePage->find('figure.main-image')[0] ?? ''; - $dinges1 = $dinges ?? ''; + $summary = $articleElement->find('div.summary')[0]; - $item['content'] = $var . - $var1 . - $dinges1; + //remove header so we are left with the article content + foreach ($articleElement->find('header') as $found) { + $found->outertext = ''; + } + + //remove comments (registering on carthrottle.com is impossible so the comment sections are empty anyway) + foreach ($articleElement->find('#lbs-comments') as $found) { + $found->outertext = ''; + } + + //these are supposed to be hidden + foreach ($articleElement->find('.visually-hidden') as $found) { + $found->outertext = ''; + } + + $item['content'] = $summary . $articleElement; array_push($this->items, $item); } } + + private function parseAuthor($articlePage) + { + $authorDivs = $articlePage->find('div address'); + if (!$authorDivs) { + return ''; + } + + $a = $authorDivs[0]->find('a'); + if ($a) { + return $a->innertext; + } + + return $authorDivs[0]->innertext; + } } diff --git a/bridges/HardwareInfoBridge.php b/bridges/HardwareInfoBridge.php index 5970ecd0..dc32c33a 100644 --- a/bridges/HardwareInfoBridge.php +++ b/bridges/HardwareInfoBridge.php @@ -6,6 +6,7 @@ class HardwareInfoBridge extends FeedExpander const URI = 'https://nl.hardware.info/'; const DESCRIPTION = 'Tech news from hardware.info (Dutch)'; const MAINTAINER = 't0stiman'; + const DONATION_URI = 'https://ko-fi.com/tostiman'; public function collectData() { diff --git a/bridges/RaceDepartmentBridge.php b/bridges/RaceDepartmentBridge.php index 7390761f..a601b4d6 100644 --- a/bridges/RaceDepartmentBridge.php +++ b/bridges/RaceDepartmentBridge.php @@ -6,6 +6,7 @@ class RaceDepartmentBridge extends FeedExpander const URI = 'https://racedepartment.com/'; const DESCRIPTION = 'Get the latest (sim)racing news from RaceDepartment.'; const MAINTAINER = 't0stiman'; + const DONATION_URI = 'https://ko-fi.com/tostiman'; public function collectData() { From 7931f37a835b3fd39093fd7de15f1b2a0263747d Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Mon, 5 Feb 2024 23:30:18 +0100 Subject: [PATCH 004/423] [PepperBridgeAbstract] Fix deal image scraping (#3953) Deal Image was moved to a vuejs element, the deal image scraping was fixed. --- bridges/PepperBridgeAbstract.php | 34 +++----------------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 73bd194d..45bfe209 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -437,37 +437,9 @@ HEREDOC; */ private function getImage($deal) { - $selectorLazy = implode( - ' ', /* Notice this is a space! */ - [ - 'thread-image', - 'width--all-auto', - 'height--all-auto', - 'imgFrame-img', - 'img--dummy', - 'js-lazy-img' - ] - ); - - $selectorPlain = implode( - ' ', /* Notice this is a space! */ - [ - 'thread-image', - 'width--all-auto', - 'height--all-auto', - 'imgFrame-img', - ] - ); - if ($deal->find('img[class=' . $selectorLazy . ']', 0) != null) { - return json_decode( - html_entity_decode( - $deal->find('img[class=' . $selectorLazy . ']', 0) - ->getAttribute('data-lazy-img') - ) - )->{'src'}; - } else { - return $deal->find('img[class*=' . $selectorPlain . ']', 0)->src ?? ''; - } + // Get thread Image JSON content + $content = Json::decode($deal->find('div[class*=threadGrid-image]', 0)->find('div[class=js-vue2]', 0)->getAttribute('data-vue2')); + return $content['props']['threadImageUrl']; } /** From 66a6847fd09c90e1cecccce38cdeb61f3e48675a Mon Sep 17 00:00:00 2001 From: Scott Colby Date: Mon, 5 Feb 2024 20:21:30 -0500 Subject: [PATCH 005/423] Two fixes to DeutscheWelle (#3954) * [DeutscheWelleBridge] Small URL fix. Reset the $item's uri value after removing the tracking query string. * [DeutscheWelleBridge] Fix "hero" images. The main "hero" image for each article has src="" and relies on the srcset attribute for the browser to pick the best image based on the actual displayed size. The call to `defaultLinkTo()` replaces the empty src with the article's link, which, not being an image, breaks the image. This change resets the src's of any such images back to "". --- bridges/DeutscheWelleBridge.php | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/bridges/DeutscheWelleBridge.php b/bridges/DeutscheWelleBridge.php index 29b478b9..51214320 100644 --- a/bridges/DeutscheWelleBridge.php +++ b/bridges/DeutscheWelleBridge.php @@ -73,12 +73,12 @@ class DeutscheWelleBridge extends FeedExpander protected function parseItem(array $item) { - $parsedUrl = parse_url($item['uri']); - unset($parsedUrl['query']); - $url = $this->unparseUrl($parsedUrl); + $parsedUri = parse_url($item['uri']); + unset($parsedUri['query']); + $item['uri'] = $this->unparseUrl($parsedUri); - $page = getSimpleHTMLDOM($url); - $page = defaultLinkTo($page, $url); + $page = getSimpleHTMLDOM($item['uri']); + $page = defaultLinkTo($page, $item['uri']); $article = $page->find('article', 0); @@ -112,6 +112,13 @@ class DeutscheWelleBridge extends FeedExpander $img->height = null; } + // remove bad img src's added by defaultLinkTo() above + // these images should have src="" and will then use + // the srcset attribute to load the best image for the displayed size + foreach ($article->find('figure > picture > img') as $img) { + $img->src = ''; + } + // replace lazy-loaded images foreach ($article->find('figure.placeholder-image') as $figure) { $img = $figure->find('img', 0); From 64f95b4990c5b890eb526af1990205cbef5ac77a Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Tue, 6 Feb 2024 02:23:12 +0100 Subject: [PATCH 006/423] [PepperBridgeAbstract,DealabsBridge,HotUKDealsBridge,MydealsBridge] Fix missing price, discount and ships from information (#3956) - DealabsBridge - HotUKDealsBridge - MydealsBridge Add the currency in the i8n data of the bridges - PepperBridgeAbstract The Price, discount data ans Ships from information are in the HTML content anymore, so switched to the js-vue2 attributes --- bridges/DealabsBridge.php | 1 + bridges/HotUKDealsBridge.php | 1 + bridges/MydealsBridge.php | 1 + bridges/PepperBridgeAbstract.php | 69 ++++++++++++++------------------ 4 files changed, 32 insertions(+), 40 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index 4d39502c..a5a3771b 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -1914,6 +1914,7 @@ class DealabsBridge extends PepperBridgeAbstract 'request-error' => 'Impossible de joindre Dealabs', 'thread-error' => 'Impossible de déterminer l\'ID de la discussion. Vérifiez l\'URL que vous avez entré', 'no-results' => 'Il n'y a rien à afficher pour le moment :(', + 'currency' => '€', 'relative-date-indicator' => [ 'il y a', ], diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index a7e62250..44da417a 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3278,6 +3278,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'request-error' => 'Could not request HotUKDeals', 'thread-error' => 'Unable to determine the thread ID. Check the URL you entered', 'no-results' => 'Ooops, looks like we could', + 'currency' => '£', 'relative-date-indicator' => [ 'ago', ], diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index d7e074a9..dda3d2a9 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2025,6 +2025,7 @@ class MydealsBridge extends PepperBridgeAbstract 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', 'no-results' => 'Ups, wir konnten nichts', + 'currency' => '€', 'relative-date-indicator' => [ 'vor', 'seit' diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 45bfe209..2516fc1e 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -113,8 +113,8 @@ class PepperBridgeAbstract extends BridgeAbstract . $this->getImage($deal) . '"/>' . $this->getHTMLTitle($item) - . $this->getPrice($deal) - . $this->getDiscount($deal) + . $this->getPrice($jsonDealData) + . $this->getDiscount($jsonDealData) . $this->getShipsFrom($deal) . $this->getShippingCost($deal) . $this->getSource($jsonDealData) @@ -273,20 +273,12 @@ HEREDOC; * Get the Price from a Deal if it exists * @return string String of the deal price */ - private function getPrice($deal) + private function getPrice($jsonDealData) { - if ( - $deal->find( - 'span[class*=thread-price]', - 0 - ) != null - ) { - return '
' . $this->i8n('price') . ' : ' - . $deal->find( - 'span[class*=thread-price]', - 0 - )->plaintext - . '
'; + if ($jsonDealData['props']['thread']['discountType'] == null) { + $price = $jsonDealData['props']['thread']['price']; + return '
' . $this->i8n('price') . ' : ' + . $price . ' ' . $this->i8n('currency') . '
'; } else { return ''; } @@ -409,23 +401,22 @@ HEREDOC; * Get the original Price and discout from a Deal if it exists * @return string String of the deal original price and discount */ - private function getDiscount($deal) + private function getDiscount($jsonDealData) { - if ($deal->find('span[class*=mute--text text--lineThrough]', 0) != null) { - $discountHtml = $deal->find('span[class=space--ml-1 size--all-l size--fromW3-xl]', 0); - if ($discountHtml != null) { - $discount = $discountHtml->plaintext; - } else { - $discount = ''; + $oldPrice = $jsonDealData['props']['thread']['nextBestPrice']; + $newPrice = $jsonDealData['props']['thread']['price']; + $percentage = $jsonDealData['props']['thread']['percentage']; + + if ($oldPrice != 0) { + // If there is no percentage calculated, then calculate it manually + if ($percentage == 0) { + $percentage = round(100 - ($newPrice * 100 / $oldPrice), 2); } return '
' . $this->i8n('discount') . ' : ' - . $deal->find( - 'span[class*=mute--text text--lineThrough]', - 0 - )->plaintext - . ' ' - . $discount - . '
'; + . $oldPrice . ' ' . $this->i8n('currency') + . '  -' + . $percentage + . ' %'; } else { return ''; } @@ -448,19 +439,17 @@ HEREDOC; */ private function getShipsFrom($deal) { - $selector = implode( - ' ', /* Notice this is a space! */ - [ - 'hide--toW2', - 'metaRibbon', - ] - ); - if ($deal->find('span[class*=' . $selector . ']', 0) != null) { - $children = $deal->find('span[class*=' . $selector . ']', 0)->children(2); - if ($children) { - return '
' . $children->plaintext . '
'; + $dealMeta = Json::decode($deal->find('div[class=threadGrid-headerMeta]', 0)->find('div[class=js-vue2]', 1)->getAttribute('data-vue2')); + $metas = $dealMeta['props']['metaRibbons']; + $shipsFrom = null; + foreach ($metas as $meta) { + if ($meta['type'] == 'dispatched-from') { + $shipsFrom = $meta['text']; } } + if ($shipsFrom != null) { + return '
' . $shipsFrom . '
'; + } return ''; } From 6878eb26aa97413ad6f04e8d15f831c65f964c32 Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 6 Feb 2024 19:32:05 +0100 Subject: [PATCH 007/423] fix: changed dom (#3958) --- bridges/ManyVidsBridge.php | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/bridges/ManyVidsBridge.php b/bridges/ManyVidsBridge.php index df4996e6..21ad950f 100644 --- a/bridges/ManyVidsBridge.php +++ b/bridges/ManyVidsBridge.php @@ -29,19 +29,20 @@ class ManyVidsBridge extends BridgeAbstract } else { throw new \Exception('nope'); } - $url = sprintf('https://www.manyvids.com/Profile/%s/Store/Videos/', $profile); $dom = getSimpleHTMLDOM($url); - $el = $dom->find('section[id="app-store-videos"]', 0); - $json = $el->getAttribute('data-store-videos'); - $json = html_entity_decode($json); - $data = Json::decode($json, false); - foreach ($data->content->items as $item) { + $videos = $dom->find('div[class^="ProfileTabGrid_card"]'); + foreach ($videos as $item) { + $a = $item->find('a', 1); + $uri = 'https://www.manyvids.com' . $a->href; + if (preg_match('#Video/(\d+)/#', $uri, $m)) { + $uid = 'manyvids/' . $m[1]; + } $this->items[] = [ - 'title' => $item->title, - 'uri' => 'https://www.manyvids.com' . $item->preview->path, - 'uid' => 'manyvids/' . $item->id, - 'content' => sprintf('', $item->videoThumb), + 'title' => $a->plaintext, + 'uri' => $uri, + 'uid' => $uid ?? $uri, + 'content' => $item->innertext, ]; } } From 6bb04d48ed80a328174e071f064c6336f1b01fe1 Mon Sep 17 00:00:00 2001 From: tillcash Date: Thu, 8 Feb 2024 00:03:25 +0530 Subject: [PATCH 008/423] [KilledbyMicrosoftBridge] New Bridge (#3961) --- bridges/KilledbyMicrosoftBridge.php | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 bridges/KilledbyMicrosoftBridge.php diff --git a/bridges/KilledbyMicrosoftBridge.php b/bridges/KilledbyMicrosoftBridge.php new file mode 100644 index 00000000..8779f0ae --- /dev/null +++ b/bridges/KilledbyMicrosoftBridge.php @@ -0,0 +1,61 @@ +formatTitle( + $service['name'], + $service['dateOpen'], + $service['dateClose'] + ); + + // Construct the content + $content = sprintf( + '

%s

Scheduled Closure: %s

', + $service['description'], + $service['dateClose'] + ); + + // Add the item to the feed + $this->items[] = [ + 'title' => $title, + 'uid' => $service['slug'], + 'uri' => $service['link'], + 'content' => $content + ]; + } + } + + private function formatTitle($name, $dateOpen, $dateClose) + { + // Extract years from dateOpen and dateClose + $yearOpen = date('Y', strtotime($dateOpen)); + $yearClose = date('Y', strtotime($dateClose)); + + // Format the title + return "{$name} ({$yearOpen} - {$yearClose})"; + } +} From 75a0a779c0587dee5c4a60d12c1bfab65d7f7ec1 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:35:24 +0100 Subject: [PATCH 009/423] Update HeiseBridge.php (#3963) fix for broken article categories --- bridges/HeiseBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index f89594ee..504bcfb5 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -179,7 +179,7 @@ class HeiseBridge extends FeedExpander } } - $categories = $article->find('.article-footer__topics ul.topics li.topics__item'); + $categories = $article->find('.article-footer__topics ul.topics li.topics__item a-topic a'); foreach ($categories as $category) { $item['categories'][] = trim($category->plaintext); } From cfef482366e296f2ab1526a97408114b4c1ef9a3 Mon Sep 17 00:00:00 2001 From: Korytov Pavel Date: Thu, 8 Feb 2024 17:36:03 +0300 Subject: [PATCH 010/423] [EconomistBridge] Handle 404s in feed gracefully (#3965) --- bridges/EconomistBridge.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bridges/EconomistBridge.php b/bridges/EconomistBridge.php index aad72275..70117cb0 100644 --- a/bridges/EconomistBridge.php +++ b/bridges/EconomistBridge.php @@ -99,7 +99,12 @@ class EconomistBridge extends FeedExpander protected function parseItem(array $item) { - $dom = getSimpleHTMLDOM($item['uri']); + try { + $dom = getSimpleHTMLDOM($item['uri']); + } catch (Exception $e) { + $item['content'] = $e->getMessage(); + return $item; + } $article = $dom->find('#new-article-template', 0); if ($article == null) { From ae2eb2f1d1af63a3068e18e9d85f92f360b60a34 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Thu, 8 Feb 2024 20:05:24 +0100 Subject: [PATCH 011/423] feat(Reddit): add parameter for web UI frontend --- bridges/RedditBridge.php | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 618463a6..9c72f996 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -38,6 +38,16 @@ class RedditBridge extends BridgeAbstract 'required' => false, 'exampleValue' => 'cats, dogs', 'title' => 'Keyword search, separated by commas' + ], + 'frontend' => [ + 'type' => 'list', + 'name' => 'frontend', + 'title' => 'choose frontend for reddit', + 'values' => [ + 'old.reddit.com' => 'https://old.reddit.com', + 'reddit.com' => 'https://reddit.com', + 'libreddit.kavin.rocks' => 'https://libreddit.kavin.rocks', + ] ] ], 'single' => [ @@ -109,6 +119,10 @@ class RedditBridge extends BridgeAbstract { $user = false; $comments = false; + $frontend = $this->getInput('frontend'); + if ($frontend == '') { + $frontend = 'https://old.reddit.com'; + } $section = $this->getInput('d'); switch ($this->queriedContext) { @@ -175,6 +189,10 @@ class RedditBridge extends BridgeAbstract $item['timestamp'] = $data->created_utc; $item['uri'] = $this->urlEncodePathParts($data->permalink); + if ($frontend != 'https://old.reddit.com') { + $item['uri'] = preg_replace('#^https://old\.reddit\.com#', $frontend, $item['uri']); + } + $item['categories'] = []; if ($post->kind == 't1') { From 8a6798a2276026ec8fe06b0056607a6e461cf330 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 9 Feb 2024 07:27:16 +0100 Subject: [PATCH 012/423] fix: escape token for html context (#3966) --- README.md | 6 +++--- bridges/AnnasArchiveBridge.php | 6 ++++-- bridges/BookMyShowBridge.php | 28 +++++++++++++++------------- config/nginx.conf | 1 + lib/BridgeCard.php | 5 ++--- 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index cadba3b9..f8d08058 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ server { server_name example.com; access_log /var/log/nginx/rss-bridge.access.log; error_log /var/log/nginx/rss-bridge.error.log; + log_not_found off; # Intentionally not setting a root folder here @@ -115,23 +116,22 @@ server { alias /var/www/rss-bridge/static/; } - # Pass off to php-fpm only when location is exactly / + # Pass off to php-fpm when location is exactly / location = / { root /var/www/rss-bridge/; include snippets/fastcgi-php.conf; + fastcgi_read_timeout 45s; fastcgi_pass unix:/run/php/rss-bridge.sock; } # Reduce spam location = /favicon.ico { access_log off; - log_not_found off; } # Reduce spam location = /robots.txt { access_log off; - log_not_found off; } } ``` diff --git a/bridges/AnnasArchiveBridge.php b/bridges/AnnasArchiveBridge.php index e8a1e8c4..acb943b4 100644 --- a/bridges/AnnasArchiveBridge.php +++ b/bridges/AnnasArchiveBridge.php @@ -126,7 +126,8 @@ class AnnasArchiveBridge extends BridgeAbstract return; } - foreach ($list->find('.w-full > .mb-4 > div > a') as $element) { + $elements = $list->find('.w-full > .mb-4 > div > a'); + foreach ($elements as $element) { $item = []; $item['title'] = $element->find('h3', 0)->plaintext; $item['author'] = $element->find('div.italic', 0)->plaintext; @@ -134,7 +135,8 @@ class AnnasArchiveBridge extends BridgeAbstract $item['content'] = $element->plaintext; $item['uid'] = $item['uri']; - if ($item_html = getSimpleHTMLDOMCached($item['uri'])) { + $item_html = getSimpleHTMLDOMCached($item['uri'], 86400 * 20); + if ($item_html) { $item_html = defaultLinkTo($item_html, self::URI); $item['content'] .= $item_html->find('main img', 0); $item['content'] .= $item_html->find('main .mt-4', 0); // Summary diff --git a/bridges/BookMyShowBridge.php b/bridges/BookMyShowBridge.php index 7064df91..6ad02fe2 100644 --- a/bridges/BookMyShowBridge.php +++ b/bridges/BookMyShowBridge.php @@ -1218,14 +1218,15 @@ EOT; $table = $this->generateEventDetailsTable($event); $imgsrc = $event['BannerURL']; + $FShareURL = $event['FShareURL']; return << -
- $table -
- More Details are available on the BookMyShow website. -EOT; + +
+ $table +
+ More Details are available on the BookMyShow website. + EOT; } /** @@ -1292,14 +1293,15 @@ EOT; $synopsis = preg_replace(self::SYNOPSIS_REGEX, '', $data['EventSynopsis']); + $eventTrailerURL = $data['EventTrailerURL']; return << -
$table
-

$innerHtml

-

${synopsis}

- More Details are available on the BookMyShow website and a trailer is available - here -EOT; + +
$table
+

$innerHtml

+

$synopsis

+ More Details are available on the BookMyShow website and a trailer is available + here + EOT; } /** diff --git a/config/nginx.conf b/config/nginx.conf index f0f189e7..c65f8e00 100644 --- a/config/nginx.conf +++ b/config/nginx.conf @@ -13,6 +13,7 @@ server { location ~ \.php$ { include snippets/fastcgi-php.conf; + fastcgi_read_timeout 45s; fastcgi_pass unix:/var/run/php/php8.2-fpm.sock; } } diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index e5456f33..c4677b9d 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -104,9 +104,8 @@ final class BridgeCard EOD; - if ($token) { - // todo: maybe escape the token? - $form .= sprintf('', $token); + if (Configuration::getConfig('authentication', 'token') && $token) { + $form .= sprintf('', e($token)); } if (!empty($contextName)) { From 6f731b20a9a0a6f7913b77dc9c92dd59a21279fc Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 9 Feb 2024 08:03:04 +0100 Subject: [PATCH 013/423] fix(DarkReading): official rss endpoint changed (#3967) --- bridges/DarkReadingBridge.php | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/bridges/DarkReadingBridge.php b/bridges/DarkReadingBridge.php index 4f1622e3..d8b88cb6 100644 --- a/bridges/DarkReadingBridge.php +++ b/bridges/DarkReadingBridge.php @@ -9,7 +9,7 @@ class DarkReadingBridge extends FeedExpander const PARAMETERS = [ [ 'feed' => [ - 'name' => 'Feed', + 'name' => 'Feed (NOT IN USE)', 'type' => 'list', 'values' => [ 'All Dark Reading Stories' => '000_AllArticles', @@ -41,17 +41,7 @@ class DarkReadingBridge extends FeedExpander public function collectData() { - $feed = $this->getInput('feed'); - $feed_splitted = explode('_', $feed); - $feed_id = $feed_splitted[0]; - $feed_name = $feed_splitted[1]; - if (empty($feed) || !ctype_digit($feed_id) || !preg_match('/[A-Za-z%20\/]/', $feed_name)) { - returnClientError('Invalid feed, please check the "feed" parameter.'); - } - $feed_url = $this->getURI() . 'rss_simple.asp'; - if ($feed_id != '000') { - $feed_url .= '?f_n=' . $feed_id . '&f_ln=' . $feed_name; - } + $feed_url = 'https://www.darkreading.com/rss.xml'; $limit = $this->getInput('limit') ?? 10; $this->collectExpandableDatas($feed_url, $limit); } @@ -71,7 +61,7 @@ class DarkReadingBridge extends FeedExpander private function extractArticleContent($article) { - $content = $article->find('div.article-content', 0)->innertext; + $content = $article->find('div.ContentModule-Wrapper', 0)->innertext; foreach ( [ From 46ac77590e275bf3724e23f191cdfcd011bb72f9 Mon Sep 17 00:00:00 2001 From: tillcash Date: Fri, 9 Feb 2024 14:09:03 +0530 Subject: [PATCH 014/423] [KilledbyMicrosoftBridge] Update: Adjusted content format for consistency (#3968) --- bridges/KilledbyMicrosoftBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/KilledbyMicrosoftBridge.php b/bridges/KilledbyMicrosoftBridge.php index 8779f0ae..918c1aaa 100644 --- a/bridges/KilledbyMicrosoftBridge.php +++ b/bridges/KilledbyMicrosoftBridge.php @@ -34,7 +34,7 @@ class KilledbyMicrosoftBridge extends BridgeAbstract // Construct the content $content = sprintf( - '

%s

Scheduled Closure: %s

', + '

%s

Scheduled closure on %s.

', $service['description'], $service['dateClose'] ); From 7b2ac362648d755874171300d1f5b2d9aea0ff46 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 9 Feb 2024 22:27:35 +0100 Subject: [PATCH 015/423] chore: move committed third-party deps to lib (#3973) --- lib/bootstrap.php | 6 +++--- {vendor => lib}/parsedown/LICENSE.txt | 0 {vendor => lib}/parsedown/Parsedown.php | 0 {vendor => lib}/php-urljoin/LICENSE | 0 {vendor => lib}/php-urljoin/src/urljoin.php | 0 {vendor => lib}/simplehtmldom/LICENSE | 0 {vendor => lib}/simplehtmldom/simple_html_dom.php | 0 phpcs.xml | 3 +++ 8 files changed, 6 insertions(+), 3 deletions(-) rename {vendor => lib}/parsedown/LICENSE.txt (100%) rename {vendor => lib}/parsedown/Parsedown.php (100%) rename {vendor => lib}/php-urljoin/LICENSE (100%) rename {vendor => lib}/php-urljoin/src/urljoin.php (100%) rename {vendor => lib}/simplehtmldom/LICENSE (100%) rename {vendor => lib}/simplehtmldom/simple_html_dom.php (100%) diff --git a/lib/bootstrap.php b/lib/bootstrap.php index 6465f5f9..29c8b97d 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -18,9 +18,9 @@ $files = [ __DIR__ . '/../lib/url.php', __DIR__ . '/../lib/seotags.php', // Vendor - __DIR__ . '/../vendor/parsedown/Parsedown.php', - __DIR__ . '/../vendor/php-urljoin/src/urljoin.php', - __DIR__ . '/../vendor/simplehtmldom/simple_html_dom.php', + __DIR__ . '/../lib/parsedown/Parsedown.php', + __DIR__ . '/../lib/php-urljoin/src/urljoin.php', + __DIR__ . '/../lib/simplehtmldom/simple_html_dom.php', ]; foreach ($files as $file) { require_once $file; diff --git a/vendor/parsedown/LICENSE.txt b/lib/parsedown/LICENSE.txt similarity index 100% rename from vendor/parsedown/LICENSE.txt rename to lib/parsedown/LICENSE.txt diff --git a/vendor/parsedown/Parsedown.php b/lib/parsedown/Parsedown.php similarity index 100% rename from vendor/parsedown/Parsedown.php rename to lib/parsedown/Parsedown.php diff --git a/vendor/php-urljoin/LICENSE b/lib/php-urljoin/LICENSE similarity index 100% rename from vendor/php-urljoin/LICENSE rename to lib/php-urljoin/LICENSE diff --git a/vendor/php-urljoin/src/urljoin.php b/lib/php-urljoin/src/urljoin.php similarity index 100% rename from vendor/php-urljoin/src/urljoin.php rename to lib/php-urljoin/src/urljoin.php diff --git a/vendor/simplehtmldom/LICENSE b/lib/simplehtmldom/LICENSE similarity index 100% rename from vendor/simplehtmldom/LICENSE rename to lib/simplehtmldom/LICENSE diff --git a/vendor/simplehtmldom/simple_html_dom.php b/lib/simplehtmldom/simple_html_dom.php similarity index 100% rename from vendor/simplehtmldom/simple_html_dom.php rename to lib/simplehtmldom/simple_html_dom.php diff --git a/phpcs.xml b/phpcs.xml index 21e1f50a..bd1aca28 100644 --- a/phpcs.xml +++ b/phpcs.xml @@ -8,6 +8,9 @@ ./static ./vendor + ./lib/parsedown + ./lib/php-urljoin + ./lib/simplehtmldom ./templates ./config.default.ini.php ./config.ini.php From df7b91a2a32762d015478bb8862e22367d18b898 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 9 Feb 2024 22:39:45 +0100 Subject: [PATCH 016/423] chore: upgrade composer root deps (#3974) composer update --root-reqs Loading composer repositories with package information Updating dependencies Lock file operations: 0 installs, 2 updates, 0 removals - Upgrading phpunit/phpunit (9.6.9 => 9.6.11) - Upgrading squizlabs/php_codesniffer (3.7.2 => 3.8.1) Writing lock file Installing dependencies from lock file (including require-dev) Package operations: 0 installs, 2 updates, 0 removals - Upgrading phpunit/phpunit (9.6.9 => 9.6.11): Extracting archive - Upgrading squizlabs/php_codesniffer (3.7.2 => 3.8.1): Extracting archive Generating autoload files 26 packages you are using are looking for funding. Use the `composer fund` command to find out more! No security vulnerability advisories found. --- composer.lock | 71 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/composer.lock b/composer.lock index 39e9cfc4..94a71227 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "24083060ddb8be9a95e75f6596e3bb83", + "content-hash": "6103a05bc4ac2c33281ef349fd8ff968", "packages": [], "packages-dev": [ { @@ -623,16 +623,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.9", + "version": "9.6.11", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "a9aceaf20a682aeacf28d582654a1670d8826778" + "reference": "810500e92855eba8a7a5319ae913be2da6f957b0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/a9aceaf20a682aeacf28d582654a1670d8826778", - "reference": "a9aceaf20a682aeacf28d582654a1670d8826778", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/810500e92855eba8a7a5319ae913be2da6f957b0", + "reference": "810500e92855eba8a7a5319ae913be2da6f957b0", "shasum": "" }, "require": { @@ -706,7 +706,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.9" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.11" }, "funding": [ { @@ -722,7 +722,7 @@ "type": "tidelift" } ], - "time": "2023-06-11T06:13:56+00:00" + "time": "2023-08-19T07:10:56+00:00" }, { "name": "sebastian/cli-parser", @@ -1690,16 +1690,16 @@ }, { "name": "squizlabs/php_codesniffer", - "version": "3.7.2", + "version": "3.8.1", "source": { "type": "git", - "url": "https://github.com/squizlabs/PHP_CodeSniffer.git", - "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879" + "url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git", + "reference": "14f5fff1e64118595db5408e946f3a22c75807f7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/ed8e00df0a83aa96acf703f8c2979ff33341f879", - "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879", + "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/14f5fff1e64118595db5408e946f3a22c75807f7", + "reference": "14f5fff1e64118595db5408e946f3a22c75807f7", "shasum": "" }, "require": { @@ -1709,11 +1709,11 @@ "php": ">=5.4.0" }, "require-dev": { - "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0" + "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.3.4" }, "bin": [ - "bin/phpcs", - "bin/phpcbf" + "bin/phpcbf", + "bin/phpcs" ], "type": "library", "extra": { @@ -1728,22 +1728,45 @@ "authors": [ { "name": "Greg Sherwood", - "role": "lead" + "role": "Former lead" + }, + { + "name": "Juliette Reinders Folmer", + "role": "Current lead" + }, + { + "name": "Contributors", + "homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer/graphs/contributors" } ], "description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.", - "homepage": "https://github.com/squizlabs/PHP_CodeSniffer", + "homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer", "keywords": [ "phpcs", "standards", "static analysis" ], "support": { - "issues": "https://github.com/squizlabs/PHP_CodeSniffer/issues", - "source": "https://github.com/squizlabs/PHP_CodeSniffer", - "wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki" + "issues": "https://github.com/PHPCSStandards/PHP_CodeSniffer/issues", + "security": "https://github.com/PHPCSStandards/PHP_CodeSniffer/security/policy", + "source": "https://github.com/PHPCSStandards/PHP_CodeSniffer", + "wiki": "https://github.com/PHPCSStandards/PHP_CodeSniffer/wiki" }, - "time": "2023-02-22T23:07:41+00:00" + "funding": [ + { + "url": "https://github.com/PHPCSStandards", + "type": "github" + }, + { + "url": "https://github.com/jrfnl", + "type": "github" + }, + { + "url": "https://opencollective.com/php_codesniffer", + "type": "open_collective" + } + ], + "time": "2024-01-11T20:47:48+00:00" }, { "name": "theseer/tokenizer", @@ -1808,9 +1831,9 @@ "ext-openssl": "*", "ext-libxml": "*", "ext-simplexml": "*", - "ext-json": "*", - "ext-intl": "*" + "ext-dom": "*", + "ext-json": "*" }, "platform-dev": [], - "plugin-api-version": "2.0.0" + "plugin-api-version": "2.6.0" } From ff7840d60f212dd5f395378103bc22f0c273641d Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 9 Feb 2024 22:51:10 +0100 Subject: [PATCH 017/423] chore: prepare for introduction of php-webdriver/webdriver (Selenium) (#3975) --- composer.json | 1 + lib/bootstrap.php | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/composer.json b/composer.json index 0e7abb84..34f1a80f 100644 --- a/composer.json +++ b/composer.json @@ -36,6 +36,7 @@ "squizlabs/php_codesniffer": "^3.6" }, "suggest": { + "php-webdriver/webdriver": "Required for Selenium usage", "ext-memcached": "Allows to use memcached as cache type", "ext-sqlite3": "Allows to use an SQLite database for caching", "ext-zip": "Required for FDroidRepoBridge", diff --git a/lib/bootstrap.php b/lib/bootstrap.php index 29c8b97d..bfc7be39 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -1,5 +1,9 @@ Date: Sat, 10 Feb 2024 04:42:22 +0100 Subject: [PATCH 018/423] Adopt WebDriverAbstract as a solution for active (JavaScript) websites (#3971) * first working version --------- Co-authored-by: Dag --- bridges/GULPProjekteBridge.php | 164 ++++++++++++++++++ bridges/ScalableCapitalBlogBridge.php | 73 ++++++++ config.default.ini.php | 10 ++ docs/05_Bridge_API/04_WebDriverAbstract.md | 83 +++++++++ ...4_XPathAbstract.md => 05_XPathAbstract.md} | 0 docs/05_Bridge_API/index.md | 3 +- lib/WebDriverAbstract.php | 141 +++++++++++++++ 7 files changed, 473 insertions(+), 1 deletion(-) create mode 100644 bridges/GULPProjekteBridge.php create mode 100644 bridges/ScalableCapitalBlogBridge.php create mode 100644 docs/05_Bridge_API/04_WebDriverAbstract.md rename docs/05_Bridge_API/{04_XPathAbstract.md => 05_XPathAbstract.md} (100%) create mode 100644 lib/WebDriverAbstract.php diff --git a/bridges/GULPProjekteBridge.php b/bridges/GULPProjekteBridge.php new file mode 100644 index 00000000..e0bb8cbe --- /dev/null +++ b/bridges/GULPProjekteBridge.php @@ -0,0 +1,164 @@ +addArguments(['--accept-lang=de']); + return $chromeOptions; + } + + /** + * @throws Facebook\WebDriver\Exception\NoSuchElementException + * @throws Facebook\WebDriver\Exception\TimeoutException + */ + protected function clickAwayCookieBanner() + { + $this->getDriver()->wait()->until(WebDriverExpectedCondition::visibilityOfElementLocated(WebDriverBy::id('onetrust-reject-all-handler'))); + $buttonRejectCookies = $this->getDriver()->findElement(WebDriverBy::id('onetrust-reject-all-handler')); + $buttonRejectCookies->click(); + $this->getDriver()->wait()->until(WebDriverExpectedCondition::invisibilityOfElementLocated(WebDriverBy::id('onetrust-reject-all-handler'))); + } + + /** + * @throws Facebook\WebDriver\Exception\NoSuchElementException + * @throws Facebook\WebDriver\Exception\TimeoutException + */ + protected function clickNextPage() + { + $nextPage = $this->getDriver()->findElement(WebDriverBy::xpath('//app-linkable-paginator//li[@id="next-page"]/a')); + $href = $nextPage->getAttribute('href'); + $nextPage->click(); + $this->getDriver()->wait()->until(WebDriverExpectedCondition::not( + WebDriverExpectedCondition::presenceOfElementLocated( + WebDriverBy::xpath('//app-linkable-paginator//li[@id="next-page"]/a[@href="' . $href . '"]') + ) + )); + } + + /** + * Returns the uri of the 'Projektanbieter' logo or false if there is + * no logo present in the item. + * + * @return string | false + */ + protected function getLogo(RemoteWebElement $item) + { + try { + $logo = $item->findElement(WebDriverBy::tagName('img'))->getAttribute('src'); + if (str_starts_with($logo, 'http')) { + // different domain + return $logo; + } else { + // relative path + $remove = substr(self::URI, strrpos(self::URI, '/') + 1); + return substr(self::URI, 0, -strlen($remove)) . $logo; + } + } catch (NoSuchElementException $e) { + return false; + } + } + + /** + * Converts a string like "vor einigen Minuten" into a reasonable timestamp. + * Long and complicated, but we don't want to be more specific than + * the information we have available. + * + * @throws Exception If the DateInterval can't be parsed. + */ + protected function getTimestamp(string $timeAgo): int + { + $dateTime = new DateTime(); + $dateArray = explode(' ', $dateTime->format('Y m d H i s')); + $quantityStr = explode(' ', $timeAgo)[1]; + // convert possible word into a number + if (in_array($quantityStr, ['einem', 'einer', 'einigen'])) { + $quantity = 1; + } else { + $quantity = intval($quantityStr); + } + // subtract time ago + inferior units for lower precision + if (str_contains($timeAgo, 'Sekunde')) { + $interval = new DateInterval('PT' . $quantity . 'S'); + } elseif (str_contains($timeAgo, 'Minute')) { + $interval = new DateInterval('PT' . $quantity . 'M' . $dateArray[5] . 'S'); + } elseif (str_contains($timeAgo, 'Stunde')) { + $interval = new DateInterval('PT' . $quantity . 'H' . $dateArray[4] . 'M' . $dateArray[5] . 'S'); + } elseif (str_contains($timeAgo, 'Tag')) { + $interval = new DateInterval('P' . $quantity . 'DT' . $dateArray[3] . 'H' . $dateArray[4] . 'M' . $dateArray[5] . 'S'); + } else { + throw new UnexpectedValueException($timeAgo); + } + $dateTime = $dateTime->sub($interval); + return $dateTime->getTimestamp(); + } + + /** + * The main loop which clicks through search result pages and puts + * the content into the $items array. + * + * @throws Facebook\WebDriver\Exception\NoSuchElementException + * @throws Facebook\WebDriver\Exception\TimeoutException + */ + public function collectData() + { + parent::collectData(); + + try { + $this->clickAwayCookieBanner(); + $this->setIcon($this->getDriver()->findElement(WebDriverBy::xpath('//link[@rel="shortcut icon"]'))->getAttribute('href')); + + while (true) { + $items = $this->getDriver()->findElements(WebDriverBy::tagName('app-project-view')); + foreach ($items as $item) { + $feedItem = new FeedItem(); + + $heading = $item->findElement(WebDriverBy::xpath('.//app-heading-tag/h1/a')); + $feedItem->setTitle($heading->getText()); + $feedItem->setURI('https://www.gulp.de' . $heading->getAttribute('href')); + $info = $item->findElement(WebDriverBy::tagName('app-icon-info-list')); + if ($logo = $this->getLogo($item)) { + $feedItem->setEnclosures([$logo]); + } + if (str_contains($info->getText(), 'Projektanbieter:')) { + $feedItem->setAuthor($info->findElement(WebDriverBy::xpath('.//li/span[2]/span'))->getText()); + } else { + // mostly "Direkt vom Auftraggeber" or "GULP Agentur" + $feedItem->setAuthor($item->findElement(WebDriverBy::tagName('b'))->getText()); + } + $feedItem->setContent($item->findElement(WebDriverBy::xpath('.//p[@class="description"]'))->getText()); + $timeAgo = $item->findElement(WebDriverBy::xpath('.//small[contains(@class, "time-ago")]'))->getText(); + $feedItem->setTimestamp($this->getTimestamp($timeAgo)); + + $this->items[] = $feedItem; + } + + if (count($this->items) < self::MAXITEMS) { + $this->clickNextPage(); + } else { + break; + } + } + } finally { + $this->cleanUp(); + } + } +} diff --git a/bridges/ScalableCapitalBlogBridge.php b/bridges/ScalableCapitalBlogBridge.php new file mode 100644 index 00000000..6f95efb3 --- /dev/null +++ b/bridges/ScalableCapitalBlogBridge.php @@ -0,0 +1,73 @@ +addArguments(['--accept-lang=de']); + return $chromeOptions; + } + + /** + * Puts the content of the first page into the $items array. + * + * @throws Facebook\WebDriver\Exception\NoSuchElementException + * @throws Facebook\WebDriver\Exception\TimeoutException + */ + public function collectData() + { + parent::collectData(); + + try { + // wait until last item is loaded + $this->getDriver()->wait()->until(WebDriverExpectedCondition::visibilityOfElementLocated( + WebDriverBy::xpath('//div[contains(@class, "articles")]//div[@class="items"]//div[contains(@class, "item")][15]') + )); + $this->setIcon($this->getDriver()->findElement(WebDriverBy::xpath('//link[@rel="shortcut icon"]'))->getAttribute('href')); + + $items = $this->getDriver()->findElements(WebDriverBy::xpath('//div[contains(@class, "articles")]//div[@class="items"]//div[contains(@class, "item")]')); + foreach ($items as $item) { + $feedItem = new FeedItem(); + + $feedItem->setEnclosures(['https://de.scalable.capital' . $item->findElement(WebDriverBy::tagName('img'))->getAttribute('src')]); + $heading = $item->findElement(WebDriverBy::tagName('a')); + $feedItem->setTitle($heading->getText()); + $feedItem->setURI('https://de.scalable.capital' . $heading->getAttribute('href')); + $feedItem->setContent($item->findElement(WebDriverBy::xpath('.//div[@class="summary"]'))->getText()); + $date = $item->findElement(WebDriverBy::xpath('.//div[@class="published-date"]'))->getText(); + $feedItem->setTimestamp($this->formatItemTimestamp($date)); + $feedItem->setAuthor($item->findElement(WebDriverBy::xpath('.//div[@class="author"]'))->getText()); + + $this->items[] = $feedItem; + } + } finally { + $this->cleanUp(); + } + } + + /** + * Converts the given date (dd.mm.yyyy) into a timestamp. + * + * @param $value string + * @return int + */ + protected function formatItemTimestamp($value) + { + $formatter = new IntlDateFormatter('de', IntlDateFormatter::LONG, IntlDateFormatter::NONE); + return $formatter->parse($value); + } +} \ No newline at end of file diff --git a/config.default.ini.php b/config.default.ini.php index 7729afcb..8f7de832 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -99,6 +99,16 @@ name = "Hidden proxy name" ; false = disabled (default) by_bridge = false +[webdriver] + +; Sets the url of the webdriver or selenium server +selenium_server_url = "http://localhost:4444" + +; Sets whether the browser should run in headless mode (no visible ui) +; true = enabled +; false = disabled (default) +headless = false + [authentication] ; HTTP basic authentication diff --git a/docs/05_Bridge_API/04_WebDriverAbstract.md b/docs/05_Bridge_API/04_WebDriverAbstract.md new file mode 100644 index 00000000..60b5e99d --- /dev/null +++ b/docs/05_Bridge_API/04_WebDriverAbstract.md @@ -0,0 +1,83 @@ +`WebDriverAbstract` extends [`BridgeAbstract`](./02_BridgeAbstract.md) and adds functionality for generating feeds +from active websites that use XMLHttpRequest (XHR) to load content and / or JavaScript to +modify content. +It highly depends on the php-webdriver library which offers Selenium WebDriver bindings for PHP. + +- https://github.com/php-webdriver/php-webdriver (Project Repository) +- https://php-webdriver.github.io/php-webdriver/latest/ (API) + +Please note that this class is intended as a solution for websites _that cannot be covered +by the other classes_. The WebDriver starts a browser and is therefore very resource-intensive. + +# Configuration + +You need a running WebDriver to use bridges that depend on `WebDriverAbstract`. +The easiest way is to start the Selenium server from the project of the same name: +``` +docker run -d -p 4444:4444 --shm-size="2g" docker.io/selenium/standalone-chrome:latest +``` + +- https://github.com/SeleniumHQ/docker-selenium + +With these parameters only one browser window can be started at a time. +On a multi-user site, Selenium Grid should be used +and the number of sessions should be adjusted to the number of processor cores. + +Finally, the `config.ini.php` file must be adjusted so that the WebDriver +can find the Selenium server: +``` +[webdriver] + +selenium_server_url = "http://localhost:4444" +``` + +# Development + +While you are programming a new bridge, it is easier to start a local WebDriver because then you can see what is happening and where the errors are. I've also had good experience recording the process with a screen video to find any timing problems. + +``` +chromedriver --port=4444 +``` + +- https://chromedriver.chromium.org/ + +If you start rss-bridge from a container, then Chrome driver is only accessible +if you call it with the `--allowed-ips` option so that it binds to all network interfaces. + +``` +chromedriver --port=4444 --allowed-ips=192.168.1.42 +``` + +The **most important rule** is that after an event such as loading the web page +or pressing a button, you often have to explicitly wait for the desired elements to appear. + +A simple example is the bridge `ScalableCapitalBlogBridge.php`. +A more complex and relatively complete example is the bridge `GULPProjekteBridge.php`. + +# Template + +Use this template to create your own bridge. + +```PHP +cleanUp(); + } + } +} + +``` \ No newline at end of file diff --git a/docs/05_Bridge_API/04_XPathAbstract.md b/docs/05_Bridge_API/05_XPathAbstract.md similarity index 100% rename from docs/05_Bridge_API/04_XPathAbstract.md rename to docs/05_Bridge_API/05_XPathAbstract.md diff --git a/docs/05_Bridge_API/index.md b/docs/05_Bridge_API/index.md index 06445246..ea6fd315 100644 --- a/docs/05_Bridge_API/index.md +++ b/docs/05_Bridge_API/index.md @@ -8,6 +8,7 @@ Base class | Description -----------|------------ [`BridgeAbstract`](./02_BridgeAbstract.md) | This class is intended for standard _Bridges_ that need to filter HTML pages for content. [`FeedExpander`](./03_FeedExpander.md) | Expand/modify existing feed urls -[`XPathAbstract`](./04_XPathAbstract.md) | This class is meant as an alternative base class for bridge implementations. It offers preliminary functionality for generating feeds based on _XPath expressions_. +[`WebDriverAbstract`](./04_WebDriverAbstract) | +[`XPathAbstract`](./05_XPathAbstract) | This class is meant as an alternative base class for bridge implementations. It offers preliminary functionality for generating feeds based on _XPath expressions_. For more information about how to create a new _Bridge_, read [How to create a new Bridge?](./01_How_to_create_a_new_bridge.md) \ No newline at end of file diff --git a/lib/WebDriverAbstract.php b/lib/WebDriverAbstract.php new file mode 100644 index 00000000..db2fb7b1 --- /dev/null +++ b/lib/WebDriverAbstract.php @@ -0,0 +1,141 @@ +driver; + } + + /** + * Returns the uri of the feed's icon. + * + * @return string + */ + public function getIcon() + { + return $this->feedIcon ?: parent::getIcon(); + } + + /** + * Sets the uri of the feed's icon. + * + * @param $iconurl string + */ + protected function setIcon($iconurl) + { + $this->feedIcon = $iconurl; + } + + /** + * Returns the ChromeOptions object. + * + * If the configuration parameter 'headless' is set to true, the + * argument '--headless' is added. Override this to change or add + * more options. + * + * @return ChromeOptions + */ + protected function getBrowserOptions() + { + $chromeOptions = new ChromeOptions(); + if (Configuration::getConfig('webdriver', 'headless')) { + $chromeOptions->addArguments(['--headless']); // --window-size=1024,1024 + } + return $chromeOptions; + } + + /** + * Returns the DesiredCapabilities object for the Chrome browser. + * + * The Chrome options are added. Override this to change or add + * more capabilities. + * + * @return WebDriverCapabilities + */ + protected function getDesiredCapabilities(): WebDriverCapabilities + { + $desiredCapabilities = DesiredCapabilities::chrome(); + $desiredCapabilities->setCapability(ChromeOptions::CAPABILITY, $this->getBrowserOptions()); + return $desiredCapabilities; + } + + /** + * Constructs the remote webdriver with the url of the remote (Selenium) + * webdriver server and the desired capabilities. + * + * This should be called in collectData() first. + */ + protected function prepareWebDriver() + { + $server = Configuration::getConfig('webdriver', 'selenium_server_url'); + $this->driver = RemoteWebDriver::create($server, $this->getDesiredCapabilities()); + } + + /** + * Maximizes the remote browser window (often important for reactive sites + * which change their appearance depending on the window size) and opens + * the uri set in the constant URI. + */ + protected function prepareWindow() + { + $this->getDriver()->manage()->window()->maximize(); + $this->getDriver()->get($this->getURI()); + } + + /** + * Closes the remote browser window and shuts down the remote webdriver + * connection. + * + * This must be called at the end of scraping, for example within a + * 'finally' block. + */ + protected function cleanUp() + { + $this->getDriver()->quit(); + } + + /** + * Do your web scraping here and fill the $items array. + * + * Override this but call parent() first. + * Don't forget to call cleanUp() at the end. + */ + public function collectData() + { + $this->prepareWebDriver(); + $this->prepareWindow(); + } +} \ No newline at end of file From 257799be8ea3e1fe81a58320c9381c9191a1ff51 Mon Sep 17 00:00:00 2001 From: Eugene Molotov Date: Sat, 10 Feb 2024 19:59:39 +0500 Subject: [PATCH 019/423] [Vk2Bridge] Alternative bridge for VK (#3878) --- bridges/Vk2Bridge.php | 323 +++++++++++++++++++++++++++++++++ docs/10_Bridge_Specific/Vk2.md | 41 +++++ 2 files changed, 364 insertions(+) create mode 100644 bridges/Vk2Bridge.php create mode 100644 docs/10_Bridge_Specific/Vk2.md diff --git a/bridges/Vk2Bridge.php b/bridges/Vk2Bridge.php new file mode 100644 index 00000000..0bc0879f --- /dev/null +++ b/bridges/Vk2Bridge.php @@ -0,0 +1,323 @@ + [ + 'name' => 'Короткое имя группы или профиля (из ссылки)', + 'exampleValue' => 'goblin_oper_ru', + 'required' => true + ], + 'hide_reposts' => [ + 'name' => 'Скрыть репосты', + 'type' => 'checkbox', + ] + ] + ]; + + const CONFIGURATION = [ + 'access_token' => [ + 'required' => true, + ], + ]; + + const TEST_DETECT_PARAMETERS = [ + 'https://vk.com/id1' => ['u' => 'id1'], + 'https://vk.com/groupname' => ['u' => 'groupname'], + 'https://m.vk.com/groupname' => ['u' => 'groupname'], + 'https://vk.com/groupname/anythingelse' => ['u' => 'groupname'], + 'https://vk.com/groupname?w=somethingelse' => ['u' => 'groupname'], + 'https://vk.com/with_underscore' => ['u' => 'with_underscore'], + 'https://vk.com/vk.cats' => ['u' => 'vk.cats'], + ]; + + protected $ownerNames = []; + protected $pageName; + private $urlRegex = '/vk\.com\/([\w.]+)/'; + private $rateLimitCacheKey = 'vk2_rate_limit'; + + public function getURI() + { + if (!is_null($this->getInput('u'))) { + return urljoin(static::URI, urlencode($this->getInput('u'))); + } + + return parent::getURI(); + } + + public function getName() + { + if ($this->pageName) { + return $this->pageName; + } + + return parent::getName(); + } + + public function detectParameters($url) + { + if (preg_match($this->urlRegex, $url, $matches)) { + return ['u' => $matches[1]]; + } + + return null; + } + + protected function getPostURI($post) + { + $r = 'https://vk.com/wall' . $post['owner_id'] . '_'; + if (isset($post['reply_post_id'])) { + $r .= $post['reply_post_id'] . '?reply=' . $post['id'] . '&thread=' . $post['parents_stack'][0]; + } else { + $r .= $post['id']; + } + return $r; + } + + // This function is based on SlackCoyote's vkfeed2rss + // https://github.com/em92/vkfeed2rss + protected function generateContentFromPost($post) + { + // it's what we will return + $ret = $post['text']; + + // html special characters convertion + $ret = htmlentities($ret, ENT_QUOTES | ENT_HTML401); + // change all linebreak to HTML compatible
+ $ret = nl2br($ret); + + $ret = "

$ret

"; + + // find URLs + $ret = preg_replace( + '/((https?|ftp|gopher)\:\/\/[a-zA-Z0-9\-\.]+(:[a-zA-Z0-9]*)?\/?([@\w\-\+\.\?\,\'\/&%\$#\=~\x5C])*)/', + "$1", + $ret + ); + + // find [id1|Pawel Durow] form links + $ret = preg_replace('/\[(\w+)\|([^\]]+)\]/', "$2", $ret); + + + // attachments + if (isset($post['attachments'])) { + // level 1 + foreach ($post['attachments'] as $attachment) { + if ($attachment['type'] == 'video') { + // VK videos + $title = e($attachment['video']['title']); + $photo = e($this->getImageURLWithLargestWidth($attachment['video']['image'])); + $href = "https://vk.com/video{$attachment['video']['owner_id']}_{$attachment['video']['id']}"; + $ret .= "

Video: {$title}
Video: {$title}

"; + } elseif ($attachment['type'] == 'audio') { + // VK audio + $artist = e($attachment['audio']['artist']); + $title = e($attachment['audio']['title']); + $ret .= "

Audio: {$artist} - {$title}

"; + } elseif ($attachment['type'] == 'doc' and $attachment['doc']['ext'] != 'gif') { + // any doc apart of gif + $doc_url = e($attachment['doc']['url']); + $title = e($attachment['doc']['title']); + $ret .= "

Документ: {$title}

"; + } + } + // level 2 + foreach ($post['attachments'] as $attachment) { + if ($attachment['type'] == 'photo') { + // JPEG, PNG photos + // GIF in vk is a document, so, not handled as photo + $photo = e($this->getImageURLWithLargestWidth($attachment['photo']['sizes'])); + $text = e($attachment['photo']['text']); + $ret .= "

{$text}

"; + } elseif ($attachment['type'] == 'doc' and $attachment['doc']['ext'] == 'gif') { + // GIF docs + $url = e($attachment['doc']['url']); + $ret .= "

"; + } elseif ($attachment['type'] == 'link') { + // links + $url = e($attachment['link']['url']); + $url = str_replace('https://m.vk.com', 'https://vk.com', $url); + $title = e($attachment['link']['title']); + if (isset($attachment['link']['photo'])) { + $photo = $this->getImageURLWithLargestWidth($attachment['link']['photo']['sizes']); + $ret .= "

{$title}
{$title}

"; + } else { + $ret .= "

{$title}

"; + } + } elseif ($attachment['type'] == 'note') { + // notes + $title = e($attachment['note']['title']); + $url = e($attachment['note']['view_url']); + $ret .= "

{$title}

"; + } elseif ($attachment['type'] == 'poll') { + // polls + $question = e($attachment['poll']['question']); + $vote_count = $attachment['poll']['votes']; + $answers = $attachment['poll']['answers']; + $ret .= "

Poll: {$question} ({$vote_count} votes)
"; + foreach ($answers as $answer) { + $text = e($answer['text']); + $votes = $answer['votes']; + $rate = $answer['rate']; + $ret .= "* {$text}: {$votes} ({$rate}%)
"; + } + $ret .= '

'; + } elseif (!in_array($attachment['type'], ['video', 'audio', 'doc'])) { + $ret .= "

Unknown attachment type: {$attachment['type']}

"; + } + } + } + + return $ret; + } + + protected function getImageURLWithLargestWidth($items) + { + usort($items, function ($a, $b) { + return $b['width'] - $a['width']; + }); + return $items[0]['url']; + } + + public function collectData() + { + if ($this->cache->get($this->rateLimitCacheKey)) { + throw new HttpException('429 Too Many Requests', 429); + } + + $u = $this->getInput('u'); + $ownerId = null; + + // getting ownerId from url + $r = preg_match('/^(club|public)(\d+)$/', $u, $matches); + if ($r) { + $ownerId = -intval($matches[2]); + } else { + $r = preg_match('/^(id)(\d+)$/', $u, $matches); + if ($r) { + $ownerId = intval($matches[2]); + } + } + + // getting owner id from API + if (is_null($ownerId)) { + $r = $this->api('groups.getById', [ + 'group_ids' => $u, + ], [100]); + if (isset($r['response'][0])) { + $ownerId = -$r['response'][0]['id']; + } else { + $r = $this->api('users.get', [ + 'user_ids' => $u, + ]); + if (count($r['response']) > 0) { + $ownerId = $r['response'][0]['id']; + } + } + } + + if (is_null($ownerId)) { + returnServerError('Could not detect owner id'); + } + + $r = $this->api('wall.get', [ + 'owner_id' => $ownerId, + 'extended' => '1', + ]); + + // preparing ownerNames dictionary + foreach ($r['response']['profiles'] as $profile) { + $this->ownerNames[$profile['id']] = $profile['first_name'] . ' ' . $profile['last_name']; + } + foreach ($r['response']['groups'] as $group) { + $this->ownerNames[-$group['id']] = $group['name']; + } + $this->generateFeed($r); + } + + protected function generateFeed($r) + { + $ownerId = 0; + + foreach ($r['response']['items'] as $post) { + if (!$ownerId) { + $ownerId = $post['owner_id']; + } + $item = new FeedItem(); + $content = $this->generateContentFromPost($post); + if (isset($post['copy_history'])) { + if ($this->getInput('hide_reposts')) { + continue; + } + $originalPost = $post['copy_history'][0]; + if ($originalPost['from_id'] < 0) { + $originalPostAuthorScreenName = 'club' . (-$originalPost['owner_id']); + } else { + $originalPostAuthorScreenName = 'id' . $originalPost['owner_id']; + } + $originalPostAuthorURI = 'https://vk.com/' . $originalPostAuthorScreenName; + $originalPostAuthorName = $this->ownerNames[$originalPost['from_id']]; + $originalPostAuthor = "$originalPostAuthorName"; + $content .= '

Репост (Пост от '; + $content .= $originalPostAuthor; + $content .= '):

'; + $content .= $this->generateContentFromPost($originalPost); + } + $item->setContent($content); + $item->setTimestamp($post['date']); + $item->setAuthor($this->ownerNames[$post['from_id']]); + $item->setTitle($this->getTitle(strip_tags($content))); + $item->setURI($this->getPostURI($post)); + + $this->items[] = $item; + } + + $this->pageName = $this->ownerNames[$ownerId]; + } + + protected function getTitle($content) + { + $content = explode('
', $content)[0]; + $content = strip_tags($content); + preg_match('/^[:\,"\w\ \p{L}\(\)\?#«»\-\–\—||&\.%\\₽\/+\;\!]+/mu', htmlspecialchars_decode($content), $result); + if (count($result) == 0) { + return 'untitled'; + } + return $result[0]; + } + + protected function api($method, array $params, $expected_error_codes = []) + { + $access_token = $this->getOption('access_token'); + if (!$access_token) { + returnServerError('You cannot run VK API methods without access_token'); + } + $params['v'] = '5.131'; + $r = json_decode( + getContents( + 'https://api.vk.com/method/' . $method . '?' . http_build_query($params), + ['Authorization: Bearer ' . $access_token] + ), + true + ); + if (isset($r['error']) && !in_array($r['error']['error_code'], $expected_error_codes)) { + if ($r['error']['error_code'] == 6) { + $this->cache->set($this->rateLimitCacheKey, true, 5); + } else if ($r['error']['error_code'] == 29) { + // wall.get has limit of 5000 requests per day + // if that limit is hit, VK returns error 29 + $this->cache->set($this->rateLimitCacheKey, true, 60 * 30); + } + returnServerError('API returned error: ' . $r['error']['error_msg'] . ' (' . $r['error']['error_code'] . ')'); + } + return $r; + } +} diff --git a/docs/10_Bridge_Specific/Vk2.md b/docs/10_Bridge_Specific/Vk2.md new file mode 100644 index 00000000..7c48ad0a --- /dev/null +++ b/docs/10_Bridge_Specific/Vk2.md @@ -0,0 +1,41 @@ +Vk2Bridge +========= + +Работа этого скрипта основана [VK API](https://dev.vk.com/reference). +По сравнению с VkBridge у этого скрипта есть свои приемущества и недостатки. + +Приемущества +------------ + +- Стабильность. + Скрипт не зависит от HTML-структуры страницы VK групп или пользователей, которые могут поменяться в любой момент. + +Недостатки +---------- + +- Требуется наличие зарегистированного в ВК пользователя. + Данный пользователь должен получить `access_token`, который используется для этого скрипта. + Подробнее в разделе "Настройка" + +- Количество запросов при выключенном кэше ограничено - [5000 запросов в сутки](https://dev.vk.com/ru/reference/roadmap#%D0%9E%D0%B3%D1%80%D0%B0%D0%BD%D0%B8%D1%87%D0%B5%D0%BD%D0%B8%D1%8F%20API%20%D0%B4%D0%BB%D1%8F%20%D0%BF%D0%BE%D0%B8%D1%81%D0%BA%D0%B0) + +Настройка +--------- + +1. Перейдите по [ссылке](https://oauth.vk.com/oauth/authorize?client_id=5149410&scope=offline&redirect_uri=https://oauth.vk.com/blank.html&display=page&response_type=token) + +2. Авторизуйтесь в приложение `my_personal_app` + +3. Получите ссылку вида `https://oauth.vk.com/blank.html#access_token=MNOGO_BUKAV&expires_in=0&user_id=123456`. + Из этой ссылки скопируйте `MNOGO_BUKAV`. + +4. В `config.ini.php` в раздел Vk2Bridge вставьте `access_token` + +``` +[Vk2Bridge] +access_token = "MNOGO_BUKAV" +``` + +Примечание: в данной инструкции используется приложение, администратор которого является [@em92](https://github.com/em92). +Допускается вместо упомянутого приложения использование своего standalone-приложения. +Для этого надо в ссылке из п.1. заменить значение `client_id` на свой. From 598ee5b51eaba62dc672f9e9f6f96ac628e56263 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 14 Feb 2024 16:02:54 +0100 Subject: [PATCH 020/423] fix(pinterest): set enclosure so it emits mrss media:content prop (#3980) --- bridges/PinterestBridge.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bridges/PinterestBridge.php b/bridges/PinterestBridge.php index fc5b1c19..8338fb25 100644 --- a/bridges/PinterestBridge.php +++ b/bridges/PinterestBridge.php @@ -39,6 +39,9 @@ class PinterestBridge extends FeedExpander $pattern = '/https\:\/\/i\.pinimg\.com\/[a-zA-Z0-9]*x\//'; foreach ($this->items as $item) { $item['content'] = preg_replace($pattern, 'https://i.pinimg.com/originals/', $item['content']); + $item['enclosures'] = [ + $item['uri'], + ]; $newitems[] = $item; } $this->items = $newitems; From 4d15ffd2cf44807210cb5d2783ce3bc5c8476275 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 16 Feb 2024 03:58:15 +0100 Subject: [PATCH 021/423] [PepperBridgeAbstract,DealabsBridge,HotUKDealsBridge,MydealsBridge] (#3982) Exclude thread results Some categories showed some thread in the middle of the deals : now only the deals are handled Updated the "no results" text to follow the sites changes --- bridges/DealabsBridge.php | 2 +- bridges/HotUKDealsBridge.php | 2 +- bridges/MydealsBridge.php | 2 +- bridges/PepperBridgeAbstract.php | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index a5a3771b..c65f0c75 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -1913,7 +1913,7 @@ class DealabsBridge extends PepperBridgeAbstract 'uri-merchant' => 'search/bons-plans?merchant-id=', 'request-error' => 'Impossible de joindre Dealabs', 'thread-error' => 'Impossible de déterminer l\'ID de la discussion. Vérifiez l\'URL que vous avez entré', - 'no-results' => 'Il n'y a rien à afficher pour le moment :(', + 'no-results' => 'Aucun résultat', 'currency' => '€', 'relative-date-indicator' => [ 'il y a', diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index 44da417a..1f059123 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3277,7 +3277,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'uri-merchant' => 'search/deals?merchant-id=', 'request-error' => 'Could not request HotUKDeals', 'thread-error' => 'Unable to determine the thread ID. Check the URL you entered', - 'no-results' => 'Ooops, looks like we could', + 'no-results' => 'no results', 'currency' => '£', 'relative-date-indicator' => [ 'ago', diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index dda3d2a9..08e32a0c 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2024,7 +2024,7 @@ class MydealsBridge extends PepperBridgeAbstract 'uri-merchant' => 'search/gutscheine?merchant-id=', 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', - 'no-results' => 'Ups, wir konnten nichts', + 'no-results' => 'keine Ergebnisse', 'currency' => '€', 'relative-date-indicator' => [ 'vor', diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 2516fc1e..d0e15238 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -44,7 +44,7 @@ class PepperBridgeAbstract extends BridgeAbstract protected function collectDeals($url) { $html = getSimpleHTMLDOM($url); - $list = $html->find('article[id]'); + $list = $html->find('article[id][class*=thread--deal]]'); // Deal Image Link CSS Selector $selectorImageLink = implode( @@ -109,9 +109,9 @@ class PepperBridgeAbstract extends BridgeAbstract $item['content'] = ' From 88ccc6067cc9b8319a6eca85de4f5684c2c62a5a Mon Sep 17 00:00:00 2001 From: July Date: Tue, 26 Nov 2024 09:54:30 -0500 Subject: [PATCH 245/423] [CubariProxyBridge] Fix favicon (#4347) --- bridges/CubariProxyBridge.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bridges/CubariProxyBridge.php b/bridges/CubariProxyBridge.php index 492d5fbb..aab7c09b 100644 --- a/bridges/CubariProxyBridge.php +++ b/bridges/CubariProxyBridge.php @@ -121,4 +121,9 @@ class CubariProxyBridge extends BridgeAbstract } return $uri; } + + public function getFavicon() + { + return parent::getURI() . '/static/favicon.ico'; + } } From 6a81fc0f519bbf10070b5ae9bffd1adf84f858a6 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 28 Nov 2024 03:50:56 +0100 Subject: [PATCH 246/423] fix(file_cache): if write failure, produce log record instead of exception (#4352) --- caches/FileCache.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/caches/FileCache.php b/caches/FileCache.php index ff939bea..24a9872f 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -54,10 +54,13 @@ class FileCache implements CacheInterface ]; $cacheFile = $this->createCacheFile($key); $bytes = file_put_contents($cacheFile, serialize($item)); - // todo: Consider tightening the permissions of the created file. It usually allow others to read, depending on umask + + // TODO: Consider tightening the permissions of the created file. + // It usually allow others to read, depending on umask + if ($bytes === false) { - // Consider just logging the error here - throw new \Exception(sprintf('Failed to write to: %s', $cacheFile)); + // Typically means no disk space remaining + $this->logger->warning(sprintf('Failed to write to: %s', $cacheFile)); } } From d956471d42174a656dc2f4817ede056ee9ea77c3 Mon Sep 17 00:00:00 2001 From: Pavel Korytov Date: Mon, 2 Dec 2024 18:46:13 +0300 Subject: [PATCH 247/423] [QwenBlogBridge] Add bridge (#4353) --- bridges/QwenBlogBridge.php | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 bridges/QwenBlogBridge.php diff --git a/bridges/QwenBlogBridge.php b/bridges/QwenBlogBridge.php new file mode 100644 index 00000000..2af3f401 --- /dev/null +++ b/bridges/QwenBlogBridge.php @@ -0,0 +1,49 @@ + [ + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => true, + 'defaultValue' => 10 + ], + ] + ]; + + public function collectData() + { + $this->collectExpandableDatas(self::URI . 'index.xml', $this->getInput('limit')); + } + + protected function parseItem(array $item) + { + $dom = getSimpleHTMLDOM($item['uri']); + $content = $dom->find('div.post-content', 0); + if ($content == null) { + return $item; + } + + // Fix code blocks + foreach ($dom->find('pre.chroma') as $code_block) { + // Somehow there are tags in 
??
+            $code_block_html = str_get_html($code_block->plaintext);
+            $code = '';
+            foreach ($code_block_html->find('span.line') as $line) {
+                $code .= $line->plaintext . "\n";
+            }
+            $code_block->outertext = '
' . $code . '
';
+        }
+
+        $item['content'] = $content;
+        return $item;
+    }
+}

From 59d77d4576b0db4d09e85cb40de60d62749be554 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pred=C3=A4?= <46051820+PredaaA@users.noreply.github.com>
Date: Wed, 4 Dec 2024 17:34:35 +0100
Subject: [PATCH 248/423] [TikTokBridge] Include author profile picture (#4354)

---
 bridges/TikTokBridge.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php
index 22fdfcef..43a9cb31 100644
--- a/bridges/TikTokBridge.php
+++ b/bridges/TikTokBridge.php
@@ -29,6 +29,7 @@ class TikTokBridge extends BridgeAbstract
         $html = getSimpleHTMLDOMCached('https://www.tiktok.com/embed/' . $this->processUsername());
 
         $author = $html->find('span[data-e2e=creator-profile-userInfo-TUXText]', 0)->plaintext ?? self::NAME;
+        $authorProfilePicture = $html->find('img[data-e2e=creator-profile-userInfo-Avatar]', 0)->src ?? '';
 
         $videos = $html->find('div[data-e2e=common-videoList-VideoContainer]');
 
@@ -44,7 +45,7 @@ class TikTokBridge extends BridgeAbstract
             $image = $video->find('video', 0)->poster;
             $views = $video->find('div[data-e2e=common-Video-Count]', 0)->plaintext;
 
-            $enclosures = [$image];
+            $enclosures = [$image, $authorProfilePicture];
 
             $item['uri'] = $url;
             $item['title'] = 'Video';

From 17d142c038e6c6ab02aa8c86532aa30e8d779f42 Mon Sep 17 00:00:00 2001
From: okbaydere 
Date: Wed, 4 Dec 2024 20:54:24 +0300
Subject: [PATCH 249/423] Add StorytelBridge for Storytel list fetching (#4355)

* Add StorytelBridge for fetching Storytel lists

* Updated StorytelBridge to include URL validation and cleaned up code
---
 bridges/StorytelBridge.php | 58 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 bridges/StorytelBridge.php

diff --git a/bridges/StorytelBridge.php b/bridges/StorytelBridge.php
new file mode 100644
index 00000000..2316aacd
--- /dev/null
+++ b/bridges/StorytelBridge.php
@@ -0,0 +1,58 @@
+ [
+            'url' => [
+                'name' => 'Storytel List URL',
+                'required' => true,
+                'exampleValue' => 'https://www.storytel.com/tr/lists/23d09e0bd8fe4d998d1832ddbfa18166',
+            ],
+        ],
+    ];
+
+    public function collectData()
+    {
+        $url = $this->getInput('url');
+
+        if (!preg_match('/^https:\/\/www\.storytel\.com/', $url)) {
+            returnServerError('Invalid URL: Only Storytel URLs are allowed.');
+        }
+
+        $html = getSimpleHTMLDOM($url);
+        if (!$html) {
+            returnServerError('Unable to fetch Storytel list');
+        }
+
+        foreach ($html->find('li.sc-4615116a-1') as $element) {
+            $item = [];
+
+            $titleElement = $element->find('span.sc-b1963858-0.hoTsmF', 0);
+            $item['title'] = $titleElement ? $titleElement->plaintext : 'No title';
+
+            $authorElement = $element->find('span.sc-b1963858-0.ghYMwH', 0);
+            $item['author'] = $authorElement ? $authorElement->plaintext : 'Unknown author';
+
+            $imgElement = $element->find('img.sc-da400893-5', 0);
+            $coverUrl = $imgElement ? $imgElement->getAttribute('srcset') : '';
+            if ($coverUrl) {
+                $coverUrls = explode(', ', $coverUrl);
+                $bestCoverUrl = trim(end($coverUrls));
+                $item['content'] = '';
+            }
+
+            $linkElement = $element->find('a', 0);
+            $item['uri'] = $linkElement ? 'https://www.storytel.com' . $linkElement->getAttribute('href') : $url;
+
+            $item['content'] .= '
Author: ' . $item['author'] . '
';
+            $item['content'] .= '
More details
';
+
+            $this->items[] = $item;
+        }
+    }
+}

From bf4a918e604f6c7a60aaae7c6ad25fee568a1bdb Mon Sep 17 00:00:00 2001
From: Pavel Korytov 
Date: Thu, 5 Dec 2024 19:30:21 +0300
Subject: [PATCH 250/423] [MistralAIBridge] Add Mistral (#4356)

---
 bridges/MistralAIBridge.php | 70 +++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 bridges/MistralAIBridge.php

diff --git a/bridges/MistralAIBridge.php b/bridges/MistralAIBridge.php
new file mode 100644
index 00000000..b1c357fe
--- /dev/null
+++ b/bridges/MistralAIBridge.php
@@ -0,0 +1,70 @@
+ [
+            'limit' => [
+                'name' => 'Limit',
+                'type' => 'number',
+                'required' => true,
+                'defaultValue' => 10
+            ],
+        ]
+    ];
+
+    public function collectData()
+    {
+        $html = getSimpleHTMLDOM(self::URI . 'news/');
+        $limit = $this->getInput('limit');
+
+        $posts = $html->find('article.news-card');
+        for ($i = 0; $i < min($limit, count($posts)); $i++) {
+            $post = $posts[$i];
+            $url = self::URI . $post->find('a', 0)->href;
+            $this->parsePage($url);
+        }
+    }
+
+    private function parsePage($url)
+    {
+        $html = getSimpleHTMLDOMCached($url, 7 * 24 * 60 * 60);
+        $title = $html->find('h1.hero-title', 0)->plaintext;
+        $timestamp_tag = $html->find('i.ti-calendar', 0)->parent;
+        $timestamp = DateTime::createFromFormat('F j, Y', $timestamp_tag->plaintext)->format('U');
+
+        $content = '';
+
+        // Subheader
+        $header = $html->find('p.hero-description', 0);
+        if ($header != null) {
+            $content .= $header->outertext;
+        }
+
+        // Main content
+        $main = $html->find('$article > div.content', 0);
+
+        // Mostly YouTube videos
+        $iframes = $main->find('iframe');
+        foreach ($iframes as $iframe) {
+            $iframe->parent->removeAttribute('style');
+            $iframe->outertext = '' . $iframe->src . '';
+        }
+
+        $main = defaultLinkTo($main, self::URI);
+        $content .= $main;
+        $this->items[] = [
+            'title' => $title,
+            'timestamp' => $timestamp,
+            'content' => $content,
+            'uri' => $url,
+        ];
+    }
+}

From 4685bbdffdb46d56716b20b98b194eaba2682e64 Mon Sep 17 00:00:00 2001
From: "Florent V." 
Date: Sun, 8 Dec 2024 19:48:44 +0100
Subject: [PATCH 251/423] [EdfPricesBridge] fixing bridge (#4360)

* [EdfPricesBridge] add new brige

* [EdfPricesBridge] bad refactor

* [EdfPricesBridge] support php 7.4

* [EdfPrices Bridge] fix errors

---------

Co-authored-by: Florent VIOLLEAU 
---
 bridges/EdfPricesBridge.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bridges/EdfPricesBridge.php b/bridges/EdfPricesBridge.php
index f67ed30b..18e6f8e6 100644
--- a/bridges/EdfPricesBridge.php
+++ b/bridges/EdfPricesBridge.php
@@ -45,14 +45,14 @@ class EdfPricesBridge extends BridgeAbstract
         }
 
         // colors
-        $ulDom = $html->find('#tarif-de-l-offre-edf-tempo-current-date-html-year', 0)->nextSibling()->nextSibling()->nextSibling();
+        $ulDom = $html->find('#tarif-de-l-offre-tempo-edf-template-date-now-y', 0)->nextSibling()->nextSibling()->nextSibling();
         $elementsDom = $ulDom->find('li');
         if ($elementsDom && count($elementsDom) === 3) {
             foreach ($elementsDom as $elementDom) {
                 $item = [];
 
                 $matches = [];
-                preg_match_all('/Jour (.*) : Heures (.*) : (.*) € \/ Heures (.*) : (.*) €/um', $elementDom->innertext, $matches, PREG_SET_ORDER, 0);
+                preg_match_all('/Jour (.*) : Heures (.*) : (.*) € \/ Heures (.*) : (.*) €/um', $elementDom->innertext, $matches, PREG_SET_ORDER, 0);
 
                 if ($matches && count($matches[0]) === 6) {
                     for ($i = 0; $i < 2; $i++) {

From 9126b0f98292beed5894b425deacfcb8f895b8e6 Mon Sep 17 00:00:00 2001
From: July 
Date: Wed, 11 Dec 2024 23:41:46 -0500
Subject: [PATCH 252/423] [CubariProxyBridge] Fix favicon properly (#4364)

---
 bridges/CubariProxyBridge.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bridges/CubariProxyBridge.php b/bridges/CubariProxyBridge.php
index aab7c09b..8afd2275 100644
--- a/bridges/CubariProxyBridge.php
+++ b/bridges/CubariProxyBridge.php
@@ -122,8 +122,8 @@ class CubariProxyBridge extends BridgeAbstract
         return $uri;
     }
 
-    public function getFavicon()
+    public function getIcon()
     {
-        return parent::getURI() . '/static/favicon.ico';
+        return parent::getURI() . '/static/favicon.png';
     }
 }

From d2370320e929b2fc207b999335fd70b85db14cfb Mon Sep 17 00:00:00 2001
From: July 
Date: Wed, 11 Dec 2024 23:43:17 -0500
Subject: [PATCH 253/423] [ScribbleHubBridge] Get best-effort information
 during 403s (#4365)

---
 bridges/ScribbleHubBridge.php | 50 +++++++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/bridges/ScribbleHubBridge.php b/bridges/ScribbleHubBridge.php
index b4f7beaa..3578a77c 100644
--- a/bridges/ScribbleHubBridge.php
+++ b/bridges/ScribbleHubBridge.php
@@ -54,44 +54,72 @@ class ScribbleHubBridge extends FeedExpander
         $this->collectExpandableDatas($url);
     }
 
-    protected $author = '';
-
     private function collectList($url)
     {
         $html = getSimpleHTMLDOMCached($url);
         foreach ($html->find('.search_main_box') as $element) {
             $item = [];
 
+            $item['author'] = $element->find('[title="Author"]', 0)->plaintext;
+            $item['enclosures'] = [$element->find('.search_img img', 0)->src];
             $title = $element->find('.search_title a', 0);
             $item['title'] = $title->plaintext;
             $item['uri'] = $title->href;
+            $item['uid'] = $item['uri'];
 
             $strdate = $element->find('[title="Last Updated"]', 0)->plaintext;
             $item['timestamp'] = strtotime($strdate);
-            $item['uid'] = $item['uri'];
 
-            $details = getSimpleHTMLDOMCached($item['uri']);
-            $item['enclosures'][] = $details->find('.fic_image img', 0)->src;
-            $item['content'] = $details->find('.wi_fic_desc', 0);
-
-            foreach ($details->find('.fic_genre') as $tag) {
+            foreach ($element->find('.fic_genre') as $tag) {
                 $item['categories'][] = $tag->plaintext;
             }
+
+            // Get minimal description in case further requests fail
+            $item['content'] = str_get_html($element->find('.search_body', 0));
+            foreach ($item['content']->firstChild()->children() as $child) {
+                $child->remove();
+            }
+
+            try {
+                $details = getSimpleHTMLDOMCached($item['uri']);
+            } catch (HttpException $e) {
+                // 403 Forbidden, This means we got anti-bot response
+                if ($e->getCode() === 403 || $e->getCode() === 429) {
+                    $this->items[] = $item;
+                    continue;
+                }
+                throw $e;
+            }
+            $item['enclosures'] = [$details->find('.fic_image img', 0)->src];
+            $item['content'] = $details->find('.wi_fic_desc', 0);
+
             foreach ($details->find('.stag') as $tag) {
                 $item['categories'][] = $tag->plaintext;
             }
 
             $read_url = $details->find('.read_buttons a', 0)->href;
-            $read_html = getSimpleHTMLDOMCached($read_url);
-            $item['content'] .= '
';
+            $item['comments'] = $read_url . '#comments';
+            try {
+                $read_html = getSimpleHTMLDOMCached($read_url);
+            } catch (HttpException $e) {
+                // 403 Forbidden, This means we got anti-bot response
+                if ($e->getCode() === 403 || $e->getCode() === 429) {
+                    $this->items[] = $item;
+                    continue;
+                }
+                throw $e;
+            }
+            $item['content'] .= "
";
             $item['content'] .= $read_html->find('.chapter-title', 0);
-            $item['content'] .= '
';
+            $item['content'] .= '';
             $item['content'] .= $read_html->find('#chp_raw', 0);
 
             $this->items[] = $item;
         }
     }
 
+    protected $author = '';
+
     protected function parseItem(array $item)
     {
         //For series, filter out other series from 'All' feed

From 8234906127f60ca1d545a5bf48bddbb8ec9b6b15 Mon Sep 17 00:00:00 2001
From: July 
Date: Thu, 12 Dec 2024 11:50:43 -0500
Subject: [PATCH 254/423] [EpicGamesFreeBridge] Add new bridge (#4366)

---
 bridges/EpicGamesFreeBridge.php | 74 +++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 bridges/EpicGamesFreeBridge.php

diff --git a/bridges/EpicGamesFreeBridge.php b/bridges/EpicGamesFreeBridge.php
new file mode 100644
index 00000000..087b95be
--- /dev/null
+++ b/bridges/EpicGamesFreeBridge.php
@@ -0,0 +1,74 @@
+ [
+            'name' => 'Language',
+            'type' => 'list',
+            'values' => [
+                'English' => 'en-US',
+                'العربية' => 'ar',
+                'Deutsch' => 'de',
+                'Español (Spain)' => 'es-ES',
+                'Español (LA)' => 'es-MX',
+                'Français' => 'fr',
+                'Italiano' => 'it',
+                '日本語' => 'ja',
+                '한국어' => 'ko',
+                'Polski' => 'pl',
+                'Português (Brasil)' => 'pt-BR',
+                'Русский' => 'ru',
+                'ไทย' => 'th',
+                'Türkçe' => 'tr',
+                '简体中文' => 'zh-CN',
+                '繁體中文' => 'zh-Hant',
+             ],
+            'title' => 'Language for game information',
+            'defaultValue' => 'en-US',
+        ],
+        'country' => [
+            'name' => 'Country',
+            'title' => 'Country store to check for deals',
+            'defaultValue' => 'US',
+        ]
+    ]];
+
+    public function collectData()
+    {
+        $url = 'https://store-site-backend-static.ak.epicgames.com/freeGamesPromotions?';
+        $params = [
+            'locale' => $this->getInput('locale'),
+            'country' => $this->getInput('country'),
+            'allowCountries' => $this->getInput('country'),
+        ];
+        $url .= http_build_query($params);
+        $json = Json::decode(getContents($url));
+
+        $data = $json['data']['Catalog']['searchStore']['elements'];
+        foreach ($data as $element) {
+            if (!isset($element['promotions']['promotionalOffers'][0])) {
+                continue;
+            }
+            $item = [
+                'author' => $element['seller']['name'],
+                'content' => $element['description'],
+                'enclosures' => array_map(fn($item) => $item['url'], $element['keyImages']),
+                'timestamp' => strtotime($element['promotions']['promotionalOffers'][0]['promotionalOffers'][0]['startDate']),
+                'title' => $element['title'],
+                'url' => parent::getURI() . $this->getInput('locale') . '/p/' . $element['urlSlug'],
+            ];
+            $this->items[] = $item;
+        }
+    }
+
+    public function getURI()
+    {
+        $uri = parent::getURI() . $this->getInput('locale') . '/free-games';
+        return $uri;
+    }
+}

From f0db6a22d1a6945314950ccc2997966b29a81944 Mon Sep 17 00:00:00 2001
From: Michael Vincent <377567+Vynce@users.noreply.github.com>
Date: Thu, 12 Dec 2024 10:52:41 -0600
Subject: [PATCH 255/423] [WirecutterDealsBridge] Add bridge (#4359)

---
 bridges/WirecutterDealsBridge.php | 119 ++++++++++++++++++++++++++++++
 1 file changed, 119 insertions(+)
 create mode 100644 bridges/WirecutterDealsBridge.php

diff --git a/bridges/WirecutterDealsBridge.php b/bridges/WirecutterDealsBridge.php
new file mode 100644
index 00000000..15e9449e
--- /dev/null
+++ b/bridges/WirecutterDealsBridge.php
@@ -0,0 +1,119 @@
+getURI());
+        $json = $html->getElementById('__NEXT_DATA__');
+        $data = json_decode($json->innertext());
+
+        foreach ($data->props->pageProps->specialEvent->eventDeals as $deal) {
+            $item = [];
+            $item['uri'] = "https://www.nytimes.com/wirecutter/deals/#deal-{$deal->id}";
+            $item['title'] = $deal->title;
+            $item['timestamp'] = $deal->date;
+            $item['content'] = $this->generateContent($deal);
+            $item['categories'] = $deal->categories;
+            $item['uid'] = strval($deal->id);
+
+            $this->items[] = $item;
+        }
+    }
+
+    public function getIcon()
+    {
+        return <<<'EOD'
+        data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAD7klEQ
+        VR4AWIgFoReXcVmdnKRuymg9WoAkiUJon2+8BnhiLP17bNtX+Bs27aN0bdtY8y1bZu9m1evIrZH
+        VTW9HVsVuTNTndH5qirz5VuP7ddL3LZdk9y2AmZdk9z2TvaZP8lj3znFY/95ssdxxWPB4GHaRI1
+        Lgq6T2ct/RyAWmExa+ySP7acZMecJlgPPK7cdOdlj/5i9qFcdTG44Ifb5Lk7Pwq5tXnUA8zbZbd
+        s7O/zf8aaCT/U6zmc7r7ESaFZwAd1fvpmuzVkhel4xzeM8K+POZcHnhBbSQ5Vb6fmmffQAC3J5d
+        IkQxDSfk5YNF9O2kQqazQClPC+fF1x4nPTOZcf+epuPemmA9JQZoXr6oCtMN+StSvL/tT+XP9+k
+        lwnywr5HWCVIOFHwZxv3km5iVlE7rWMBd7CdD7OJtUEaoul+lyAn7G8Kkk6c7cXUQrrFCSAz0gH
+        AOqb4HMcaAFDnwuz12I2XbdXL6f2uEP09kE+e0RoaYjMTgC16uaJE7d8ZDKcimTJqITcLmJrtj1
+        Rto6fq99CSoSLqor604CGqkyYqjG26+U5aeogGelWV1oMVW2j1cAn/fnX2Mp7hP7EkY+XK12YG5
+        vMkvK1oHT3dsIfWDpeiSnB6mfnBb5+jgdszOV4WWUwvt7ipm/pwFXztqqxl9HVfNl97s91n+L7a
+        6jHPHZ7/vtHQWDI5ftITNbL62cZ99MdAXmIOWAbAEn+Dhq6mcsJdJ9b93aUbUkvTAIBjXzRUhBw
+        x2ysiANAldBCU4UfdYazhrtMAICg4AL9zqdFIQLXZGzVVBdxetC4xEGhYCuDdzkDiGq7JzAk04A
+        TyZQ5IvISX4n6lALAOWsbvPwfyTVQB7L+wBiUjcQDxCHYlBQAbuybDrsleriKj9RpklMzhxeYDS
+        YFiVC8HIKiCK2NL6bPemIqMvtSg4WQOd5asT2U4gDIA9NMgjhucIAQAUsJvBYCZGlojmoMkSaiB
+        OpMAgHa/68/G9UAjGL6JAKAJ7IMF3P/G/FXSCviAPjhYw4CAlKF8oWm/sNE4Bwvp3rJNdGlkkQH
+        2utwVnKrrqIP7+Edr1cc/NqBeZXyAF2/Uy5RdD2IFrTe1Fd/HAEp23zZr34KjtcQB9cpMKrPQWv
+        VxzHc6AgqRan9F+I8H1KuCNOi1Vi+1ULcycBN10ZP1u1Wlt423YdGAdIZ6haPqNB6r3cWFye6RS
+        l6ae0eqyDFYCH1gSDCJFfGjVw1IZ6hXZjTBVnSx23WqZmZAOkO9TlRwHLt855IBfoB6NTjCktnb
+        kHD8zq0OqFcISGi4cQRuRJ1Ldm1tYBfQcJBRUDIQEwiEloquhsaCoKBXznAmx/9MoY9pELa5wwA
+        AAABJRU5ErkJggg==
+        EOD;
+    }
+
+    private function jsonToHtml($node)
+    {
+        if ($node['type'] == 'text') {
+            return htmlspecialchars($node['data'], ENT_QUOTES, 'UTF-8');
+        }
+
+        if ($node['type'] == 'tag') {
+            $html = "<{$node['name']}";
+
+            foreach ($node['attribs'] as $key => $value) {
+                $html .= sprintf(
+                    ' %s="%s"',
+                    htmlspecialchars($key, ENT_QUOTES, 'UTF-8'),
+                    htmlspecialchars($value, ENT_QUOTES, 'UTF-8')
+                );
+            }
+
+            $html .= '>';
+
+            foreach ($node['children'] as $child) {
+                $html .= $this->jsonToHtml($child);
+            }
+
+            $html .= "";
+
+            return $html;
+        }
+
+        return '';
+    }
+
+    private function generateContent($deal)
+    {
+        $img_link = $deal->image->source;
+        $content = "
";
+
+        $content .= "
\${$deal->price} \${$deal->streetPrice}
";
+
+        foreach ($deal->buyButtons as $buy) {
+            $content .= "
Buy from url}\">$buy->merchant";
+            if ($buy->promo->effect) {
+                $content .= " {$buy->promo->effect}";
+            }
+            if ($buy->promo->code) {
+                $content .= " (Use promo code {$buy->promo->code})";
+            }
+            $content .= '
';
+        }
+
+        $content .= '
 
';
+        $structuredContent = json_decode($deal->structuredContent, true);
+        foreach ($structuredContent as $node) {
+            $content .= $this->jsonToHtml($node);
+        }
+
+        if ($deal->relatedArticle) {
+            $review = $deal->relatedArticle;
+            $content .= '
 
';
+            $content .= "
Read the review: link}\">{$review->title}
";
+        }
+
+        return $content;
+    }
+}

From 152e96d3d06b626574ecc79781479d46c7d9867d Mon Sep 17 00:00:00 2001
From: Dag 
Date: Mon, 30 Dec 2024 00:19:18 +0100
Subject: [PATCH 256/423] fix: broken if_not_modified_since (#4377)

---
 lib/contents.php | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/lib/contents.php b/lib/contents.php
index 56a3db20..752c0ff2 100644
--- a/lib/contents.php
+++ b/lib/contents.php
@@ -24,6 +24,13 @@ function getContents(
 
     // TODO: consider url validation at this point
 
+    $config = [
+        'useragent'     => Configuration::getConfig('http', 'useragent'),
+        'timeout'       => Configuration::getConfig('http', 'timeout'),
+        'retries'       => Configuration::getConfig('http', 'retries'),
+        'curl_options'  => $curlOptions,
+    ];
+
     $httpHeadersNormalized = [];
     foreach ($httpHeaders as $httpHeader) {
         $parts = explode(':', $httpHeader);
@@ -69,13 +76,7 @@ function getContents(
         'TE' => 'trailers',
     ];
 
-    $config = [
-        'useragent' => Configuration::getConfig('http', 'useragent'),
-        'timeout' => Configuration::getConfig('http', 'timeout'),
-        'retries' => Configuration::getConfig('http', 'retries'),
-        'headers' => array_merge($defaultHttpHeaders, $httpHeadersNormalized),
-        'curl_options' => $curlOptions,
-    ];
+    $config['headers'] = array_merge($defaultHttpHeaders, $httpHeadersNormalized);
 
     $maxFileSize = Configuration::getConfig('http', 'max_filesize');
     if ($maxFileSize) {

From 7d544f1fab0e7c2d7518f0a6e3495dbc4f8e7cb3 Mon Sep 17 00:00:00 2001
From: Dag 
Date: Thu, 2 Jan 2025 16:33:56 +0100
Subject: [PATCH 257/423] feat(reddit): support video (#4380)

---
 bridges/RedditBridge.php | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php
index 03f279d8..2a3824f4 100644
--- a/bridges/RedditBridge.php
+++ b/bridges/RedditBridge.php
@@ -234,11 +234,14 @@ class RedditBridge extends BridgeAbstract
                 } elseif ($data->is_video) {
                     // Video
 
-                    // Higher index -> Higher resolution
-                    end($data->preview->images[0]->resolutions);
-                    $index = key($data->preview->images[0]->resolutions);
-
-                    $item['content'] = $this->createFigureLink($data->url, $data->preview->images[0]->resolutions[$index]->url, 'Video');
+                    if ($data->media->reddit_video) {
+                        $item['content'] = $this->createVideoContent($data->media->reddit_video);
+                    } else {
+                        // Higher index -> Higher resolution
+                        end($data->preview->images[0]->resolutions);
+                        $index = key($data->preview->images[0]->resolutions);
+                        $item['content'] = $this->createFigureLink($data->url, $data->preview->images[0]->resolutions[$index]->url, 'Video');
+                    }
                 } elseif (isset($data->media) && $data->media->type == 'youtube.com') {
                     // Youtube link
                     $item['content'] = $this->createFigureLink($data->url, $data->media->oembed->thumbnail_url, 'YouTube');
@@ -318,6 +321,16 @@ class RedditBridge extends BridgeAbstract
         return sprintf('%s', $href, $text);
     }
 
+    private function createVideoContent(\stdClass $video): string
+    {
+        return <<
+                
+                Your browser does not support the video tag.
+            
+        HTML;
+    }
+
     public function detectParameters($url)
     {
         try {

From b4a63e7040946b0772950d71d9e244f24ee2cf2e Mon Sep 17 00:00:00 2001
From: "Florent V." 
Date: Thu, 2 Jan 2025 16:45:33 +0100
Subject: [PATCH 258/423] [EdfPrices Bridge] add HC/HP, base and EJP (#4369)

* [EdfPrices Bridge] add HC/HP, base and EJP

* [EdfPrices Bridge] lint

* [EdfPrices Bridge] fix missing variable
---
 bridges/EdfPricesBridge.php | 231 ++++++++++++++++++++++++++++++------
 1 file changed, 194 insertions(+), 37 deletions(-)

diff --git a/bridges/EdfPricesBridge.php b/bridges/EdfPricesBridge.php
index 18e6f8e6..06cd30b7 100644
--- a/bridges/EdfPricesBridge.php
+++ b/bridges/EdfPricesBridge.php
@@ -12,8 +12,28 @@ class EdfPricesBridge extends BridgeAbstract
             'contract' => [
                 'name' => 'Choisir un contrat',
                 'type' => 'list',
-                // we can add later HCHP, EJP, base
-                'values' => ['Tempo' => '/energie/edf/tarifs/tempo'],
+                // we can add later more option prices
+                'values' => [
+                    'Base' => '/energie/edf/tarifs/tarif-bleu#base',
+                    'HPHC' => '/energie/edf/tarifs/tarif-bleu#hphc',
+                    'EJP' => '/energie/edf/tarifs/tarif-bleu#ejp',
+                    'Tempo' => '/energie/edf/tarifs/tempo'
+                ],
+            ],
+            'power' => [
+                'name' => 'Choisir une puissance',
+                'type' => 'list',
+                'values' => [
+                    '3 kVA' => 3,
+                    '6 kVA' => 6,
+                    '9 kVA' => 9,
+                    '12 kVA' => 12,
+                    '15 kVA' => 15,
+                    '18 kVA' => 18,
+                    '24 kVA' => 24,
+                    '30 kVA' => 30,
+                    '36 kVA' => 36
+                ]
             ]
         ]
     ];
@@ -24,36 +44,20 @@ class EdfPricesBridge extends BridgeAbstract
      * @param string $contractUri
      * @return void
      */
-    private function tempo(simple_html_dom $html, string $contractUri): void
+    private function tempo(simple_html_dom $html, string $contractUri, int $power): void
     {
-        // current color and next
-        $daysDom = $html->find('#calendrier', 0)->nextSibling()->find('.card--ejp');
-        if ($daysDom && count($daysDom) === 2) {
-            foreach ($daysDom as $dayDom) {
-                $day = trim($dayDom->find('.card__title', 0)->innertext) . '/' . (new \DateTime('now'))->format(('Y'));
-                $dayColor = $dayDom->find('.card-ejp__icon span', 0)->innertext;
-
-                $text = $day . ' - ' . $dayColor;
-                $item['uri'] = self::URI . $contractUri;
-                $item['title'] = $text;
-                $item['author'] = self::MAINTAINER;
-                $item['content'] = $text;
-                $item['uid'] = hash('sha256', $item['title']);
-
-                $this->items[] = $item;
-            }
-        }
-
         // colors
         $ulDom = $html->find('#tarif-de-l-offre-tempo-edf-template-date-now-y', 0)->nextSibling()->nextSibling()->nextSibling();
         $elementsDom = $ulDom->find('li');
         if ($elementsDom && count($elementsDom) === 3) {
+            // price per kWh is same for all powers
             foreach ($elementsDom as $elementDom) {
                 $item = [];
 
                 $matches = [];
                 preg_match_all('/Jour (.*) : Heures (.*) : (.*) € \/ Heures (.*) : (.*) €/um', $elementDom->innertext, $matches, PREG_SET_ORDER, 0);
 
+                // for tempo contract we have 2x3 colors
                 if ($matches && count($matches[0]) === 6) {
                     for ($i = 0; $i < 2; $i++) {
                         $text = 'Jour ' . $matches[0][1] . ' - Heures ' . $matches[0][2 + 2 * $i] . ' : ' . $matches[0][3 + 2 * $i] . '€';
@@ -69,26 +73,166 @@ class EdfPricesBridge extends BridgeAbstract
             }
         }
 
-        // powers
-        $ulPowerContract = $ulDom->nextSibling()->nextSibling();
-        $elementsPowerContractDom = $ulPowerContract->find('li');
-        if ($elementsPowerContractDom && count($elementsPowerContractDom) === 4) {
-            foreach ($elementsPowerContractDom as $elementPowerContractDom) {
+        // add subscription power info
+        $tablePrices = $ulDom->nextSibling()->nextSibling()->nextSibling()->find('.table--responsive', 0);
+        $this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 7);
+    }
+
+    /**
+     * @param simple_html_dom $html
+     * @param string $contractUri
+     * @return void
+     */
+    private function base(simple_html_dom $html, string $contractUri, int $power): void
+    {
+        $tablePrices = $html
+                            ->find('#grille-tarifaire-et-prix-du-kwh-du-tarif-reglemente-edf-en-option-base', 0)
+                            ->nextSibling()
+                            ->nextSibling()
+                            ->nextSibling();
+
+        $prices = $tablePrices->find('.table--stripped tbody tr');
+        // last element is useless because part of another table
+        array_pop($prices);
+
+        // price per kWh is same for all powers
+        if ($prices && count($prices) === 9) {
+            $item = [];
+
+            $text = 'Base : ' . $prices[0]->children(2);
+            $item['uri'] = self::URI . $contractUri;
+            $item['title'] = $text;
+            $item['author'] = self::MAINTAINER;
+            $item['content'] = $text;
+            $item['uid'] = hash('sha256', $item['title']);
+
+            $this->items[] = $item;
+        }
+
+        $this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 9);
+    }
+
+    /**
+     * @param simple_html_dom $html
+     * @param string $contractUri
+     * @return void
+     */
+    private function hphc(simple_html_dom $html, string $contractUri, int $power): void
+    {
+        $tablePrices = $html
+                            ->find('#grille-tarifaire-et-prix-du-kwh-du-tarif-reglemente-edf-en-option-heures-pleines-heures-creuses', 0)
+                            ->nextSibling()
+                            ->nextSibling()
+                            ->nextSibling();
+
+        $prices = $tablePrices->find('.table--stripped tbody tr');
+        // last element is useless because part of another table
+        array_pop($prices);
+
+        // price per kWh is same for all powers
+        if ($prices && count($prices) === 8) {
+            $values = ['HC', 'HP'];
+            foreach ($values as $key => $value) {
+                $i++;
                 $item = [];
 
-                $matches = [];
-                preg_match_all('/(.*) kVA : (.*) €/um', $elementPowerContractDom->innertext, $matches, PREG_SET_ORDER, 0);
+                $text = $values[$key] . ' : ' . $prices[0]->children($key + 2);
+                $item['uri'] = self::URI . $contractUri;
+                $item['title'] = $text;
+                $item['author'] = self::MAINTAINER;
+                $item['content'] = $text;
+                $item['uid'] = hash('sha256', $item['title']);
 
-                if ($matches && count($matches[0]) === 3) {
-                    $text = $matches[0][1] . ' kVA : ' . $matches[0][2] . '€';
-                    $item['uri'] = self::URI . $contractUri;
-                    $item['title'] = $text;
-                    $item['author'] = self::MAINTAINER;
-                    $item['content'] = $text;
-                    $item['uid'] = hash('sha256', $item['title']);
+                $this->items[] = $item;
+            }
+        }
 
-                    $this->items[] = $item;
+        $this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 8);
+    }
+
+    /**
+     * @param simple_html_dom $html
+     * @param string $contractUri
+     * @return void
+     */
+    private function ejp(simple_html_dom $html, string $contractUri, int $power): void
+    {
+        $tablePrices = $html
+                            ->find('#grille-tarifaire-et-prix-du-kwh-du-tarif-reglemente-edf-en-option-ejp', 0)
+                            ->nextSibling()
+                            ->nextSibling()
+                            ->nextSibling();
+
+        $prices = $tablePrices->find('.table--stripped tbody tr');
+        // last element is useless because part of another table
+        array_pop($prices);
+
+        // price per kWh is same for all powers
+        if ($prices && count($prices) === 5) {
+            $values = ['Non EJP', 'EJP'];
+            foreach ($values as $key => $value) {
+                $i++;
+                $item = [];
+
+                $text = $values[$key] . ' : ' . $prices[0]->children($key + 2);
+                $item['uri'] = self::URI . $contractUri;
+                $item['title'] = $text;
+                $item['author'] = self::MAINTAINER;
+                $item['content'] = $text;
+                $item['uid'] = hash('sha256', $item['title']);
+
+                $this->items[] = $item;
+            }
+        }
+
+        $this->addSubscriptionPowerInfo($tablePrices, $contractUri, $power, 5);
+    }
+
+    private function addSubscriptionPowerInfo(simple_html_dom_node $tablePrices, string $contractUri, int $power, int $numberOfPrices): void
+    {
+        $prices = $tablePrices->find('.table--stripped tbody tr');
+        // last element is useless because part of another table
+        array_pop($prices);
+
+        // 7 contracts for tempo: 6, 9, 12, 15, 18, 30 and 36 kVA
+        // 9 contracts for base: 3, 6, 9, 12, 15, 18, 24, 30 and 36 kVA
+        // 7 contracts for HPHC: 6, 9, 12, 15, 18, 24, 30 and 36 kVA
+        // 5 contracts for EJP: 9, 12, 15, 18 and 36 kVA
+        if ($prices && count($prices) === $numberOfPrices) {
+            $powerFound = false;
+            foreach ($prices as $price) {
+                $powerText = $price->firstChild()->firstChild()->innertext;
+                $powerValue = (int)substr($powerText, 0, strpos($powerText, ' kVA'));
+
+                if ($powerValue !== $power) {
+                    continue;
                 }
+
+                $item = [];
+
+                $text = $powerText . ' : ' . $price->children(1) . '/an';
+                $item['uri'] = self::URI . $contractUri;
+                $item['title'] = $text;
+                $item['author'] = self::MAINTAINER;
+                $item['content'] = $text;
+                $item['uid'] = hash('sha256', $item['title']);
+
+                $this->items[] = $item;
+                $powerFound = true;
+                break;
+            }
+
+            if (!$powerFound) {
+                $item = [];
+
+                $text = 'Pas de tarif abonnement pour cette puissance et ce contrat';
+                $item['uri'] = self::URI . $contractUri;
+                $item['title'] = $text;
+                $item['author'] = self::MAINTAINER;
+                $item['content'] = $text;
+                $item['uid'] = hash('sha256', $item['title']);
+
+                $this->items[] = $item;
             }
         }
     }
@@ -97,10 +241,23 @@ class EdfPricesBridge extends BridgeAbstract
     {
         $contract = $this->getKey('contract');
         $contractUri = $this->getInput('contract');
+        $power = $this->getInput('power');
         $html = getSimpleHTMLDOM(self::URI . $contractUri);
 
         if ($contract === 'Tempo') {
-            $this->tempo($html, $contractUri);
+            $this->tempo($html, $contractUri, $power);
+        }
+
+        if ($contract === 'Base') {
+            $this->base($html, $contractUri, $power);
+        }
+
+        if ($contract === 'HPHC') {
+            $this->hphc($html, $contractUri, $power);
+        }
+
+        if ($contract === 'EJP') {
+            $this->ejp($html, $contractUri, $power);
         }
     }
 }

From 4b4d622333fa810cb271b6f3972234bea0a9db51 Mon Sep 17 00:00:00 2001
From: "Quentin B." 
Date: Thu, 2 Jan 2025 17:14:10 +0100
Subject: [PATCH 259/423] [CentreFranceBridge] Update parser to handle latest
 website layout changes (#4372)

---
 bridges/CentreFranceBridge.php | 69 ++++++++++++++--------------------
 1 file changed, 28 insertions(+), 41 deletions(-)

diff --git a/bridges/CentreFranceBridge.php b/bridges/CentreFranceBridge.php
index a6dea227..1abfae98 100644
--- a/bridges/CentreFranceBridge.php
+++ b/bridges/CentreFranceBridge.php
@@ -48,6 +48,11 @@ class CentreFranceBridge extends BridgeAbstract
         ]
     ];
 
+    private static array $monthNumberByFrenchName = [
+        'janvier' => 1, 'février' => 2, 'mars' => 3, 'avril' => 4, 'mai' => 5, 'juin' => 6, 'juillet' => 7,
+        'août' => 8, 'septembre' => 9, 'octobre' => 10, 'novembre' => 11, 'décembre' => 12
+    ];
+
     public function collectData()
     {
         $value = $this->getInput('limit');
@@ -130,14 +135,22 @@ class CentreFranceBridge extends BridgeAbstract
             'enclosures' => [],
         ];
 
-        $articleInformations = $html->find('.c-article-informations p');
+        $articleInformations = $html->find('#content hgroup > div.typo-p3 > *');
         if (is_array($articleInformations) && $articleInformations !== []) {
-            $authorPosition = 1;
+            $publicationDateIndex = 0;
+
+            // Article author
+            $probableAuthorName = strip_tags($articleInformations[0]->innertext);
+            if (str_starts_with($probableAuthorName, 'Par ')) {
+                $publicationDateIndex = 1;
+                $item['author'] = substr($probableAuthorName, 4);
+            }
 
             // Article publication date
-            if (preg_match('/(\d{2})\/(\d{2})\/(\d{4})( à (\d{2})h(\d{2}))?/', $articleInformations[0]->innertext, $articleDateParts) > 0) {
+            preg_match('/Publié le (\d{2}) (.+) (\d{4})( à (\d{2})h(\d{2}))?/', strip_tags($articleInformations[$publicationDateIndex]->innertext), $articleDateParts);
+            if ($articleDateParts !== [] && array_key_exists($articleDateParts[2], self::$monthNumberByFrenchName)) {
                 $articleDate = new \DateTime('midnight');
-                $articleDate->setDate($articleDateParts[3], $articleDateParts[2], $articleDateParts[1]);
+                $articleDate->setDate($articleDateParts[3], self::$monthNumberByFrenchName[$articleDateParts[2]], $articleDateParts[1]);
 
                 if (count($articleDateParts) === 7) {
                     $articleDate->setTime($articleDateParts[5], $articleDateParts[6]);
@@ -145,57 +158,31 @@ class CentreFranceBridge extends BridgeAbstract
 
                 $item['timestamp'] = $articleDate->getTimestamp();
             }
-
-            // Article update date
-            if (count($articleInformations) >= 2 && preg_match('/(\d{2})\/(\d{2})\/(\d{4})( à (\d{2})h(\d{2}))?/', $articleInformations[1]->innertext, $articleDateParts) > 0) {
-                $authorPosition = 2;
-
-                $articleDate = new \DateTime('midnight');
-                $articleDate->setDate($articleDateParts[3], $articleDateParts[2], $articleDateParts[1]);
-
-                if (count($articleDateParts) === 7) {
-                    $articleDate->setTime($articleDateParts[5], $articleDateParts[6]);
-                }
-
-                $item['timestamp'] = $articleDate->getTimestamp();
-            }
-
-            if (count($articleInformations) === ($authorPosition + 1)) {
-                $item['author'] = $articleInformations[$authorPosition]->innertext;
-            }
         }
 
-        $articleContent = $html->find('.b-article .contenu > *');
-        if (is_array($articleContent)) {
-            $item['content'] = '';
-
-            foreach ($articleContent as $contentPart) {
-                if (in_array($contentPart->getAttribute('id'), ['cf-audio-player', 'poool-widget'], true)) {
-                    continue;
+        $articleContent = $html->find('#content>div.flex+div.grid section>.z-10')[0] ?? null;
+        if ($articleContent instanceof \simple_html_dom_node) {
+            $articleHiddenParts = $articleContent->find('.ad-slot, #cf-digiteka-player');
+            if (is_array($articleHiddenParts)) {
+                foreach ($articleHiddenParts as $articleHiddenPart) {
+                    $articleContent->removeChild($articleHiddenPart);
                 }
-
-                $articleHiddenParts = $contentPart->find('.bloc, .p402_hide');
-                if (is_array($articleHiddenParts)) {
-                    foreach ($articleHiddenParts as $articleHiddenPart) {
-                        $contentPart->removeChild($articleHiddenPart);
-                    }
-                }
-
-                $item['content'] .= $contentPart->innertext;
             }
+
+            $item['content'] = $articleContent->innertext;
         }
 
-        $articleIllustration  = $html->find('.photo-wrapper .photo-box img');
+        $articleIllustration  = $html->find('#content>div.flex+div.grid section>figure>img');
         if (is_array($articleIllustration) && count($articleIllustration) === 1) {
             $item['enclosures'][] = $articleIllustration[0]->getAttribute('src');
         }
 
-        $articleAudio = $html->find('#cf-audio-player-container audio');
+        $articleAudio = $html->find('audio[src^="https://api.octopus.saooti.com/"]');
         if (is_array($articleAudio) && count($articleAudio) === 1) {
             $item['enclosures'][] = $articleAudio[0]->getAttribute('src');
         }
 
-        $articleTags = $html->find('.b-article > ul.c-tags > li > a.t-simple');
+        $articleTags = $html->find('#content>div.flex+div.grid section>.bg-gray-light>a.border-gray-dark');
         if (is_array($articleTags)) {
             $item['categories'] = array_map(static fn ($articleTag) => $articleTag->innertext, $articleTags);
         }

From 974f00cd6a3fb4f4ccda08069408edb9c7e1d6f4 Mon Sep 17 00:00:00 2001
From: Sebastian Wolf <117176763+swofl@users.noreply.github.com>
Date: Thu, 2 Jan 2025 17:17:54 +0100
Subject: [PATCH 260/423] [MixologyBridge] adapt to latest site changes (#4368)

* [MixologyBridge] adapt to latest site changes

* [MixologyBridge] fix category selector
---
 bridges/MixologyBridge.php | 66 ++++++++++++++++++++++++++------------
 1 file changed, 45 insertions(+), 21 deletions(-)

diff --git a/bridges/MixologyBridge.php b/bridges/MixologyBridge.php
index 1246b4db..e954d0d6 100644
--- a/bridges/MixologyBridge.php
+++ b/bridges/MixologyBridge.php
@@ -1,49 +1,73 @@
  self::LIMIT,
-    ] ];
 
     public function collectData()
     {
-        $feed_url = self::URI . '/feed';
-        $limit = $this->getInput('limit') ?? 10;
-        $this->collectExpandableDatas($feed_url, $limit);
+        $html = getSimpleHTMLDOM(self::URI);
+
+        $teasers = [];
+        $teaserElements = [];
+
+        $teaserElements[] = $html->find('.aufmacher .views-view-responsive-grid__item-inner', 0);
+        foreach ($html->find('.block-views-blockmixology-frontpage-block-2 .views-col') as $teaser) {
+            $teaserElements[] = $teaser;
+        }
+
+        foreach ($teaserElements as $teaser) {
+            $teasers[] = $this->parseTeaser($teaser);
+        }
+
+        foreach ($teasers as $article) {
+            $this->items[] = $this->parseItem($article);
+        }
+    }
+
+    protected function parseTeaser($teaser)
+    {
+        $result = [];
+
+        $title = $teaser->find('.views-field-title a', 0);
+        $result['title'] = $title->plaintext;
+        $result['uri'] = self::URI . $title->href;
+        $result['enclosures'] = [];
+        $result['enclosures'][] = self::URI . $teaser->find('img', 0)->src;
+        $result['uid'] = hash('sha256', $result['title']);
+
+        $categories = $teaser->find('.views-field-field-kategorie', 0);
+        if ($categories) {
+            $result['categories'] = [];
+            foreach ($categories->find('a') as $category) {
+                $result['categories'][] = $category->innertext;
+            }
+        }
+
+        return $result;
     }
 
     protected function parseItem(array $item)
     {
         $article = getSimpleHTMLDOMCached($item['uri']);
 
+        $item['author'] = $article->find('.beitrag-author a', 0)->plaintext;
+        $item['timestamp'] = strtotime($article->find('.beitrag-date time', 0)->datetime);
+
         $content = '';
 
-        $headerImage = $article->find('div.edgtf-full-width img.wp-post-image', 0);
+        $content .= '';
 
-        if (is_object($headerImage)) {
-            $item['enclosures'] = [];
-            $item['enclosures'][] = $headerImage->src;
-            $content .= '';
-        }
-
-        foreach ($article->find('article .wpb_content_element > .wpb_wrapper') as $element) {
+        foreach ($article->find('article .wpb_content_element>.wpb_wrapper, article .field--type-text-with-summary>.wp-block-columns>.wp-block-column') as $element) {
             $content .= $element->innertext;
         }
 
         $item['content'] = $content;
 
-        $item['categories'] = [];
-
-        foreach ($article->find('.edgtf-tags > a') as $tag) {
-            $item['categories'][] = $tag->plaintext;
-        }
-
         return $item;
     }
 }

From 2a44a006b222c51a421b2f27f0539263d5bf3bcf Mon Sep 17 00:00:00 2001
From: mruac 
Date: Fri, 3 Jan 2025 03:09:07 +1030
Subject: [PATCH 261/423] Update BlueskyBridge.php (#4367)

* Update BlueskyBridge.php

* Used human readable terms
* Include quote and reply post
* Added video support
* Replaced Youtube embed with thumbnail preview
* Added link embed preview
* Included visible alt text to images

* appease the lint

* remove unused test code

* fix unset displayName

* appease the lint
---
 bridges/BlueskyBridge.php | 561 +++++++++++++++++++++++++++++++-------
 1 file changed, 459 insertions(+), 102 deletions(-)

diff --git a/bridges/BlueskyBridge.php b/bridges/BlueskyBridge.php
index 8dab82f4..89e5f3bc 100644
--- a/bridges/BlueskyBridge.php
+++ b/bridges/BlueskyBridge.php
@@ -2,10 +2,12 @@
 
 class BlueskyBridge extends BridgeAbstract
 {
-    const NAME = 'Bluesky';
+    //Initial PR by [RSSBridge contributors](https://github.com/RSS-Bridge/rss-bridge/issues/4058).
+    //Modified from [©DIYgod and contributors at RSSHub](https://github.com/DIYgod/RSSHub/tree/master/lib/routes/bsky), MIT License';
+    const NAME = 'Bluesky Bridge';
     const URI = 'https://bsky.app';
     const DESCRIPTION = 'Fetches posts from Bluesky';
-    const MAINTAINER = 'Code modified from rsshub (TonyRL https://github.com/TonyRL) and expanded';
+    const MAINTAINER = 'mruac';
     const PARAMETERS = [
         [
             'data_source' => [
@@ -17,24 +19,39 @@ class BlueskyBridge extends BridgeAbstract
                 ],
                 'title' => 'Select the type of data source to fetch from Bluesky.'
             ],
-            'handle' => [
-                'name' => 'User Handle',
+            'user_id' => [
+                'name' => 'User Handle or DID',
                 'type' => 'text',
                 'required' => true,
-                'exampleValue' => 'jackdodo.bsky.social',
-                'title' => 'Handle found in URL'
+                'exampleValue' => 'did:plc:z72i7hdynmk6r22z27h6tvur',
+                'title' => 'ATProto / Bsky.app handle or DID'
             ],
-            'filter' => [
-                'name' => 'Filter',
+            'feed_filter' => [
+                'name' => 'Feed type',
                 'type' => 'list',
                 'defaultValue' => 'posts_and_author_threads',
                 'values' => [
-                    'posts_and_author_threads' => 'posts_and_author_threads',
-                    'posts_with_replies' => 'posts_with_replies',
-                    'posts_no_replies' => 'posts_no_replies',
-                    'posts_with_media' => 'posts_with_media',
-                ],
-                'title' => 'Combinations of post/repost types to include in response.'
+                    'Posts feed' => 'posts_and_author_threads',
+                    'All posts and replies' => 'posts_with_replies',
+                    'Root posts only' => 'posts_no_replies',
+                    'Media only' => 'posts_with_media',
+                ]
+            ],
+
+            'include_reposts' => [
+                'name' => 'Include Reposts?',
+                'type' => 'checkbox',
+                'defaultValue' => 'checked'
+            ],
+
+            'include_reply_context' => [
+                'name' => 'Include Reply context?',
+                'type' => 'checkbox'
+            ],
+
+            'verbose_title' => [
+                'name' => 'Use verbose feed item titles?',
+                'type' => 'checkbox'
             ]
         ]
     ];
@@ -44,7 +61,11 @@ class BlueskyBridge extends BridgeAbstract
     public function getName()
     {
         if (isset($this->profile)) {
-            return sprintf('%s (@%s) - Bluesky', $this->profile['displayName'], $this->profile['handle']);
+            if ($this->profile['handle'] === 'handle.invalid') {
+                return sprintf('Bluesky - %s', $this->profile['displayName']);
+            } else {
+                return sprintf('Bluesky - %s (@%s)', $this->profile['displayName'], $this->profile['handle']);
+            }
         }
         return parent::getName();
     }
@@ -52,7 +73,11 @@ class BlueskyBridge extends BridgeAbstract
     public function getURI()
     {
         if (isset($this->profile)) {
-            return self::URI . '/profile/' . $this->profile['handle'];
+            if ($this->profile['handle'] === 'handle.invalid') {
+                return self::URI . '/profile/' . $this->profile['did'];
+            } else {
+                return self::URI . '/profile/' . $this->profile['handle'];
+            }
         }
         return parent::getURI();
     }
@@ -77,118 +102,365 @@ class BlueskyBridge extends BridgeAbstract
     {
         $description = '';
         $externalUri = $external['uri'];
-        $externalTitle = htmlspecialchars($external['title'], ENT_QUOTES, 'UTF-8');
-        $externalDescription = htmlspecialchars($external['description'], ENT_QUOTES, 'UTF-8');
+        $externalTitle = e($external['title']);
+        $externalDescription = e($external['description']);
         $thumb = $external['thumb'] ?? null;
 
-        if (preg_match('/youtube\.com\/watch\?v=([^\&\?\/]+)/', $externalUri, $id) || preg_match('/youtu\.be\/([^\&\?\/]+)/', $externalUri, $id)) {
-            $videoId = $id[1];
-            $description .= "
External Link: $externalTitle
";
-            $description .= "";
+        if (preg_match('/http(|s):\/\/media\.tenor\.com/', $externalUri)) {
+            //tenor gif embed
+            $tenorInterstitial = str_replace('media.tenor.com', 'media1.tenor.com/m', $externalUri);
+            $description .= "
$externalTitle
";
         } else {
-            $description .= "
External Link: $externalTitle
";
-            $description .= "
$externalDescription
";
-
-            if ($thumb) {
-                $thumbUrl = 'https://cdn.bsky.app/img/feed_thumbnail/plain/' . $did . '/' . $thumb['ref']['$link'] . '@jpeg';
-                $description .= "
\"External
";
-            }
+            //link embed preview
+            $host = parse_url($externalUri)['host'];
+            $thumbDesc = $thumb ? ('') : '';
+            $externalDescription = strlen($externalDescription) > 0 ? "
($host) $externalDescription
" : '';
+            $description .= '
' . $externalTitle . '';
+            $description .= '
' . $thumbDesc . $externalDescription . '
';
         }
         return $description;
     }
 
-    private function textToDescription($text)
+    private function textToDescription($record)
     {
-        $text = nl2br(htmlspecialchars($text, ENT_QUOTES, 'UTF-8'));
-        $text = preg_replace('/(https?:\/\/[^\s]+)/i', '$1', $text);
-
+        if (isset($record['value'])) {
+            $record = $record['value'];
+        }
+        $text = $record['text'];
+        $text_copy = $text;
+        $text = nl2br(e($text));
+        if (isset($record['facets'])) {
+            $facets = $record['facets'];
+            foreach ($facets as $facet) {
+                if ($facet['features'][0]['$type'] === 'app.bsky.richtext.facet#link') {
+                    $substring = substr($text_copy, $facet['index']['byteStart'], $facet['index']['byteEnd'] - $facet['index']['byteStart']);
+                    $text = str_replace($substring, '' . $substring . '', $text);
+                }
+            }
+        }
         return $text;
     }
 
     public function collectData()
     {
-        $handle = $this->getInput('handle');
-        $filter = $this->getInput('filter') ?: 'posts_and_author_threads';
+        $user_id = $this->getInput('user_id');
+        $handle_match = preg_match('/(?:[a-zA-Z]*\.)+([a-zA-Z](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)/', $user_id, $handle_res); //gets the TLD in $handle_match[1]
+        $did_match = preg_match('/did:plc:[a-z2-7]{24}/', $user_id); //https://github.com/did-method-plc/did-method-plc#identifier-syntax
+        $exclude = ['alt', 'arpa', 'example', 'internal', 'invalid', 'local', 'localhost', 'onion']; //https://en.wikipedia.org/wiki/Top-level_domain#Reserved_domains
+        if ($handle_match == true && array_search($handle_res[1], $exclude) == false) {
+            //valid bsky handle
+            $did = $this->resolveHandle($user_id);
+        } elseif ($did_match == true) {
+            //valid DID
+            $did = $user_id;
+        } else {
+            returnClientError('Invalid ATproto handle or DID provided.');
+        }
+
+        $filter = $this->getInput('feed_filter') ?: 'posts_and_author_threads';
+        $replyContext = $this->getInput('include_reply_context');
 
-        $did = $this->resolveHandle($handle);
         $this->profile = $this->getProfile($did);
         $authorFeed = $this->getAuthorFeed($did, $filter);
 
         foreach ($authorFeed['feed'] as $post) {
+            $postRecord = $post['post']['record'];
+
             $item = [];
-            $item['uri'] = self::URI . '/profile/' . $post['post']['author']['handle'] . '/post/' . explode('app.bsky.feed.post/', $post['post']['uri'])[1];
-            $item['title'] = strtok($post['post']['record']['text'], "\n");
-            $item['timestamp'] = strtotime($post['post']['record']['createdAt']);
-            $item['author'] = $this->profile['displayName'];
+            $item['uri'] = self::URI . '/profile/' . $this->fallbackAuthor($post['post']['author'], 'url') . '/post/' . explode('app.bsky.feed.post/', $post['post']['uri'])[1];
+            $item['title'] = $this->getInput('verbose_title') ? $this->generateVerboseTitle($post) : strtok($postRecord['text'], "\n");
+            $item['timestamp'] = strtotime($postRecord['createdAt']);
+            $item['author'] = $this->fallbackAuthor($post['post']['author'], 'display');
 
-            $description = $this->textToDescription($post['post']['record']['text']);
+            $postAuthorDID = $post['post']['author']['did'];
+            $postAuthorHandle = $post['post']['author']['handle'] !== 'handle.invalid' ? '@' . $post['post']['author']['handle'] . ' ' : '';
+            $postDisplayName = $post['post']['author']['displayName'] ?? '';
+            $postDisplayName = e($postDisplayName);
+            $postUri = $item['uri'];
 
-            // Retrieve DID for constructing image URLs
-            $authorDid = $post['post']['author']['did'];
-
-            if (isset($post['post']['record']['embed']['$type']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.external') {
-                $description .= $this->parseExternal($post['post']['record']['embed']['external'], $authorDid);
+            if (Debug::isEnabled()) {
+                $url = explode('/', $post['post']['uri']);
+                error_log('https://bsky.app/profile/' . $url[2] . '/post/' . $url[4]);
             }
 
-            if (isset($post['post']['record']['embed']['$type']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.video') {
-                $thumbnail = $post['post']['embed']['thumbnail'] ?? null;
-                if ($thumbnail) {
-                                    $itemUri = self::URI . '/profile/' . $post['post']['author']['handle'] . '/post/' . explode('app.bsky.feed.post/', $post['post']['uri'])[1];
-                    $description .= "
\"Video
";
+            $description = '';
+            $description .= '
';
+            //post
+            $description .= $this->getPostDescription(
+                $postDisplayName,
+                $postAuthorHandle,
+                $postUri,
+                $postRecord,
+                'post'
+            );
+
+            if (isset($postRecord['embed']['$type'])) {
+                //post link embed
+                if ($postRecord['embed']['$type'] === 'app.bsky.embed.external') {
+                    $description .= $this->parseExternal($postRecord['embed']['external'], $postAuthorDID);
+                } elseif (
+                    $postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                    $postRecord['embed']['media']['$type'] === 'app.bsky.embed.external'
+                ) {
+                    $description .= $this->parseExternal($postRecord['embed']['media']['external'], $postAuthorDID);
+                }
+
+                //post images
+                if (
+                    $postRecord['embed']['$type'] === 'app.bsky.embed.images' ||
+                    (
+                        $postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                        $postRecord['embed']['media']['$type'] === 'app.bsky.embed.images'
+                    )
+                ) {
+                    $images = $post['post']['embed']['images'] ?? $post['post']['embed']['media']['images'];
+                    foreach ($images as $image) {
+                        $description .= $this->getPostImageDescription($image);
+                    }
+                }
+
+                //post video
+                if (
+                    $postRecord['embed']['$type'] === 'app.bsky.embed.video' ||
+                    (
+                        $postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                        $postRecord['embed']['media']['$type'] === 'app.bsky.embed.video'
+                    )
+                ) {
+                    $description .= $this->getPostVideoDescription(
+                        $postRecord['embed']['video'] ?? $postRecord['embed']['media']['video'],
+                        $postAuthorDID
+                    );
                 }
             }
+            $description .= '
';
 
-            if (isset($post['post']['record']['embed']['$type']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.recordWithMedia#view') {
-                $thumbnail = $post['post']['embed']['media']['thumbnail'] ?? null;
-                $playlist = $post['post']['embed']['media']['playlist'] ?? null;
-                if ($thumbnail) {
-                    $description .= "
';
-                }
-            }
+            //quote post
+            if (
+                isset($postRecord['embed']) &&
+                (
+                    $postRecord['embed']['$type'] === 'app.bsky.embed.record' ||
+                    $postRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia'
+                ) &&
+                isset($post['post']['embed']['record'])
+            ) {
+                $description .= '
';
+                $quotedRecord = $post['post']['embed']['record']['record'] ?? $post['post']['embed']['record'];
 
-            if (!empty($post['post']['record']['embed']['images'])) {
-                foreach ($post['post']['record']['embed']['images'] as $image) {
-                    $linkRef = $image['image']['ref']['$link'];
-                    $thumbnailUrl = $this->resolveThumbnailUrl($authorDid, $linkRef);
-                    $fullsizeUrl = $this->resolveFullsizeUrl($authorDid, $linkRef);
-                    $description .= "

\"Image\"";
-                }
-            }
+                if (isset($quotedRecord['notFound']) && $quotedRecord['notFound']) { //deleted post
+                    $description .= 'Quoted post deleted.';
+                } elseif (isset($quotedRecord['detached']) && $quotedRecord['detached']) { //detached quote
+                    $uri_explode = explode('/', $quotedRecord['uri']);
+                    $uri_reconstructed = self::URI . '/profile/' . $uri_explode[2] . '/post/' . $uri_explode[4];
+                    $description .= 'Quoted post detached.';
+                } elseif (isset($quotedRecord['blocked']) && $quotedRecord['blocked']) { //blocked by quote author
+                    $description .= 'Author of quoted post has blocked OP.';
+                } else {
+                    $quotedAuthorDid = $quotedRecord['author']['did'];
+                    $quotedDisplayName = $quotedRecord['author']['displayName'] ?? '';
+                    $quotedDisplayName = e($quotedDisplayName);
+                    $quotedAuthorHandle = $quotedRecord['author']['handle'] !== 'handle.invalid' ? '@' . $quotedRecord['author']['handle'] . '' : '';
 
-            // Enhanced handling for quote posts with images
-            if (isset($post['post']['record']['embed']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.record') {
-                $quotedRecord = $post['post']['record']['embed']['record'];
-                $quotedAuthor = $post['post']['embed']['record']['author']['handle'] ?? null;
-                $quotedDisplayName = $post['post']['embed']['record']['author']['displayName'] ?? null;
-                $quotedText = $post['post']['embed']['record']['value']['text'] ?? null;
-
-                if ($quotedAuthor && isset($quotedRecord['uri'])) {
                     $parts = explode('/', $quotedRecord['uri']);
                     $quotedPostId = end($parts);
-                    $quotedPostUri = self::URI . '/profile/' . $quotedAuthor . '/post/' . $quotedPostId;
-                }
+                    $quotedPostUri = self::URI . '/profile/' . $this->fallbackAuthor($quotedRecord['author'], 'url') . '/post/' . $quotedPostId;
 
-                if ($quotedText) {
-                    $description .= '
Quote from ' . htmlspecialchars($quotedDisplayName) . ' (@ ' . htmlspecialchars($quotedAuthor) . '):
';
-                    $description .= $this->textToDescription($quotedText);
-                    if (isset($quotedPostUri)) {
-                        $description .= "
View original quote post
";
+                    //quoted post - post
+                    $description .= $this->getPostDescription(
+                        $quotedDisplayName,
+                        $quotedAuthorHandle,
+                        $quotedPostUri,
+                        $quotedRecord,
+                        'quote'
+                    );
+
+                    if (isset($quotedRecord['value']['embed']['$type'])) {
+                        //quoted post - post link embed
+                        if ($quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.external') {
+                            $description .= $this->parseExternal($quotedRecord['value']['embed']['external'], $quotedAuthorDid);
+                        }
+
+                        //quoted post - post video
+                        if (
+                            $quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.video' ||
+                            (
+                                $quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                                $quotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.video'
+                            )
+                        ) {
+                            $description .= $this->getPostVideoDescription(
+                                $quotedRecord['value']['embed']['video'] ?? $quotedRecord['value']['embed']['media']['video'],
+                                $quotedAuthorDid
+                            );
+                        }
+
+                        //quoted post - post images
+                        if (
+                            $quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.images' ||
+                            (
+                                $quotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                                $quotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.images'
+                            )
+                        ) {
+                            foreach ($quotedRecord['embeds'] as $embed) {
+                                if (
+                                    $embed['$type'] === 'app.bsky.embed.images#view' ||
+                                    ($embed['$type'] === 'app.bsky.embed.recordWithMedia#view' && $embed['media']['$type'] === 'app.bsky.embed.images#view')
+                                ) {
+                                    $images = $embed['images'] ?? $embed['media']['images'];
+                                    foreach ($images as $image) {
+                                        $description .= $this->getPostImageDescription($image);
+                                    }
+                                }
+                            }
+                        }
                     }
                 }
+                $description .= '
';
             }
 
-            if (isset($post['post']['embed']['record']['value']['embed']['images'])) {
-                $quotedImages = $post['post']['embed']['record']['value']['embed']['images'];
-                foreach ($quotedImages as $image) {
-                    $linkRef = $image['image']['ref']['$link'] ?? null;
-                    if ($linkRef) {
-                        $quotedAuthorDid = $post['post']['embed']['record']['author']['did'] ?? null;
-                        $thumbnailUrl = $this->resolveThumbnailUrl($quotedAuthorDid, $linkRef);
-                        $fullsizeUrl = $this->resolveFullsizeUrl($quotedAuthorDid, $linkRef);
-                        $description .= "

\"Quoted";
+            //reply
+            if ($replyContext && isset($post['reply']) && !isset($post['reply']['parent']['notFound'])) {
+                $replyPost = $post['reply']['parent'];
+                $replyPostRecord = $replyPost['record'];
+                $description .= '
';
+                $description .= '
';
+
+                $replyPostAuthorDID = $replyPost['author']['did'];
+                $replyPostAuthorHandle = $replyPost['author']['handle'] !== 'handle.invalid' ? '@' . $replyPost['author']['handle'] . ' ' : '';
+                $replyPostDisplayName = $replyPost['author']['displayName'] ?? '';
+                $replyPostDisplayName = e($replyPostDisplayName);
+                $replyPostUri = self::URI . '/profile/' . $this->fallbackAuthor($replyPost['author'], 'url') . '/post/' . explode('app.bsky.feed.post/', $replyPost['uri'])[1];
+
+                // reply post
+                $description .= $this->getPostDescription(
+                    $replyPostDisplayName,
+                    $replyPostAuthorHandle,
+                    $replyPostUri,
+                    $replyPostRecord,
+                    'reply'
+                );
+
+                if (isset($replyPostRecord['embed']['$type'])) {
+                    //post link embed
+                    if ($replyPostRecord['embed']['$type'] === 'app.bsky.embed.external') {
+                        $description .= $this->parseExternal($replyPostRecord['embed']['external'], $replyPostAuthorDID);
+                    } elseif (
+                        $replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                        $replyPostRecord['embed']['media']['$type'] === 'app.bsky.embed.external'
+                    ) {
+                        $description .= $this->parseExternal($replyPostRecord['embed']['media']['external'], $replyPostAuthorDID);
                     }
+
+                    //post images
+                    if (
+                        $replyPostRecord['embed']['$type'] === 'app.bsky.embed.images' ||
+                        (
+                            $replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                            $replyPostRecord['embed']['media']['$type'] === 'app.bsky.embed.images'
+                        )
+                    ) {
+                        $images = $replyPost['embed']['images'] ?? $replyPost['embed']['media']['images'];
+                        foreach ($images as $image) {
+                            $description .= $this->getPostImageDescription($image);
+                        }
+                    }
+
+                    //post video
+                    if (
+                        $replyPostRecord['embed']['$type'] === 'app.bsky.embed.video' ||
+                        (
+                            $replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                            $replyPostRecord['embed']['media']['$type'] === 'app.bsky.embed.video'
+                        )
+                    ) {
+                        $description .= $this->getPostVideoDescription(
+                            $replyPostRecord['embed']['video'] ?? $replyPostRecord['embed']['media']['video'],
+                            $replyPostAuthorDID
+                        );
+                    }
+                }
+                $description .= '
';
+
+                //quote post
+                if (
+                    isset($replyPostRecord['embed']) &&
+                    ($replyPostRecord['embed']['$type'] === 'app.bsky.embed.record' || $replyPostRecord['embed']['$type'] === 'app.bsky.embed.recordWithMedia') &&
+                    isset($replyPost['embed']['record'])
+                ) {
+                    $description .= '
';
+                    $replyQuotedRecord = $replyPost['embed']['record']['record'] ?? $replyPost['embed']['record'];
+
+                    if (isset($replyQuotedRecord['notFound']) && $replyQuotedRecord['notFound']) { //deleted post
+                        $description .= 'Quoted post deleted.';
+                    } elseif (isset($replyQuotedRecord['detached']) && $replyQuotedRecord['detached']) { //detached quote
+                        $uri_explode = explode('/', $replyQuotedRecord['uri']);
+                        $uri_reconstructed = self::URI . '/profile/' . $uri_explode[2] . '/post/' . $uri_explode[4];
+                        $description .= 'Quoted post detached.';
+                    } elseif (isset($replyQuotedRecord['blocked']) && $replyQuotedRecord['blocked']) { //blocked by quote author
+                        $description .= 'Author of quoted post has blocked OP.';
+                    } else {
+                        $quotedAuthorDid = $replyQuotedRecord['author']['did'];
+                        $quotedDisplayName = $replyQuotedRecord['author']['displayName'] ?? '';
+                        $quotedDisplayName = e($quotedDisplayName);
+                        $quotedAuthorHandle = $replyQuotedRecord['author']['handle'] !== 'handle.invalid' ? '@' . $replyQuotedRecord['author']['handle'] . '' : '';
+
+                        $parts = explode('/', $replyQuotedRecord['uri']);
+                        $quotedPostId = end($parts);
+                        $quotedPostUri = self::URI . '/profile/' . $this->fallbackAuthor($replyQuotedRecord['author'], 'url') . '/post/' . $quotedPostId;
+
+                        //quoted post - post
+                        $description .= $this->getPostDescription(
+                            $quotedDisplayName,
+                            $quotedAuthorHandle,
+                            $quotedPostUri,
+                            $replyQuotedRecord,
+                            'quote'
+                        );
+
+                        if (isset($replyQuotedRecord['value']['embed']['$type'])) {
+                            //quoted post - post link embed
+                            if ($replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.external') {
+                                $description .= $this->parseExternal($replyQuotedRecord['value']['embed']['external'], $quotedAuthorDid);
+                            }
+
+                            //quoted post - post video
+                            if (
+                                $replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.video' ||
+                                (
+                                    $replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                                    $replyQuotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.video'
+                                )
+                            ) {
+                                $description .= $this->getPostVideoDescription(
+                                    $replyQuotedRecord['value']['embed']['video'] ?? $replyQuotedRecord['value']['embed']['media']['video'],
+                                    $quotedAuthorDid
+                                );
+                            }
+
+                            //quoted post - post images
+                            if (
+                                $replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.images' ||
+                                (
+                                    $replyQuotedRecord['value']['embed']['$type'] === 'app.bsky.embed.recordWithMedia' &&
+                                    $replyQuotedRecord['value']['embed']['media']['$type'] === 'app.bsky.embed.images'
+                                )
+                            ) {
+                                foreach ($replyQuotedRecord['embeds'] as $embed) {
+                                    if (
+                                        $embed['$type'] === 'app.bsky.embed.images#view' ||
+                                        ($embed['$type'] === 'app.bsky.embed.recordWithMedia#view' && $embed['media']['$type'] === 'app.bsky.embed.images#view')
+                                    ) {
+                                        $images = $embed['images'] ?? $embed['media']['images'];
+                                        foreach ($images as $image) {
+                                            $description .= $this->getPostImageDescription($image);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    $description .= '
';
                 }
             }
 
@@ -197,6 +469,98 @@ class BlueskyBridge extends BridgeAbstract
         }
     }
 
+    private function getPostVideoDescription(array $video, $authorDID)
+    {
+        //https://video.bsky.app/watch/$did/$cid/thumbnail.jpg
+        $videoCID = $video['ref']['$link'];
+        $videoMime = $video['mimeType'];
+        $thumbnail = "poster=\"https://video.bsky.app/watch/$authorDID/$videoCID/thumbnail.jpg\"" ?? '';
+        $videoURL = "https://bsky.social/xrpc/com.atproto.sync.getBlob?did=$authorDID&cid=$videoCID";
+        return "
";
+    }
+
+    private function getPostImageDescription(array $image)
+    {
+        $thumbnailUrl = $image['thumb'];
+        $fullsizeUrl = $image['fullsize'];
+        $alt = strlen($image['alt']) > 0 ? '
' . e($image['alt']) . '
' : '';
+        return "
$alt
";
+    }
+
+    private function getPostDescription(
+        string $postDisplayName,
+        string $postAuthorHandle,
+        string $postUri,
+        array $postRecord,
+        string $type
+    ) {
+        $description = '';
+        if ($type === 'quote') {
+            // Quoted post/reply from bbb @bbb.com:
+            $postType = isset($postRecord['reply']) ? 'reply' : 'post';
+            $description .= "Quoted $postType from $postDisplayName $postAuthorHandle:
";
+        } elseif ($type === 'reply') {
+            // Replying to aaa @aaa.com's post/reply:
+            $postType = isset($postRecord['reply']) ? 'reply' : 'post';
+            $description .= "Replying to $postDisplayName $postAuthorHandle's $postType:
";
+        } else {
+            // aaa @aaa.com posted:
+            $description .= "$postDisplayName $postAuthorHandle posted:
";
+        }
+        $description .= $this->textToDescription($postRecord);
+        return $description;
+    }
+
+    //used if handle verification fails, fallsback to displayName or DID depending on context.
+    private function fallbackAuthor($author, $reason)
+    {
+        if ($author['handle'] === 'handle.invalid') {
+            switch ($reason) {
+                case 'url':
+                    return $author['did'];
+                case 'display':
+                    $displayName = $author['displayName'] ?? '';
+                    return e($displayName);
+            }
+        }
+        return $author['handle'];
+    }
+
+    private function generateVerboseTitle($post)
+    {
+        //use "Post by A, replying to B, quoting C" instead of post contents
+        $title = '';
+        if (isset($post['reason']) && str_contains($post['reason']['$type'], 'reasonRepost')) {
+            $title .= 'Repost by ' . $this->fallbackAuthor($post['reason']['by'], 'display') . ', post by ' . $this->fallbackAuthor($post['post']['author'], 'display');
+        } else {
+            $title .= 'Post by ' . $this->fallbackAuthor($post['post']['author'], 'display');
+        }
+
+        if (isset($post['reply'])) {
+            if (isset($post['reply']['parent']['blocked'])) {
+                $replyAuthor = 'blocked user';
+            } elseif (isset($post['reply']['parent']['notFound'])) {
+                $replyAuthor = 'deleted post';
+            } else {
+                $replyAuthor = $this->fallbackAuthor($post['reply']['parent']['author'], 'display');
+            }
+            $title .= ', replying to ' . $replyAuthor;
+        }
+        if (isset($post['post']['embed']) && isset($post['post']['embed']['record'])) {
+            if (isset($post['post']['embed']['record']['blocked'])) {
+                $quotedAuthor = 'blocked user';
+            } elseif (isset($post['post']['embed']['record']['notFound'])) {
+                $quotedAuthor = 'deleted post';
+            } elseif (isset($post['post']['embed']['record']['detached'])) {
+                $quotedAuthor = 'detached post';
+            } else {
+                $quotedAuthor = $this->fallbackAuthor($post['post']['embed']['record']['record']['author'] ?? $post['post']['embed']['record']['author'], 'display');
+            }
+            $title .= ', quoting ' . $quotedAuthor;
+        }
+        return $title;
+    }
+
     private function resolveHandle($handle)
     {
         $uri = 'https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=' . urlencode($handle);
@@ -214,17 +578,10 @@ class BlueskyBridge extends BridgeAbstract
     private function getAuthorFeed($did, $filter)
     {
         $uri = 'https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed?actor=' . urlencode($did) . '&filter=' . urlencode($filter) . '&limit=30';
+        if (Debug::isEnabled()) {
+            error_log($uri);
+        }
         $response = json_decode(getContents($uri), true);
         return $response;
     }
-
-    private function resolveThumbnailUrl($authorDid, $linkRef)
-    {
-        return 'https://cdn.bsky.app/img/feed_thumbnail/plain/' . $authorDid . '/' . $linkRef . '@jpeg';
-    }
-
-    private function resolveFullsizeUrl($authorDid, $linkRef)
-    {
-        return 'https://cdn.bsky.app/img/feed_fullsize/plain/' . $authorDid . '/' . $linkRef . '@jpeg';
-    }
 }

From 97eebfb562f80de0d6e1c665a2eb49b7841fb837 Mon Sep 17 00:00:00 2001
From: Niehztog 
Date: Thu, 2 Jan 2025 17:44:36 +0100
Subject: [PATCH 262/423] [BlizzardNewsBridge] fix BlizzardNewsBridge (#4379)

* fix BlizzardNewsBridge

* fix linter warnings

* fix linter warnings

* fix linter warnings
---
 bridges/BlizzardNewsBridge.php | 68 +++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 14 deletions(-)

diff --git a/bridges/BlizzardNewsBridge.php b/bridges/BlizzardNewsBridge.php
index 993492d4..4d82b318 100644
--- a/bridges/BlizzardNewsBridge.php
+++ b/bridges/BlizzardNewsBridge.php
@@ -1,6 +1,6 @@
 getInput('locale');
         if ('zh-cn' === $locale) {
-            return 'https://cn.news.blizzard.com';
+            $baseUrl = 'https://cn.news.blizzard.com' . self::API_PATH;
+        } else {
+            $baseUrl = 'https://news.blizzard.com/' . $locale . self::API_PATH;
         }
-        return 'https://news.blizzard.com/' . $locale;
+        return $baseUrl .= http_build_query([
+            'feedCxpProductIds' => self::PRODUCT_IDS
+        ]);
+    }
+
+    public function collectData()
+    {
+        $feedContent = json_decode(getContents($this->getSourceUrl()), true);
+
+        foreach ($feedContent['feed']['contentItems'] as $entry) {
+            $properties = $entry['properties'];
+
+            $item = [];
+
+            $item['title'] = $this->filterChars($properties['title']);
+            $item['content'] = $this->filterChars($properties['summary']);
+            $item['uri'] = $properties['newsUrl'];
+            $item['author'] = $this->filterChars($properties['author']);
+            $item['timestamp'] = strtotime($properties['lastUpdated']);
+            $item['enclosures'] = [$properties['staticAsset']['imageUrl']];
+            $item['categories'] = [$this->filterChars($properties['cxpProduct']['title'])];
+
+            $this->items[] = $item;
+        }
+    }
+
+    private function filterChars($content)
+    {
+        return htmlspecialchars($content, ENT_XML1);
     }
 
     public function getIcon()
     {
         return <<
Date: Thu, 2 Jan 2025 18:22:47 +0100
Subject: [PATCH 263/423] fix(rumble): exterminate double leading slashes in
 item url (#4381)

Fixed for items with pub date newer than 31. jan 2025
---
 bridges/RumbleBridge.php | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/bridges/RumbleBridge.php b/bridges/RumbleBridge.php
index 8d92db3b..11755b51 100644
--- a/bridges/RumbleBridge.php
+++ b/bridges/RumbleBridge.php
@@ -60,15 +60,10 @@ class RumbleBridge extends BridgeAbstract
 
         $dom = getSimpleHTMLDOM($url);
         foreach ($dom->find('ol.thumbnail__grid div.thumbnail__grid--item') as $video) {
-            $itemUrlString = self::URI . $video->find('a', 0)->href;
-            $itemUrl = Url::fromString($itemUrlString);
+            $href = $video->find('a', 0)->href;
 
             $item = [
                 'title'     => $video->find('h3', 0)->plaintext,
-
-                // Remove tracking parameter in query string
-                'uri'       => $itemUrl->withQueryString(null)->__toString(),
-
                 'author'    => $account . '@rumble.com',
                 'content'   => defaultLinkTo($video, self::URI)->innertext,
             ];
@@ -78,6 +73,14 @@ class RumbleBridge extends BridgeAbstract
                 $publishedAt = new \DateTimeImmutable($time->getAttribute('datetime'));
                 $item['timestamp'] = $publishedAt->getTimestamp();
             }
+
+            if (isset($publishedAt) && $publishedAt > new \DateTimeImmutable('2025-01-31')) {
+                $href = ltrim($href, '/');
+            }
+            $itemUrl = Url::fromString(self::URI . $href);
+            // Remove tracking parameter in query string
+            $item['uri'] = $itemUrl->withQueryString(null)->__toString();
+
             $this->items[] = $item;
         }
     }

From 45ee018a6e43ce444ed4ab8f13b62e886926f779 Mon Sep 17 00:00:00 2001
From: Sebastian Wolf <117176763+swofl@users.noreply.github.com>
Date: Fri, 3 Jan 2025 01:43:39 +0100
Subject: [PATCH 264/423] [MixologyBridge] add null checks for author and
 timestamp elements (#4383)

* [MixologyBridge] add null checks for author and timestamp elements

* [MixologyBridge] fix formatting
---
 bridges/MixologyBridge.php | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/bridges/MixologyBridge.php b/bridges/MixologyBridge.php
index e954d0d6..42192471 100644
--- a/bridges/MixologyBridge.php
+++ b/bridges/MixologyBridge.php
@@ -55,8 +55,15 @@ class MixologyBridge extends BridgeAbstract
     {
         $article = getSimpleHTMLDOMCached($item['uri']);
 
-        $item['author'] = $article->find('.beitrag-author a', 0)->plaintext;
-        $item['timestamp'] = strtotime($article->find('.beitrag-date time', 0)->datetime);
+        $authorLink = $article->find('.beitrag-author a', 0);
+        if (!empty($authorLink)) {
+            $item['author'] = $authorLink->plaintext;
+        }
+
+        $timeElement = $article->find('.beitrag-date time', 0);
+        if (!empty($timeElement)) {
+            $item['timestamp'] = strtotime($timeElement->datetime);
+        }
 
         $content = '';
 

From 7c6d4a932c7ce9678bdf3a5fcedf89a86d82ad44 Mon Sep 17 00:00:00 2001
From: Dag 
Date: Fri, 3 Jan 2025 01:58:38 +0100
Subject: [PATCH 265/423] fix: upgrade hardcoded version number, fix #4382
 (#4384)

---
 lib/Configuration.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/Configuration.php b/lib/Configuration.php
index 187848fb..c0cfe0c2 100644
--- a/lib/Configuration.php
+++ b/lib/Configuration.php
@@ -7,7 +7,7 @@
  */
 final class Configuration
 {
-    private const VERSION = '2024-02-02';
+    private const VERSION = '2025-01-02';
 
     private static $config = [];
 

From c44a76ff17833709d9bb336778fdcfd159f620b4 Mon Sep 17 00:00:00 2001
From: Dag 
Date: Fri, 3 Jan 2025 05:04:49 +0100
Subject: [PATCH 266/423] refactor: remove dead code (#4385)

---
 bridges/AirBreizhBridge.php                   |  3 +--
 bridges/AmazonPriceTrackerBridge.php          |  2 +-
 bridges/AssociatedPressNewsBridge.php         |  3 +--
 bridges/BAEBridge.php                         |  2 +-
 bridges/BandcampDailyBridge.php               |  6 ++---
 bridges/BundestagParteispendenBridge.php      |  6 ++---
 bridges/CrewbayBridge.php                     |  2 +-
 bridges/DacksnackBridge.php                   |  6 ++---
 bridges/DagensNyheterDirektBridge.php         |  3 +--
 bridges/FindACrewBridge.php                   |  2 +-
 bridges/FurAffinityUserBridge.php             |  3 +--
 bridges/GiteaBridge.php                       |  9 +++----
 bridges/GlowficBridge.php                     |  2 +-
 bridges/GogsBridge.php                        |  3 +--
 bridges/GoogleScholarBridge.php               |  4 ++--
 bridges/ItakuBridge.php                       | 24 +++++++------------
 bridges/JohannesBlickBridge.php               |  3 +--
 bridges/JustETFBridge.php                     |  3 +--
 bridges/KernelBugTrackerBridge.php            |  4 ----
 bridges/LaTeX3ProjectNewslettersBridge.php    |  2 +-
 bridges/MozillaBugTrackerBridge.php           |  4 ----
 bridges/OMonlineBridge.php                    |  6 ++---
 .../SchweinfurtBuergerinformationenBridge.php |  6 ++---
 bridges/SkimfeedBridge.php                    | 12 ++++------
 bridges/StanfordSIRbookreviewBridge.php       |  3 +--
 bridges/StockFilingsBridge.php                |  2 +-
 bridges/StorytelBridge.php                    |  3 ---
 bridges/TapasBridge.php                       |  2 +-
 bridges/TestFaktaBridge.php                   |  6 ++---
 bridges/UsesTechBridge.php                    |  3 +--
 bridges/VproTegenlichtBridge.php              |  3 +--
 bridges/WikipediaBridge.php                   |  4 ----
 bridges/XenForoBridge.php                     | 12 ++++------
 33 files changed, 52 insertions(+), 106 deletions(-)

diff --git a/bridges/AirBreizhBridge.php b/bridges/AirBreizhBridge.php
index a822625f..272c74ee 100644
--- a/bridges/AirBreizhBridge.php
+++ b/bridges/AirBreizhBridge.php
@@ -32,8 +32,7 @@ class AirBreizhBridge extends BridgeAbstract
     public function collectData()
     {
         $html = '';
-        $html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'))
-            or returnClientError('No results for this query.');
+        $html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'));
 
         foreach ($html->find('article') as $article) {
             $item = [];
diff --git a/bridges/AmazonPriceTrackerBridge.php b/bridges/AmazonPriceTrackerBridge.php
index b07bdb7c..5f93eb49 100644
--- a/bridges/AmazonPriceTrackerBridge.php
+++ b/bridges/AmazonPriceTrackerBridge.php
@@ -146,7 +146,7 @@ EOT;
     {
         $uri = $this->getURI();
 
-        return getSimpleHTMLDOM($uri) ?: returnServerError('Could not request Amazon.');
+        return getSimpleHTMLDOM($uri);
     }
 
     private function scrapePriceFromMetrics($html)
diff --git a/bridges/AssociatedPressNewsBridge.php b/bridges/AssociatedPressNewsBridge.php
index 0f8846eb..db62c826 100644
--- a/bridges/AssociatedPressNewsBridge.php
+++ b/bridges/AssociatedPressNewsBridge.php
@@ -105,8 +105,7 @@ class AssociatedPressNewsBridge extends BridgeAbstract
 
     private function collectCardData()
     {
-        $json = getContents($this->getTagURI())
-            or returnServerError('Could not request: ' . $this->getTagURI());
+        $json = getContents($this->getTagURI());
 
         $tagContents = json_decode($json, true);
 
diff --git a/bridges/BAEBridge.php b/bridges/BAEBridge.php
index 6807d548..a0e4c536 100644
--- a/bridges/BAEBridge.php
+++ b/bridges/BAEBridge.php
@@ -29,7 +29,7 @@ class BAEBridge extends BridgeAbstract
     public function collectData()
     {
         $url = $this->getURI();
-        $html = getSimpleHTMLDOM($url) or returnClientError('No results for this query.');
+        $html = getSimpleHTMLDOM($url);
 
         $annonces = $html->find('main article');
         foreach ($annonces as $annonce) {
diff --git a/bridges/BandcampDailyBridge.php b/bridges/BandcampDailyBridge.php
index 57299a17..1f9a031d 100644
--- a/bridges/BandcampDailyBridge.php
+++ b/bridges/BandcampDailyBridge.php
@@ -93,8 +93,7 @@ class BandcampDailyBridge extends BridgeAbstract
 
     public function collectData()
     {
-        $html = getSimpleHTMLDOM($this->getURI())
-            or returnServerError('Could not request: ' . $this->getURI());
+        $html = getSimpleHTMLDOM($this->getURI());
 
         $html = defaultLinkTo($html, self::URI);
 
@@ -105,8 +104,7 @@ class BandcampDailyBridge extends BridgeAbstract
 
             $articlePath = $article->find('a.title', 0)->href;
 
-            $articlePageHtml = getSimpleHTMLDOMCached($articlePath, 3600)
-                or returnServerError('Could not request: ' . $articlePath);
+            $articlePageHtml = getSimpleHTMLDOMCached($articlePath, 3600);
 
             $item['uri'] = $articlePath;
             $item['title'] = $articlePageHtml->find('article-title', 0)->innertext;
diff --git a/bridges/BundestagParteispendenBridge.php b/bridges/BundestagParteispendenBridge.php
index cdf398e8..30817d96 100644
--- a/bridges/BundestagParteispendenBridge.php
+++ b/bridges/BundestagParteispendenBridge.php
@@ -26,8 +26,7 @@ TMPL;
 https://www.bundestag.de/ajax/filterlist/de/parlament/praesidium/parteienfinanzierung/fundstellen50000/462002-462002
 URI;
         // Get the main page
-        $html = getSimpleHTMLDOMCached($ajaxUri, self::CACHE_TIMEOUT)
-            or returnServerError('Could not request AJAX list.');
+        $html = getSimpleHTMLDOMCached($ajaxUri, self::CACHE_TIMEOUT);
 
         // Build the URL from the first anchor element. The list is sorted by year, descending, so the first element is the current year.
         $firstAnchor = $html->find('a', 0)
@@ -36,8 +35,7 @@ URI;
         $url = 'https://www.bundestag.de' . $firstAnchor->href;
 
         // Get the actual page with the soft money donations
-        $html = getSimpleHTMLDOMCached($url, self::CACHE_TIMEOUT)
-            or returnServerError('Could not request ' . $url);
+        $html = getSimpleHTMLDOMCached($url, self::CACHE_TIMEOUT);
 
         $rows = $html->find('table.table > tbody > tr')
             or returnServerError('Could not find the proper HTML elements.');
diff --git a/bridges/CrewbayBridge.php b/bridges/CrewbayBridge.php
index 0ca017c2..0fb9def6 100644
--- a/bridges/CrewbayBridge.php
+++ b/bridges/CrewbayBridge.php
@@ -109,7 +109,7 @@ class CrewbayBridge extends BridgeAbstract
     public function collectData()
     {
         $url = $this->getURI();
-        $html = getSimpleHTMLDOM($url) or returnClientError('No results for this query.');
+        $html = getSimpleHTMLDOM($url);
 
         $annonces = $html->find('#SearchResults div.result');
         $limit = 0;
diff --git a/bridges/DacksnackBridge.php b/bridges/DacksnackBridge.php
index 7aab48d1..a031706e 100644
--- a/bridges/DacksnackBridge.php
+++ b/bridges/DacksnackBridge.php
@@ -53,8 +53,7 @@ class DacksnackBridge extends BridgeAbstract
     public function collectData()
     {
         $NEWSURL = self::URI;
-        $html = getSimpleHTMLDOMCached($NEWSURL, 18000) or
-            returnServerError('Could not request: ' . $NEWSURL);
+        $html = getSimpleHTMLDOMCached($NEWSURL, 18000);
 
         foreach ($html->find('a.main-news-item') as $element) {
             // Debug::log($element);
@@ -64,8 +63,7 @@ class DacksnackBridge extends BridgeAbstract
             $url = self::URI . $element->getAttribute('href');
             $published = $this->parseSwedishDates(trim($element->find('.published', 0)->plaintext));
 
-            $article_html = getSimpleHTMLDOMCached($url, 18000) or
-                returnServerError('Could not request: ' . $url);
+            $article_html = getSimpleHTMLDOMCached($url, 18000);
             $article_content = $article_html->find('#ctl00_ContentPlaceHolder1_NewsArticleVeiw_pnlArticle', 0);
 
             $figure = self::URI . $article_content->find('img.news-image', 0)->getAttribute('src');
diff --git a/bridges/DagensNyheterDirektBridge.php b/bridges/DagensNyheterDirektBridge.php
index f0748b76..fa219a09 100644
--- a/bridges/DagensNyheterDirektBridge.php
+++ b/bridges/DagensNyheterDirektBridge.php
@@ -18,8 +18,7 @@ class DagensNyheterDirektBridge extends BridgeAbstract
     {
         $NEWSURL = self::BASEURL . '/ajax/direkt/';
 
-        $html = getSimpleHTMLDOM($NEWSURL) or
-            returnServerError('Could not request: ' . $NEWSURL);
+        $html = getSimpleHTMLDOM($NEWSURL);
 
         foreach ($html->find('article') as $element) {
             $link = $element->find('button', 0)->getAttribute('data-link');
diff --git a/bridges/FindACrewBridge.php b/bridges/FindACrewBridge.php
index 9119535b..0bc181f3 100644
--- a/bridges/FindACrewBridge.php
+++ b/bridges/FindACrewBridge.php
@@ -60,7 +60,7 @@ class FindACrewBridge extends BridgeAbstract
             CURLOPT_POSTFIELDS => http_build_query($data) . "\n"
         ];
 
-        $html = getSimpleHTMLDOM($url, $header, $opts) or returnClientError('No results for this query.');
+        $html = getSimpleHTMLDOM($url, $header, $opts);
 
         $annonces = $html->find('.css_SrhRst');
         $limit = $this->getInput('limit') ?? 10;
diff --git a/bridges/FurAffinityUserBridge.php b/bridges/FurAffinityUserBridge.php
index fa10d7ae..1866308d 100644
--- a/bridges/FurAffinityUserBridge.php
+++ b/bridges/FurAffinityUserBridge.php
@@ -34,8 +34,7 @@ class FurAffinityUserBridge extends BridgeAbstract
 
         $url = self::URI . '/gallery/' . $this->getInput('searchUsername');
 
-        $html = getSimpleHTMLDOM($url, [], $opt)
-            or returnServerError('Could not load the user\'s gallery page.');
+        $html = getSimpleHTMLDOM($url, [], $opt);
 
         $submissions = $html->find('section[id=gallery-gallery]', 0)->find('figure');
         foreach ($submissions as $submission) {
diff --git a/bridges/GiteaBridge.php b/bridges/GiteaBridge.php
index f7f426e9..8433b6dd 100644
--- a/bridges/GiteaBridge.php
+++ b/bridges/GiteaBridge.php
@@ -155,8 +155,7 @@ class GiteaBridge extends BridgeAbstract
 
     public function collectData()
     {
-        $html = getSimpleHTMLDOM($this->getURI())
-            or returnServerError('Could not request ' . $this->getURI());
+        $html = getSimpleHTMLDOM($this->getURI());
         $html = defaultLinkTo($html, $this->getURI());
 
         $this->title = $html->find('[property="og:title"]', 0)->content;
@@ -246,8 +245,7 @@ class GiteaBridge extends BridgeAbstract
             ];
 
             if ($this->getInput('include_description')) {
-                $issue_html = getSimpleHTMLDOMCached($uri, 3600)
-                    or returnServerError('Unable to load issue description');
+                $issue_html = getSimpleHTMLDOMCached($uri, 3600);
 
                 $issue_html = defaultLinkTo($issue_html, $uri);
 
@@ -308,8 +306,7 @@ class GiteaBridge extends BridgeAbstract
             ];
 
             if ($this->getInput('include_description')) {
-                $issue_html = getSimpleHTMLDOMCached($uri, 3600)
-                    or returnServerError('Unable to load issue description');
+                $issue_html = getSimpleHTMLDOMCached($uri, 3600);
 
                 $issue_html = defaultLinkTo($issue_html, $uri);
 
diff --git a/bridges/GlowficBridge.php b/bridges/GlowficBridge.php
index 0e4b8d93..7a58a08f 100644
--- a/bridges/GlowficBridge.php
+++ b/bridges/GlowficBridge.php
@@ -28,7 +28,7 @@ class GlowficBridge extends BridgeAbstract
     public function collectData()
     {
         $url = $this->getAPIURI();
-        $metadata = get_headers($url . '/replies', true) or returnClientError('Post did not return reply headers.');
+        $metadata = get_headers($url . '/replies', true);
         $metadata['Last-Page'] = ceil($metadata['Total'] / $metadata['Per-Page']);
         if (
             !is_null($this->getInput('start_page')) &&
diff --git a/bridges/GogsBridge.php b/bridges/GogsBridge.php
index 685e5ba2..d838674f 100644
--- a/bridges/GogsBridge.php
+++ b/bridges/GogsBridge.php
@@ -171,8 +171,7 @@ class GogsBridge extends BridgeAbstract
             ];
 
             if ($this->getInput('include_description')) {
-                $issue_html = getSimpleHTMLDOMCached($uri, 3600)
-                    or returnServerError('Unable to load issue description');
+                $issue_html = getSimpleHTMLDOMCached($uri, 3600);
 
                 $issue_html = defaultLinkTo($issue_html, $uri);
 
diff --git a/bridges/GoogleScholarBridge.php b/bridges/GoogleScholarBridge.php
index 11dc123b..3004180f 100644
--- a/bridges/GoogleScholarBridge.php
+++ b/bridges/GoogleScholarBridge.php
@@ -109,7 +109,7 @@ class GoogleScholarBridge extends BridgeAbstract
             case 'user':
                 $userId = $this->getInput('userId');
                 $uri = self::URI . '/citations?hl=en&view_op=list_works&sortby=pubdate&user=' . $userId;
-                $html = getSimpleHTMLDOM($uri) or returnServerError('Could not fetch Google Scholar data.');
+                $html = getSimpleHTMLDOM($uri);
 
                 $publications = $html->find('tr[class="gsc_a_tr"]');
 
@@ -184,7 +184,7 @@ class GoogleScholarBridge extends BridgeAbstract
                 $uri .= $sortBy ? '&scisbd=1' : '';
                 $uri .= $numResults ? '&num=' . $numResults : '';
 
-                $html = getSimpleHTMLDOM($uri) or returnServerError('Could not fetch Google Scholar data.');
+                $html = getSimpleHTMLDOM($uri);
 
                 $publications = $html->find('div[class="gs_r gs_or gs_scl"]');
 
diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php
index b231b143..22d7529f 100644
--- a/bridges/ItakuBridge.php
+++ b/bridges/ItakuBridge.php
@@ -438,8 +438,7 @@ class ItakuBridge extends BridgeAbstract
     private function getOwnerID($username)
     {
         $url = self::URI . "/api/user_profiles/{$username}/?format=json";
-        $data = $this->getData($url, true, true)
-            or returnServerError("Could not load $url");
+        $data = $this->getData($url, true, true);
 
         return $data['owner'];
     }
@@ -451,8 +450,7 @@ class ItakuBridge extends BridgeAbstract
         }
         $uri = self::URI . '/posts/' . $id;
         $url = self::URI . '/api/posts/' . $id . '/?format=json';
-        $data = $metadata ?? $this->getData($url, true, true)
-            or returnServerError("Could not load $url");
+        $data = $metadata ?? $this->getData($url, true, true);
 
         $content_str = nl2br($data['content']);
         $content = "
{$content_str}

"; //TODO: Add link and itaku user mention detection and convert into links.
@@ -497,8 +495,7 @@ class ItakuBridge extends BridgeAbstract
                 $content .= "{$title}
";
                 if ($media['is_thumbnail_for_video']) {
                     $url = self::URI . '/api/galleries/images/' . $media['id'] . '/?format=json';
-                    $media_data = $this->getData($url, true, true)
-                        or returnServerError("Could not load $url");
+                    $media_data = $this->getData($url, true, true);
                     $content .= "
' . $this->getImage($deal) - . '"/>' + . '' . $this->getHTMLTitle($item) . $this->getPrice($jsonDealData) . $this->getDiscount($jsonDealData) @@ -430,7 +430,7 @@ HEREDOC; { // Get thread Image JSON content $content = Json::decode($deal->find('div[class*=threadGrid-image]', 0)->find('div[class=js-vue2]', 0)->getAttribute('data-vue2')); - return $content['props']['threadImageUrl']; + return ''; } /** From 7813f4564e9c4f6054c827cead26fe6a6b2605c5 Mon Sep 17 00:00:00 2001 From: July Date: Thu, 15 Feb 2024 22:14:17 -0500 Subject: [PATCH 022/423] AO3Bridge: add options to fetch chapter contents and list titles (#3981) * AO3Bridge: add options to fetch chapter contents and titles for list feeds and add downloads for each fic to enclosures * AO3Bridge: fix list default value * AO3Bridge: fix erroneous dynamic property usage * AO3Bridge: fix unit test failure for getURI --- bridges/AO3Bridge.php | 110 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 91 insertions(+), 19 deletions(-) diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php index e30c6b70..32bbb0a2 100644 --- a/bridges/AO3Bridge.php +++ b/bridges/AO3Bridge.php @@ -12,8 +12,20 @@ class AO3Bridge extends BridgeAbstract 'url' => [ 'name' => 'url', 'required' => true, - // Example: F/F tag, complete works only - 'exampleValue' => 'https://archiveofourown.org/works?work_search[complete]=T&tag_id=F*s*F', + // Example: F/F tag + 'exampleValue' => 'https://archiveofourown.org/tags/F*s*F/works', + ], + 'range' => [ + 'name' => 'Chapter Content', + 'title' => 'Chapter(s) to include in each work\'s feed entry', + 'defaultValue' => null, + 'type' => 'list', + 'values' => [ + 'None' => null, + 'First' => 'first', + 'Latest' => 'last', + 'Entire work' => 'all', + ], ], ], 'Bookmarks' => [ @@ -39,18 +51,13 @@ class AO3Bridge extends BridgeAbstract { switch ($this->queriedContext) { case 'Bookmarks': - $user = $this->getInput('user'); - $this->title = $user; - $url = self::URI - . '/users/' . $user - . '/bookmarks?bookmark_search[sort_column]=bookmarkable_date'; - $this->collectList($url); + $this->collectList($this->getURI()); break; case 'List': - $this->collectList($this->getInput('url')); + $this->collectList($this->getURI()); break; case 'Work': - $this->collectWork($this->getInput('id')); + $this->collectWork($this->getURI()); break; } } @@ -61,9 +68,21 @@ class AO3Bridge extends BridgeAbstract */ private function collectList($url) { - $html = getSimpleHTMLDOM($url); + $httpClient = RssBridge::getHttpClient(); + $version = 'v0.0.1'; + $agent = ['useragent' => "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"]; + + $response = $httpClient->request($url, $agent); + $html = \str_get_html($response->getBody()); $html = defaultLinkTo($html, self::URI); + // Get list title. Will include page range + count in some cases + $heading = ($html->find('#main > h2', 0)); + if ($heading->find('a.tag')) { + $heading = $heading->find('a.tag', 0); + } + $this->title = $heading->plaintext; + foreach ($html->find('.index.group > li') as $element) { $item = []; @@ -83,6 +102,36 @@ class AO3Bridge extends BridgeAbstract $chapters = (isset($chapters) ? $chapters->plaintext : 0); $item['uid'] = $item['uri'] . "/$strdate/$chapters"; + // Fetch workskin of desired chapter(s) in list + if ($this->getInput('range')) { + $url = $item['uri']; + switch ($this->getInput('range')) { + case ('all'): + $url .= '?view_full_work=true'; + break; + case ('first'): + break; + case ('last'): + // only way to get this is using the navigate page unfortunately + $url .= '/navigate'; + $response = $httpClient->request($url, $agent); + $html = \str_get_html($response->getBody()); + $html = defaultLinkTo($html, self::URI); + $url = $html->find('ol.index.group > li > a', -1)->href; + break; + } + $response = $httpClient->request($url, $agent); + $html = \str_get_html($response->getBody()); + $html = defaultLinkTo($html, self::URI); + $item['content'] .= $html->find('#workskin', 0); + } + + // Use predictability of download links to generate enclosures + $wid = explode('/', $item['uri'])[4]; + foreach (['azw3', 'epub', 'mobi', 'pdf', 'html'] as $ext) { + $item['enclosures'][] = 'https://archiveofourown.org/downloads/' . $wid . '/work.' . $ext; + } + $this->items[] = $item; } } @@ -90,26 +139,29 @@ class AO3Bridge extends BridgeAbstract /** * Feed for recent chapters of a specific work. */ - private function collectWork($id) + private function collectWork($url) { - $url = self::URI . "/works/$id/navigate"; $httpClient = RssBridge::getHttpClient(); - $version = 'v0.0.1'; - $response = $httpClient->request($url, [ - 'useragent' => "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)", - ]); + $agent = ['useragent' => "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"]; + $response = $httpClient->request($url . '/navigate', $agent); $html = \str_get_html($response->getBody()); $html = defaultLinkTo($html, self::URI); + $response = $httpClient->request($url . '?view_full_work=true', $agent); + $workhtml = \str_get_html($response->getBody()); + $workhtml = defaultLinkTo($workhtml, self::URI); + $this->title = $html->find('h2 a', 0)->plaintext; - foreach ($html->find('ol.index.group > li') as $element) { + $nav = $html->find('ol.index.group > li'); + for ($i = 0; $i < count($nav); $i++) { $item = []; + $element = $nav[$i]; $item['title'] = $element->find('a', 0)->plaintext; - $item['content'] = $element; + $item['content'] = $workhtml->find('#chapter-' . ($i + 1), 0); $item['uri'] = $element->find('a', 0)->href; $strdate = $element->find('span.datetime', 0)->plaintext; @@ -138,4 +190,24 @@ class AO3Bridge extends BridgeAbstract { return self::URI . '/favicon.ico'; } + + public function getURI() + { + $url = parent::getURI(); + switch ($this->queriedContext) { + case 'Bookmarks': + $user = $this->getInput('user'); + $url = self::URI + . '/users/' . $user + . '/bookmarks?bookmark_search[sort_column]=bookmarkable_date'; + break; + case 'List': + $url = $this->getInput('url'); + break; + case 'Work': + $url = self::URI . '/works/' . $this->getInput('id'); + break; + } + return $url; + } } From e65155f440b9d66771fd6096f4c13b77323a6217 Mon Sep 17 00:00:00 2001 From: Korytov Pavel Date: Sat, 17 Feb 2024 00:24:13 +0300 Subject: [PATCH 023/423] [OpenCVEBridge] Add bridge (#3978) * [OpenCVEBridge] Add bridge * [OpenCVEBridge] Fix tests * [OpenCVEBridge] Fix description of the filter parameter --- bridges/OpenCVEBridge.php | 427 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 427 insertions(+) create mode 100644 bridges/OpenCVEBridge.php diff --git a/bridges/OpenCVEBridge.php b/bridges/OpenCVEBridge.php new file mode 100644 index 00000000..594bb9ec --- /dev/null +++ b/bridges/OpenCVEBridge.php @@ -0,0 +1,427 @@ + [ + 'instance' => [ + 'name' => 'OpenCVE Instance', + 'required' => true, + 'defaultValue' => 'https://www.opencve.io', + 'exampleValue' => 'https://www.opencve.io' + ], + 'login' => [ + 'name' => 'Login', + 'type' => 'text', + 'required' => true + ], + 'password' => [ + 'name' => 'Password', + 'type' => 'text', + 'required' => true + ], + 'pages' => [ + 'name' => 'Number of pages', + 'type' => 'number', + 'required' => false, + 'exampleValue' => 1, + 'defaultValue' => 1 + ], + 'filter' => [ + 'name' => 'Filter', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'search:jenkins;product:gitlab,cvss:critical', + 'title' => 'Syntax: param1:value1,param2:value2;param1query2:param2query2. See https://docs.opencve.io/api/cve/ for parameters' + ], + 'upd_timestamp' => [ + 'name' => 'Use updated_at instead of created_at as timestamp', + 'type' => 'checkbox' + ], + 'trunc_summary' => [ + 'name' => 'Truncate summary for header', + 'type' => 'number', + 'defaultValue' => 100 + ], + 'fetch_contents' => [ + 'name' => 'Fetch detailed contents for CVEs', + 'defaultValue' => 'checked', + 'type' => 'checkbox' + ] + ] + ]; + + const CSS = ' + '; + + public function collectData() + { + $creds = $this->getInput('login') . ':' . $this->getInput('password'); + $authHeader = 'Authorization: Basic ' . base64_encode($creds); + $instance = $this->getInput('instance'); + + $queries = []; + $filter = $this->getInput('filter'); + $filterValues = []; + if ($filter && mb_strlen($filter) > 0) { + $filterValues = explode(';', $filter); + } else { + $queries[''] = []; + } + foreach ($filterValues as $filterValue) { + $params = explode(',', $filterValue); + $queryName = $filterValue; + $query = []; + foreach ($params as $param) { + [$key, $value] = explode(':', $param); + if ($key == 'title') { + $queryName = $value; + } else { + $query[$key] = $value; + } + } + $queries[$queryName] = $query; + } + + $fetchedIds = []; + + foreach ($queries as $queryName => $query) { + for ($i = 1; $i <= $this->getInput('pages'); $i++) { + $queryPaginated = array_merge($query, ['page' => $i]); + $url = $instance . '/api/cve?' . http_build_query($queryPaginated); + $response = getContents( + $url, + [$authHeader] + ); + $titlePrefix = ''; + if (count($queries) > 1) { + $titlePrefix = '[' . $queryName . '] '; + } + + foreach (json_decode($response) as $cveItem) { + if (array_key_exists($cveItem->id, $fetchedIds)) { + continue; + } + $fetchedIds[$cveItem->id] = true; + $item = [ + 'uri' => $instance . '/cve/' . $cveItem->id, + 'uid' => $cveItem->id, + ]; + if ($this->getInput('upd_timestamp') == 1) { + $item['timestamp'] = strtotime($cveItem->updated_at); + } else { + $item['timestamp'] = strtotime($cveItem->created_at); + } + if ($this->getInput('fetch_contents')) { + [$content, $title] = $this->fetchContents( + $cveItem, + $titlePrefix, + $instance, + $authHeader + ); + $item['content'] = $content; + $item['title'] = $title; + } else { + $item['content'] = $cveItem->summary . $this->getLinks($cveItem->id); + $item['title'] = $this->getTitle($titlePrefix, $cveItem); + } + $this->items[] = $item; + } + } + } + usort($this->items, function ($a, $b) { + return $b['timestamp'] - $a['timestamp']; + }); + } + + private function getTitle($titlePrefix, $cveItem) + { + $summary = $cveItem->summary; + $limit = $this->getInput('limit'); + if ($limit && mb_strlen($summary) > 100) { + $summary = mb_substr($summary, 0, $limit) + '...'; + } + return $titlePrefix . $cveItem->id . '. ' . $summary; + } + + private function fetchContents($cveItem, $titlePrefix, $instance, $authHeader) + { + $url = $instance . '/api/cve/' . $cveItem->id; + $response = getContents( + $url, + [$authHeader] + ); + $datum = json_decode($response); + + $title = $this->getTitleFromDatum($datum, $titlePrefix); + + $result = self::CSS; + $result .= '

' . $cveItem->id . '

'; + $result .= $this->getCVSSLabels($datum); + $result .= '

' . $datum->summary . '

'; + $result .= <<Information: +

+

    +
  • Publication date: {$datum->raw_nvd_data->published} +
  • Last modified: {$datum->raw_nvd_data->lastModified} +
  • Last modified: {$datum->raw_nvd_data->lastModified} +
+

+ EOD; + + $result .= $this->getV3Table($datum); + $result .= $this->getV2Table($datum); + + $result .= $this->getLinks($datum->id); + $result .= $this->getReferences($datum); + + $result .= $this->getVendors($datum); + + return [$result, $title]; + } + + private function getTitleFromDatum($datum, $titlePrefix) + { + $title = $titlePrefix; + if ($datum->cvss->v3) { + $title .= "[v3: {$datum->cvss->v3}] "; + } + if ($datum->cvss->v2) { + $title .= "[v2: {$datum->cvss->v2}] "; + } + $title .= $datum->id . '. '; + $titlePostfix = $datum->summary; + $limit = $this->getInput('limit'); + if ($limit && mb_strlen($titlePostfix) > 100) { + $titlePostfix = mb_substr($titlePostfix, 0, $limit) + '...'; + } + $title .= $titlePostfix; + return $title; + } + + private function getCVSSLabels($datum) + { + $CVSSv2Text = 'n/a'; + $CVSSv2Class = 'cvss-na-color'; + if ($datum->cvss->v2) { + $importance = ''; + if ($datum->cvss->v2 >= 7) { + $importance = 'HIGH'; + $CVSSv2Class = 'cvss-high-color'; + } else if ($datum->cvss->v2 >= 4) { + $importance = 'MEDIUM'; + $CVSSv2Class = 'cvss-medium-color'; + } else { + $importance = 'LOW'; + $CVSSv2Class = 'cvss-low-color'; + } + $CVSSv2Text = sprintf('[%s] %.1f', $importance, $datum->cvss->v2); + } + $CVSSv2Item = "
CVSS v2:
{$CVSSv2Text}
"; + + $CVSSv3Text = 'n/a'; + $CVSSv3Class = 'cvss-na-color'; + if ($datum->cvss->v3) { + $importance = ''; + if ($datum->cvss->v3 >= 9) { + $importance = 'CRITICAL'; + $CVSSv3Class = 'cvss-crit-color'; + } else if ($datum->cvss->v3 >= 7) { + $importance = 'HIGH'; + $CVSSv3Class = 'cvss-high-color'; + } else if ($datum->cvss->v3 >= 4) { + $importance = 'MEDIUM'; + $CVSSv3Class = 'cvss-medium-color'; + } else { + $importance = 'LOW'; + $CVSSv3Class = 'cvss-low-color'; + } + $CVSSv3Text = sprintf('[%s] %.1f', $importance, $datum->cvss->v3); + } + $CVSSv3Item = "
CVSS v3:
{$CVSSv3Text}
"; + return '
' . $CVSSv3Item . $CVSSv2Item . '
'; + } + + private function getReferences($datum) + { + if (count($datum->raw_nvd_data->references) == 0) { + return ''; + } + $res = '

References:

    '; + foreach ($datum->raw_nvd_data->references as $ref) { + $item = '
  • '; + if (isset($ref->tags) && count($ref->tags) > 0) { + $item .= '[' . implode(', ', $ref->tags) . '] '; + } + $item .= "url}\">{$ref->url}"; + $item .= '
  • '; + $res .= $item; + } + $res .= '

'; + return $res; + } + + private function getLinks($id) + { + return <<Links +

+

+

+ EOD; + } + + private function getV3Table($datum) + { + $metrics = $datum->raw_nvd_data->metrics; + if (!isset($metrics->cvssMetricV31) || count($metrics->cvssMetricV31) == 0) { + return ''; + } + $v3 = $metrics->cvssMetricV31[0]; + $data = $v3->cvssData; + return << +

CVSS v3 details

+ + + + + + + + + + + + + + + + + + + + + +
Impact score{$v3->impactScore}Exploitability score{$v3->exploitabilityScore}
Attack vector{$data->attackVector}Confidentiality Impact{$data->confidentialityImpact}
Attack complexity{$data->attackComplexity}Integrity Impact{$data->integrityImpact}
Privileges Required{$data->privilegesRequired}Availability Impact{$data->availabilityImpact}
User Interaction{$data->userInteraction}Scope{$data->scope}
+ + EOD; + } + + private function getV2Table($datum) + { + $metrics = $datum->raw_nvd_data->metrics; + if (!isset($metrics->cvssMetricV2) || count($metrics->cvssMetricV2) == 0) { + return ''; + } + $v2 = $metrics->cvssMetricV2[0]; + $data = $v2->cvssData; + return << +

CVSS v2 details

+ + + + + + + + + + + + + + + + + + +
Impact score{$v2->impactScore}Exploitability score{$v2->exploitabilityScore}
Access Vector{$data->accessVector}Confidentiality Impact{$data->confidentialityImpact}
Access Complexity{$data->accessComplexity}Integrity Impact{$data->integrityImpact}
Authentication{$data->authentication}Availability Impact{$data->availabilityImpact}
+ + EOD; + } + + private function getVendors($datum) + { + if (count((array)$datum->vendors) == 0) { + return ''; + } + $res = '

Affected products

    '; + foreach ($datum->vendors as $vendor => $products) { + $res .= "
  • {$vendor}"; + if (count($products) > 0) { + $res .= '
      '; + foreach ($products as $product) { + $res .= '
    • ' . $product . '
      • '; + } + $res .= '
    '; + } + $res .= '
    • '; + } + $res .= '

'; + } +} From 932f20d4341344807d6ae25d53ba811ba1e4280e Mon Sep 17 00:00:00 2001 From: hleskien <34342248+hleskien@users.noreply.github.com> Date: Sun, 18 Feb 2024 19:19:33 +0100 Subject: [PATCH 024/423] fixed date with time in LuftfahrtBundesAmtBridge (#3987) --- bridges/LuftfahrtBundesAmtBridge.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bridges/LuftfahrtBundesAmtBridge.php b/bridges/LuftfahrtBundesAmtBridge.php index 2b0384a2..406d2476 100644 --- a/bridges/LuftfahrtBundesAmtBridge.php +++ b/bridges/LuftfahrtBundesAmtBridge.php @@ -26,8 +26,13 @@ class LuftfahrtBundesAmtBridge extends XPathAbstract protected function formatItemTimestamp($value) { $value = trim($value); - $dti = DateTimeImmutable::createFromFormat('d.m.Y', $value); - $dti = $dti->setTime(0, 0, 0); + if (strpos($value, 'Uhr') !== false) { + $value = str_replace(' Uhr', '', $value); + $dti = DateTimeImmutable::createFromFormat('d.m.Y G:i', $value); + } else { + $dti = DateTimeImmutable::createFromFormat('d.m.Y', $value); + $dti = $dti->setTime(0, 0); + } return $dti->getTimestamp(); } From 35f6e62e458c88f47bc857a3a186cf24edcae502 Mon Sep 17 00:00:00 2001 From: xduugu Date: Tue, 20 Feb 2024 07:03:04 +0000 Subject: [PATCH 025/423] docker: Use pre-built curl-impersonate library from github releases (#3984) The docker image is only available for `amd64` architecture and therefore cannot be used for arm images. Fixes #3983 --- Dockerfile | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f1f4f3d..1326dba0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,3 @@ -FROM lwthiker/curl-impersonate:0.5-ff-slim-buster AS curlimpersonate - FROM debian:12-slim AS rssbridge LABEL description="RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one." @@ -7,7 +5,8 @@ LABEL repository="https://github.com/RSS-Bridge/rss-bridge" LABEL website="https://github.com/RSS-Bridge/rss-bridge" ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && \ +RUN set -xe && \ + apt-get update && \ apt-get install --yes --no-install-recommends \ ca-certificates \ nginx \ @@ -24,18 +23,44 @@ RUN apt-get update && \ php-xml \ php-zip \ # php-zlib is enabled by default with PHP 8.2 in Debian 12 + # for downloading libcurl-impersonate + curl \ && \ + # install curl-impersonate library + curlimpersonate_version=0.6.0 && \ + { \ + { \ + [ $(arch) = 'aarch64' ] && \ + archive="libcurl-impersonate-v${curlimpersonate_version}.aarch64-linux-gnu.tar.gz" && \ + sha512sum="d04b1eabe71f3af06aa1ce99b39a49c5e1d33b636acedcd9fad163bc58156af5c3eb3f75aa706f335515791f7b9c7a6c40ffdfa47430796483ecef929abd905d" \ + ; } \ + || { \ + [ $(arch) = 'armv7l' ] && \ + archive="libcurl-impersonate-v${curlimpersonate_version}.arm-linux-gnueabihf.tar.gz" && \ + sha512sum="05906b4efa1a6ed8f3b716fd83d476b6eea6bfc68e3dbc5212d65a2962dcaa7bd1f938c9096a7535252b11d1d08fb93adccc633585ff8cb8cec5e58bfe969bc9" \ + ; } \ + || { \ + [ $(arch) = 'x86_64' ] && \ + archive="libcurl-impersonate-v${curlimpersonate_version}.x86_64-linux-gnu.tar.gz" && \ + sha512sum="480bbe9452cd9aff2c0daaaf91f1057b3a96385f79011628a9237223757a9b0d090c59cb5982dc54ea0d07191657299ea91ca170a25ced3d7d410fcdff130ace" \ + ; } \ + } && \ + curl -LO "https://github.com/lwthiker/curl-impersonate/releases/download/v${curlimpersonate_version}/${archive}" && \ + echo "$sha512sum $archive" | sha512sum -c - && \ + mkdir -p /usr/local/lib/curl-impersonate && \ + tar xaf "$archive" -C /usr/local/lib/curl-impersonate --wildcards 'libcurl-impersonate-ff.so*' && \ + rm "$archive" && \ + apt-get purge --assume-yes curl && \ rm -rf /var/lib/apt/lists/* +ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so +ENV CURL_IMPERSONATE ff91esr + # logs should go to stdout / stderr RUN ln -sfT /dev/stderr /var/log/nginx/error.log; \ ln -sfT /dev/stdout /var/log/nginx/access.log; \ chown -R --no-dereference www-data:adm /var/log/nginx/ -COPY --from=curlimpersonate /usr/local/lib/libcurl-impersonate-ff.so /usr/local/lib/curl-impersonate/ -ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so -ENV CURL_IMPERSONATE ff91esr - COPY ./config/nginx.conf /etc/nginx/sites-available/default COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.ini From 4c355ba3083ded7a2d621baa42c1233a46cd51dd Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 20 Feb 2024 19:32:31 +0100 Subject: [PATCH 026/423] fix(FilterBridge): trim title so that regex filter works as expected (#3989) The fix is in FeedParser, so this fixes all usages of FeedParser where title is now trimmed. fix #3985 --- lib/FeedParser.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 37d3005b..b774cc14 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -92,7 +92,7 @@ final class FeedParser $item['uri'] = (string)$feedItem->id; } if (isset($feedItem->title)) { - $item['title'] = html_entity_decode((string)$feedItem->title); + $item['title'] = trim(html_entity_decode((string)$feedItem->title)); } if (isset($feedItem->updated)) { $item['timestamp'] = strtotime((string)$feedItem->updated); @@ -154,7 +154,7 @@ final class FeedParser $item['uri'] = (string)$feedItem->link; } if (isset($feedItem->title)) { - $item['title'] = html_entity_decode((string)$feedItem->title); + $item['title'] = trim(html_entity_decode((string)$feedItem->title)); } if (isset($feedItem->description)) { $item['content'] = (string)$feedItem->description; From 683c968d646dbaf9719a1ef7b59797240fec7617 Mon Sep 17 00:00:00 2001 From: D5k H3h <85834680+dhuschde@users.noreply.github.com> Date: Fri, 1 Mar 2024 20:24:14 +0100 Subject: [PATCH 027/423] [Rooster Teeth] Add Camp Camp channel (#3992) --- bridges/RoosterTeethBridge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/bridges/RoosterTeethBridge.php b/bridges/RoosterTeethBridge.php index 21bac4fe..464c83a8 100644 --- a/bridges/RoosterTeethBridge.php +++ b/bridges/RoosterTeethBridge.php @@ -17,6 +17,7 @@ class RoosterTeethBridge extends BridgeAbstract 'values' => [ 'All channels' => 'all', 'Achievement Hunter' => 'achievement-hunter', + 'Camp Camp' => 'camp-camp', 'Cow Chop' => 'cow-chop', 'Death Battle' => 'death-battle', 'Friends of RT' => 'friends-of-rt', From 7a7f8d5050177dfc4eac763feb257745ce39c238 Mon Sep 17 00:00:00 2001 From: July Date: Tue, 5 Mar 2024 19:28:24 -0500 Subject: [PATCH 028/423] AnnasArchiveBridge: correctly handling partial matches and file links (#3997) --- bridges/AnnasArchiveBridge.php | 40 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/bridges/AnnasArchiveBridge.php b/bridges/AnnasArchiveBridge.php index acb943b4..b857fadf 100644 --- a/bridges/AnnasArchiveBridge.php +++ b/bridges/AnnasArchiveBridge.php @@ -126,30 +126,36 @@ class AnnasArchiveBridge extends BridgeAbstract return; } - $elements = $list->find('.w-full > .mb-4 > div > a'); + $elements = $list->find('.w-full > .mb-4 > div'); foreach ($elements as $element) { - $item = []; - $item['title'] = $element->find('h3', 0)->plaintext; - $item['author'] = $element->find('div.italic', 0)->plaintext; - $item['uri'] = $element->href; - $item['content'] = $element->plaintext; - $item['uid'] = $item['uri']; + // stop added entries once partial match list starts + if (str_contains($element->innertext, 'partial match')) { + break; + } + if ($element = $element->find('a', 0)) { + $item = []; + $item['title'] = $element->find('h3', 0)->plaintext; + $item['author'] = $element->find('div.italic', 0)->plaintext; + $item['uri'] = $element->href; + $item['content'] = $element->plaintext; + $item['uid'] = $item['uri']; - $item_html = getSimpleHTMLDOMCached($item['uri'], 86400 * 20); - if ($item_html) { - $item_html = defaultLinkTo($item_html, self::URI); - $item['content'] .= $item_html->find('main img', 0); - $item['content'] .= $item_html->find('main .mt-4', 0); // Summary - if ($links = $item_html->find('main ul.mb-4', -1)) { - foreach ($links->find('li > a.js-download-link') as $file) { - $item['enclosures'][] = $file->href; + $item_html = getSimpleHTMLDOMCached($item['uri'], 86400 * 20); + if ($item_html) { + $item_html = defaultLinkTo($item_html, self::URI); + $item['content'] .= $item_html->find('main img', 0); + $item['content'] .= $item_html->find('main .mt-4', 0); // Summary + foreach ($item_html->find('main ul.mb-4 > li > a.js-download-link') as $file) { + if (!str_contains($file->href, 'fast_download')) { + $item['enclosures'][] = $file->href; + } } // Remove bulk torrents from enclosures list $item['enclosures'] = array_diff($item['enclosures'], [self::URI . 'datasets']); } - } - $this->items[] = $item; + $this->items[] = $item; + } } } From f7c1b7193961b957dfa0fe192b4a95d09ad9b1c6 Mon Sep 17 00:00:00 2001 From: July Date: Wed, 6 Mar 2024 13:40:59 -0500 Subject: [PATCH 029/423] NyaaTorrentsBridge: add torrent to enclosures and generate better feed name (#3996) * NyaaTorrentsBridge: add torrent to enclosures and generate better feed name * NyaaTorrentsBridge: fix accidental () in bridge name --- bridges/NyaaTorrentsBridge.php | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php index fcf2b197..36708411 100644 --- a/bridges/NyaaTorrentsBridge.php +++ b/bridges/NyaaTorrentsBridge.php @@ -66,22 +66,20 @@ class NyaaTorrentsBridge extends BridgeAbstract $feed = $feedParser->parseFeed(getContents($this->getURI())); foreach ($feed['items'] as $item) { - $item['id'] = str_replace(['https://nyaa.si/download/', '.torrent'], '', $item['uri']); - $item['uri'] = str_replace('/download/', '/view/', $item['uri']); + $item['enclosures'] = [$item['uri']]; $item['uri'] = str_replace('.torrent', '', $item['uri']); + $item['uri'] = str_replace('/download/', '/view/', $item['uri']); + $item['id'] = str_replace('https://nyaa.si/view/', '', $item['uri']); $dom = getSimpleHTMLDOMCached($item['uri']); if ($dom) { $description = $dom->find('#torrent-description', 0)->innertext ?? ''; - $itemDom = str_get_html(markdownToHtml(html_entity_decode($description))); - $item_image = $this->getURI() . 'static/img/avatar/default.png'; - foreach ($itemDom->find('img') as $img) { - if (strpos($img->src, 'prez') === false) { - $item_image = $img->src; - break; - } - } - $item['enclosures'] = [$item_image]; - $item['content'] = (string) $itemDom; + $item['content'] = markdownToHtml(html_entity_decode($description)); + + $magnet = $dom->find('div.panel-footer.clearfix > a', 1)->href; + // can't put raw magnet link in enclosure, this gives information on + // magnet contents and works a way to sent magnet value + $magnet = 'https://torrent.parts/#' . html_entity_decode($magnet); + array_push($item['enclosures'], $magnet); } $this->items[] = $item; if (count($this->items) >= 10) { @@ -90,6 +88,15 @@ class NyaaTorrentsBridge extends BridgeAbstract } } + public function getName() + { + $name = parent::getName(); + $name .= $this->getInput('u') ? ' - ' . $this->getInput('u') : ''; + $name .= $this->getInput('q') ? ' - ' . $this->getInput('q') : ''; + $name .= $this->getInput('c') ? ' (' . $this->getKey('c') . ')' : ''; + return $name; + } + public function getIcon() { return self::URI . 'static/favicon.png'; From 79699131e87e831fc5fa4fb785c210f5396e2bbd Mon Sep 17 00:00:00 2001 From: tillcash Date: Fri, 8 Mar 2024 17:16:32 +0530 Subject: [PATCH 030/423] [MaalaimalarBridge] New Bridge (#4001) --- bridges/MaalaimalarBridge.php | 117 ++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 bridges/MaalaimalarBridge.php diff --git a/bridges/MaalaimalarBridge.php b/bridges/MaalaimalarBridge.php new file mode 100644 index 00000000..87f85694 --- /dev/null +++ b/bridges/MaalaimalarBridge.php @@ -0,0 +1,117 @@ + [ + 'name' => 'topic', + 'type' => 'list', + 'values' => [ + 'news' => [ + 'tamilnadu' => 'news/state', + 'puducherry' => 'puducherry', + 'india' => 'news/national', + 'world' => 'news/world', + ], + 'district' => [ + 'chennai' => 'chennai', + 'ariyalur' => 'ariyalur', + 'chengalpattu' => 'chengalpattu', + 'coimbatore' => 'coimbatore', + 'cuddalore' => 'cuddalore', + 'dharmapuri' => 'dharmapuri', + 'dindugal' => 'dindugal', + 'erode' => 'erode', + 'kaanchepuram' => 'kaanchepuram', + 'kallakurichi' => 'kallakurichi', + 'kanyakumari' => 'kanyakumari', + 'karur' => 'karur', + 'krishnagiri' => 'krishnagiri', + 'madurai' => 'madurai', + 'mayiladuthurai' => 'mayiladuthurai', + 'nagapattinam' => 'nagapattinam', + 'namakal' => 'namakal', + 'nilgiris' => 'nilgiris', + 'perambalur' => 'perambalur', + 'pudukottai' => 'pudukottai', + 'ramanathapuram' => 'ramanathapuram', + 'ranipettai' => 'ranipettai', + 'salem' => 'salem', + 'sivagangai' => 'sivagangai', + 'tanjore' => 'tanjore', + 'theni' => 'theni', + 'thenkasi' => 'thenkasi', + 'thiruchirapalli' => 'thiruchirapalli', + 'thirunelveli' => 'thirunelveli', + 'thirupathur' => 'thirupathur', + 'thiruvarur' => 'thiruvarur', + 'thoothukudi' => 'thoothukudi', + 'tirupur' => 'tirupur', + 'tiruvallur' => 'tiruvallur', + 'tiruvannamalai' => 'tiruvannamalai', + 'vellore' => 'vellore', + 'villupuram' => 'villupuram', + 'virudhunagar' => 'virudhunagar', + ], + 'cinema' => [ + 'news' => 'cinema/cinemanews', + 'gossip' => 'cinema/gossip', + ], + ], + ], + ], + ]; + + public function getName() + { + $topic = $this->getKey('topic'); + return self::NAME . ($topic ? ' - ' . ucfirst($topic) : ''); + } + + public function collectData() + { + $dom = getSimpleHTMLDOM(self::URI . $this->getInput('topic')); + $articles = $dom->find('div.mb-20.infinite-card-wrapper.white-section'); + + foreach ($articles as $article) { + $titleElement = $article->find('h2.title a', 0); + if (!$titleElement) { + continue; + } + + $dateElement = $article->find('time.h-date span', 0); + $date = $dateElement ? $dateElement->{'data-datestring'} . 'UTC' : ''; + + $content = $this->constructContent($article); + + $this->items[] = [ + 'content' => $content, + 'timestamp' => $date, + 'title' => $titleElement->plaintext, + 'uid' => $titleElement->href, + 'uri' => self::URI . $titleElement->href, + ]; + } + } + + private function constructContent($article) + { + $content = ''; + $imageElement = $article->find('div.ignore-autoplay img', 0); + if ($imageElement) { + $content .= '

'; + } + + $storyElement = $article->find('div.story-content', 0); + if ($storyElement) { + $content .= $storyElement->innertext; + } + + return $content; + } +} From 84b93e0f8f67dd69a242e3ecb1cfd743b5e167e4 Mon Sep 17 00:00:00 2001 From: Jonathan Kay Date: Sun, 10 Mar 2024 10:18:50 -0400 Subject: [PATCH 031/423] [ComicsKingdomBridge] Fix/Rewrite of ComicsKingdom Bridge (#4003) * Rewrite ComicsKingdom Bridge Rewrite of bridge as the existing one no longer works: - Now uses REST API - Added optional limit to get desired number of comics - Author now reflects the comic creators name - Feed name and comic titles now pulled from site - Added myself as the maintainer as I've been the one maintaining, and the existing code no longer is used * Change API to URI to pass test * Remove whitespace, add curly braces and switch to single quotes --- bridges/ComicsKingdomBridge.php | 66 ++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/bridges/ComicsKingdomBridge.php b/bridges/ComicsKingdomBridge.php index 8baf7511..227426c4 100644 --- a/bridges/ComicsKingdomBridge.php +++ b/bridges/ComicsKingdomBridge.php @@ -2,59 +2,65 @@ class ComicsKingdomBridge extends BridgeAbstract { - const MAINTAINER = 'stjohnjohnson'; + const MAINTAINER = 'TReKiE'; + // const MAINTAINER = 'stjohnjohnson'; const NAME = 'Comics Kingdom Unofficial RSS'; - const URI = 'https://comicskingdom.com/'; + const URI = 'https://wp.comicskingdom.com/wp-json/wp/v2/ck_comic'; const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Comics Kingdom Unofficial RSS'; const PARAMETERS = [ [ 'comicname' => [ - 'name' => 'comicname', + 'name' => 'Name of comic', 'type' => 'text', 'exampleValue' => 'mutts', 'title' => 'The name of the comic in the URL after https://comicskingdom.com/', 'required' => true + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'title' => 'The number of recent comics to get', + 'defaultValue' => 10 ] ]]; + protected $comicName; + public function collectData() { - $html = getSimpleHTMLDOM($this->getURI(), [], [], true, false); + $json = getContents($this->getURI()); + $data = json_decode($json, false); - // Get author from first page - $author = $html->find('div.author p', 0); - ; + if (isset($data[0]->_embedded->{'wp:term'}[0][0])) { + $this->comicName = $data[0]->_embedded->{'wp:term'}[0][0]->name; + } - // Get current date/link - $link = $html->find('meta[property=og:url]', -1)->content; - for ($i = 0; $i < 3; $i++) { + foreach ($data as $comicitem) { $item = []; - $page = getSimpleHTMLDOM($link); - - $imagelink = $page->find('meta[property=og:image]', 0)->content; - - $date = explode('/', $link); - - $item['id'] = $imagelink; - $item['uri'] = $link; - $item['author'] = $author; - $item['title'] = 'Comics Kingdom ' . $this->getInput('comicname'); - $item['timestamp'] = DateTime::createFromFormat('Y-m-d', $date[count($date) - 1])->getTimestamp(); - $item['content'] = ''; - + $item['id'] = $comicitem->id; + $item['uri'] = $comicitem->yoast_head_json->og_url; + $item['author'] = str_ireplace('By ', '', $comicitem->ck_comic_byline); + $item['title'] = $comicitem->yoast_head_json->title; + $item['timestamp'] = $comicitem->date; + $item['content'] = ''; $this->items[] = $item; - $link = $page->find('div.comic-viewer-inline a', 0)->href; - if (empty($link)) { - break; // allow bridge to continue if there's less than 3 comics - } } } public function getURI() { if (!is_null($this->getInput('comicname'))) { - return self::URI . urlencode($this->getInput('comicname')); + $params = [ + 'ck_feature' => $this->getInput('comicname'), + 'per_page' => $this->getInput('limit'), + 'date_inclusive' => 'true', + 'order' => 'desc', + 'page' => '1', + '_embed' => 'true' + ]; + + return self::URI . '?' . http_build_query($params); } return parent::getURI(); @@ -62,8 +68,8 @@ class ComicsKingdomBridge extends BridgeAbstract public function getName() { - if (!is_null($this->getInput('comicname'))) { - return $this->getInput('comicname') . ' - Comics Kingdom'; + if ($this->comicName) { + return $this->comicName . ' - Comics Kingdom'; } return parent::getName(); From 254efc281255a5d3308d33007e09ffd3bc2df49e Mon Sep 17 00:00:00 2001 From: Mynacol Date: Sun, 10 Mar 2024 22:21:10 +0100 Subject: [PATCH 032/423] [ZeitBridge] Remove doubled text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first two paragraphs were repeated at the end of articles. The first CSS selector filters those out (example 1). The second CSS selector removes a "Zum Anschauen benötigen wir Ihre Zustimmung" line from a poll widget. We can't load the widget successfully, therefore we should remove all embeds that seem to use javascript (example 2). 1: https://www.zeit.de/campus/2024-03/bundesregierung-wissenschaft-arbeitsvertrag-regeln 2: https://www.zeit.de/campus/2024-03/ausbildung-abgebrochen-gruende-azubi-aufruf --- bridges/ZeitBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index 0ed9276b..b9806e5a 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -87,7 +87,7 @@ class ZeitBridge extends FeedExpander // remove known bad elements foreach ( $article->find( - 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge, .article-heading__container--podcast' + 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge, .article-heading__container--podcast, div[data-paywall], .js-embed-consent' ) as $bad ) { $bad->remove(); From ecf61f6fa777fcaeb6ce0bdf1c82cdcbc89e74a1 Mon Sep 17 00:00:00 2001 From: tillcash Date: Tue, 12 Mar 2024 00:44:10 +0530 Subject: [PATCH 033/423] [DailythanthiBridge] New Bridge (#4006) --- bridges/DailythanthiBridge.php | 96 ++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 bridges/DailythanthiBridge.php diff --git a/bridges/DailythanthiBridge.php b/bridges/DailythanthiBridge.php new file mode 100644 index 00000000..114f42d8 --- /dev/null +++ b/bridges/DailythanthiBridge.php @@ -0,0 +1,96 @@ + [ + 'name' => 'topic', + 'type' => 'list', + 'values' => [ + 'news' => [ + 'tamilnadu' => 'news/state', + 'india' => 'news/india', + 'world' => 'news/world', + 'sirappu-katturaigal' => 'news/sirappukatturaigal', + ], + 'cinema' => [ + 'news' => 'cinema/cinemanews', + ], + 'sports' => [ + 'sports' => 'sports', + 'cricket' => 'sports/cricket', + 'football' => 'sports/football', + 'tennis' => 'sports/tennis', + 'hockey' => 'sports/hockey', + 'other-sports' => 'sports/othersports', + ], + 'devotional' => [ + 'devotional' => 'others/devotional', + 'aalaya-varalaru' => 'aalaya-varalaru', + ], + ], + ], + ], + ]; + + public function getName() + { + $topic = $this->getKey('topic'); + return self::NAME . ($topic ? ' - ' . ucfirst($topic) : ''); + } + + public function collectData() + { + $dom = getSimpleHTMLDOM(self::URI . $this->getInput('topic')); + + foreach ($dom->find('div.ListingNewsWithMEDImage') as $element) { + $slug = $element->find('a', 1); + $title = $element->find('h3', 0); + if (!$slug || !$title) { + continue; + } + + $url = self::URI . $slug->href; + $date = $element->find('span', 1); + $date = $date ? $date->{'data-datestring'} : ''; + + $this->items[] = [ + 'content' => $this->constructContent($url), + 'timestamp' => $date ? $date . 'UTC' : '', + 'title' => $title->plaintext, + 'uid' => $slug->href, + 'uri' => $url, + ]; + } + } + + private function constructContent($url) + { + $dom = getSimpleHTMLDOMCached($url); + + $article = $dom->find('div.details-content-story', 0); + if (!$article) { + return 'Content Not Found'; + } + + // Remove ads + foreach ($article->find('div[id*="_ad"]') as $remove) { + $remove->outertext = ''; + } + + // Correct image tag in $article + foreach ($article->find('h-img') as $img) { + $img->parent->outertext = sprintf('

', $img->src); + } + + $image = $dom->find('div.main-image-caption-container img', 0); + $image = $image ? '

' . $image->outertext . '

' : ''; + + return $image . $article; + } +} From 5b80af978fdd0a7ea566d9a69d669e0b22b0378a Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 12 Mar 2024 19:46:21 +0100 Subject: [PATCH 034/423] docs: improve README (#4009) --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f8d08058..6124a4ea 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ These instructions have been tested on a fresh Debian 12 VM from Digital Ocean ( ```shell timedatectl set-timezone Europe/Oslo -apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl +apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl php-intl # Create a new user account useradd --shell /bin/bash --create-home rss-bridge @@ -167,12 +167,10 @@ Restart fpm and nginx: ```shell # Lint and restart php-fpm -php-fpm8.2 -t -systemctl restart php8.2-fpm +php-fpm8.2 -t && systemctl restart php8.2-fpm # Lint and restart nginx -nginx -t -systemctl restart nginx +nginx -t && systemctl restart nginx ``` ### How to install from Composer From 4bad1c140a25d8ef8577d0fa7b0a60e27a5d7649 Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 12 Mar 2024 23:59:10 +0100 Subject: [PATCH 035/423] fix(reddit): url encoding (#4010) --- bridges/RedditBridge.php | 55 ++++++++++++++++-------------- tests/BridgeImplementationTest.php | 23 ------------- tests/RedditBridgeTest.php | 33 ++++++++++++++++++ 3 files changed, 62 insertions(+), 49 deletions(-) create mode 100644 tests/RedditBridgeTest.php diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 9c72f996..e2f79b11 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -139,36 +139,13 @@ class RedditBridge extends BridgeAbstract break; } - if (!($this->getInput('search') === '')) { - $keywords = $this->getInput('search'); - $keywords = str_replace([',', ' '], '%20', $keywords); - $keywords = $keywords . '%20'; - } else { - $keywords = ''; - } - - if (!empty($this->getInput('f')) && $this->queriedContext == 'single') { - $flair = $this->getInput('f'); - $flair = str_replace(' ', '%20', $flair); - $flair = 'flair%3A%22' . $flair . '%22%20'; - } else { - $flair = ''; - } + $search = $this->getInput('search'); + $flareInput = $this->getInput('f'); foreach ($subreddits as $subreddit) { - $name = trim($subreddit); - $url = self::URI - . '/search.json?q=' - . $keywords - . $flair - . ($user ? 'author%3A' : 'subreddit%3A') - . $name - . '&sort=' - . $this->getInput('d') - . '&include_over_18=on'; - $version = 'v0.0.1'; $useragent = "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"; + $url = self::createUrl($search, $flareInput, $subreddit, $user, $section, $this->queriedContext); $json = getContents($url, ['User-Agent: ' . $useragent]); $parsedJson = Json::decode($json, false); @@ -278,6 +255,32 @@ class RedditBridge extends BridgeAbstract }); } + public static function createUrl($search, $flareInput, $subreddit, bool $user, $section, $queriedContext): string + { + if ($search === '') { + $keywords = ''; + } else { + $keywords = $search; + $keywords = str_replace([',', ' '], ' ', $keywords); + $keywords = $keywords . ' '; + } + + if ($flareInput && $queriedContext == 'single') { + $flair = $flareInput; + $flair = str_replace([',', ' '], ' ', $flair); + $flair = 'flair:"' . $flair . '" '; + } else { + $flair = ''; + } + $name = trim($subreddit); + $query = [ + 'q' => $keywords . $flair . ($user ? 'author:' : 'subreddit:') . $name, + 'sort' => $section, + 'include_over_18' => 'on', + ]; + return 'https://old.reddit.com/search.json?' . http_build_query($query); + } + public function getIcon() { return 'https://www.redditstatic.com/desktop2x/img/favicon/favicon-96x96.png'; diff --git a/tests/BridgeImplementationTest.php b/tests/BridgeImplementationTest.php index d2f74931..dd68934e 100644 --- a/tests/BridgeImplementationTest.php +++ b/tests/BridgeImplementationTest.php @@ -157,29 +157,6 @@ class BridgeImplementationTest extends TestCase } } - /** - * @dataProvider dataBridgesProvider - */ - public function testVisibleMethods($path) - { - $bridgeAbstractMethods = get_class_methods(BridgeAbstract::class); - sort($bridgeAbstractMethods); - $feedExpanderMethods = get_class_methods(FeedExpander::class); - sort($feedExpanderMethods); - - $this->setBridge($path); - - $publicMethods = get_class_methods($this->bridge); - sort($publicMethods); - foreach ($publicMethods as $publicMethod) { - if ($this->bridge instanceof FeedExpander) { - $this->assertContains($publicMethod, $feedExpanderMethods); - } else { - $this->assertContains($publicMethod, $bridgeAbstractMethods); - } - } - } - /** * @dataProvider dataBridgesProvider */ diff --git a/tests/RedditBridgeTest.php b/tests/RedditBridgeTest.php new file mode 100644 index 00000000..17a62e68 --- /dev/null +++ b/tests/RedditBridgeTest.php @@ -0,0 +1,33 @@ +assertSame($expected, $actual); + + // https://old.reddit.com/search.json?q=author:RavenousRandy&sort=hot&include_over_18=on + $expected = 'https://old.reddit.com/search.json?q=author%3ARavenousRandy&sort=hot&include_over_18=on'; + $actual = RedditBridge::createUrl('', '', 'RavenousRandy', true, 'hot', 'user'); + $this->assertSame($expected, $actual); + + // https://old.reddit.com/search.json?q=cats dogs hen flair:"Proxy" subreddit:php&sort=hot&include_over_18=on + $expected = 'https://old.reddit.com/search.json?q=cats+dogs+hen+flair%3A%22Proxy%22+subreddit%3Aphp&sort=hot&include_over_18=on'; + $actual = RedditBridge::createUrl('cats,dogs hen', 'Proxy', 'php', false, 'hot', 'single'); + $this->assertSame($expected, $actual); + + // https://old.reddit.com/search.json?q=cats dogs hen flair:"Proxy Linux Server" subreddit:php&sort=hot&include_over_18=on + $expected = 'https://old.reddit.com/search.json?q=cats+dogs+hen+flair%3A%22Proxy+Linux+Server%22+subreddit%3Aphp&sort=hot&include_over_18=on'; + $actual = RedditBridge::createUrl('cats,dogs hen', 'Proxy,Linux Server', 'php', false, 'hot', 'single'); + $this->assertSame($expected, $actual); + } +} From e6cb5fdc89f2ad4137657cccea79fbdc23bb37cb Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Wed, 13 Mar 2024 23:47:46 +0100 Subject: [PATCH 036/423] [IdealoBridge] Fix Feed items & Feed title customisation (#4013) - Feed items with new price tracking had "Max Price Used" instead of "Max Price New" - Feed Title is now customised with the product name and the Price limits - Fixed logic for saving prices in cache - remove undefined variable notices --- bridges/IdealoBridge.php | 83 +++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 14 deletions(-) diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php index cef2b812..fe13e13d 100644 --- a/bridges/IdealoBridge.php +++ b/bridges/IdealoBridge.php @@ -40,6 +40,47 @@ class IdealoBridge extends BridgeAbstract return 'https://cdn.idealo.com/storage/ids-assets/ico/favicon.ico'; } + /** + * Returns the RSS Feed title when a RSS feed is rendered + * @return string the RSS feed Title + */ + private function getFeedTitle() + { + $cacheDuration = 604800; + $link = $this->getInput('Link'); + $keyTITLE = $link . 'TITLE'; + $product = $this->loadCacheValue($keyTITLE, $cacheDuration); + + // The cache does not contain the title of the bridge, we must get it and save it in the cache + if ($product === null) { + $header = [ + 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15' + ]; + $html = getSimpleHTMLDOM($link, $header); + $product = $html->find('.oopStage-title', 0)->find('span', 0)->plaintext; + $this->saveCacheValue($keyTITLE, $product); + } + + $MaxPriceUsed = $this->getInput('MaxPriceUsed'); + $MaxPriceNew = $this->getInput('MaxPriceNew'); + $titleParts = []; + + $titleParts[] = $product; + + // Add Max Prices to the title + if ($MaxPriceUsed !== null) { + $titleParts[] = 'Max Price Used : ' . $MaxPriceUsed . '€'; + } + if ($MaxPriceNew !== null) { + $titleParts[] = 'Max Price New : ' . $MaxPriceNew . '€'; + } + + $title = implode(' ', $titleParts); + + + return $title . ' - ' . $this::NAME; + } + public function collectData() { // Needs header with user-agent to function properly. @@ -69,12 +110,16 @@ class IdealoBridge extends BridgeAbstract $FirstButton = $html->find('.oopStage-conditionButton-wrapper-text', 0); if ($FirstButton) { $PriceNew = $FirstButton->find('strong', 0)->plaintext; + // Save current price + $this->saveCacheValue($KeyNEW, $PriceNew); } // Second Button is used $SecondButton = $html->find('.oopStage-conditionButton-wrapper-text', 1); if ($SecondButton) { $PriceUsed = $SecondButton->find('strong', 0)->plaintext; + // Save current price + $this->saveCacheValue($KeyUSED, $PriceUsed); } // Only continue if a price has changed @@ -83,16 +128,16 @@ class IdealoBridge extends BridgeAbstract $image = $html->find('.datasheet-cover-image', 0)->src; // Generate Content - if ($PriceNew > 1) { + if (isset($PriceNew) && $PriceNew > 1) { $content = "

Price New:
$PriceNew

"; $content .= "

Price New before:
$OldPriceNew

"; } if ($this->getInput('MaxPriceNew') != '') { - $content .= sprintf('

Max Price Used:
%s,00 €

', $this->getInput('MaxPriceNew')); + $content .= sprintf('

Max Price New:
%s,00 €

', $this->getInput('MaxPriceNew')); } - if ($PriceUsed > 1) { + if (isset($PriceUsed) && $PriceUsed > 1) { $content .= "

Price Used:
$PriceUsed

"; $content .= "

Price Used before:
$OldPriceUsed

"; } @@ -110,8 +155,8 @@ class IdealoBridge extends BridgeAbstract // Currently under Max new price if ($this->getInput('MaxPriceNew') != '') { - if ($PriceNew < $this->getInput('MaxPriceNew')) { - $title = sprintf($Pricealarm, 'Used', $PriceNew, $Productname, $now); + if (isset($PriceNew) && $PriceNew < $this->getInput('MaxPriceNew')) { + $title = sprintf($Pricealarm, 'New', $PriceNew, $Productname, $now); $item = [ 'title' => $title, 'uri' => $link, @@ -124,7 +169,7 @@ class IdealoBridge extends BridgeAbstract // Currently under Max used price if ($this->getInput('MaxPriceUsed') != '') { - if ($PriceUsed < $this->getInput('MaxPriceUsed')) { + if (isset($PriceUsed) && $PriceUsed < $this->getInput('MaxPriceUsed')) { $title = sprintf($Pricealarm, 'Used', $PriceUsed, $Productname, $now); $item = [ 'title' => $title, @@ -143,23 +188,23 @@ class IdealoBridge extends BridgeAbstract (!$this->getInput('ExcludeNew') && $PriceNew != $OldPriceNew ) || (!$this->getInput('ExcludeUsed') && $PriceUsed != $OldPriceUsed ) ) { - $title .= 'Priceupdate! '; + $title = 'Priceupdate! '; if (!$this->getInput('ExcludeNew')) { - if ($PriceNew < $OldPriceNew) { + if (isset($PriceNew) && $PriceNew < $OldPriceNew) { $title .= 'NEW:⬇ '; // Arrow Down Emoji } - if ($PriceNew > $OldPriceNew) { + if (isset($PriceNew) && $PriceNew > $OldPriceNew) { $title .= 'NEW:⬆ '; // Arrow Up Emoji } } if (!$this->getInput('ExcludeUsed')) { - if ($PriceUsed < $OldPriceUsed) { + if (isset($PriceUsed) && $PriceUsed < $OldPriceUsed) { $title .= 'USED:⬇ '; // Arrow Down Emoji } - if ($PriceUsed > $OldPriceUsed) { + if (isset($PriceUsed) && $PriceUsed > $OldPriceUsed) { $title .= 'USED:⬆ '; // Arrow Up Emoji } } @@ -177,9 +222,19 @@ class IdealoBridge extends BridgeAbstract } } } + } - // Save current price - $this->saveCacheValue($KeyNEW, $PriceNew); - $this->saveCacheValue($KeyUSED, $PriceUsed); + /** + * Returns the RSS Feed title according to the parameters + * @return string the RSS feed Tile + */ + public function getName() + { + switch ($this->queriedContext) { + case '0': + return $this->getFeedTitle(); + default: + return parent::getName(); + } } } From 36147a082d33faeef6e42fa103a3283cc5cf5dc3 Mon Sep 17 00:00:00 2001 From: Tim-Florian Feulner <50834839+R3dError@users.noreply.github.com> Date: Fri, 15 Mar 2024 19:20:04 +0100 Subject: [PATCH 037/423] Fix NACSouthGermanyMediaLibraryBridge for new website layout (#4014) --- bridges/NACSouthGermanyMediaLibraryBridge.php | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bridges/NACSouthGermanyMediaLibraryBridge.php b/bridges/NACSouthGermanyMediaLibraryBridge.php index fff6c554..030ded8f 100644 --- a/bridges/NACSouthGermanyMediaLibraryBridge.php +++ b/bridges/NACSouthGermanyMediaLibraryBridge.php @@ -31,7 +31,7 @@ class NACSouthGermanyMediaLibraryBridge extends BridgeAbstract public function getIcon() { - return 'https://www.nak-stuttgart.de/static/themes/nak_sued/images/nak-logo.png'; + return 'https://nak-sued.de/static/themes/sued/images/logo.png'; } private static function parseTimestamp($title) @@ -66,9 +66,12 @@ class NACSouthGermanyMediaLibraryBridge extends BridgeAbstract private static function collectDataForBayern2($parent, $item) { # Find link - $playerDom = getSimpleHTMLDOMCached(self::BASE_URI . $parent->find('a', 0)->href); - $sourceURI = $playerDom->find('source', 0)->src; - $item['enclosures'] = [self::BASE_URI . $sourceURI]; + $relativeURICode = $parent->find('a', 0)->onclick; + if (preg_match('/window\.open\(\'([^\']*)\'/', $relativeURICode, $matches)) { + $playerDom = getSimpleHTMLDOMCached(self::BASE_URI . $matches[1]); + $sourceURI = $playerDom->find('source', 0)->src; + $item['enclosures'] = [self::BASE_URI . $sourceURI]; + } # Add time to timestamp $item['timestamp'] .= ' 06:45'; @@ -78,14 +81,14 @@ class NACSouthGermanyMediaLibraryBridge extends BridgeAbstract private function collectDataInList($pageURI, $customizeItemCall) { - $page = getSimpleHTMLDOM(self::BASE_URI . $pageURI); + $page = getSimpleHTMLDOM($pageURI); - foreach ($page->find('div.grids') as $parent) { + foreach ($page->find('div.flex-columns.entry') as $parent) { # Find title $title = $parent->find('h2', 0)->plaintext; # Find content - $contentBlock = $parent->find('ul.contentlist', 0); + $contentBlock = $parent->find('ul', 0); $content = ''; foreach ($contentBlock->find('li') as $li) { $content .= '

' . $li->plaintext . '

'; @@ -103,7 +106,7 @@ class NACSouthGermanyMediaLibraryBridge extends BridgeAbstract private function collectDataFromAllPages($rootURI, $customizeItemCall) { $rootPage = getSimpleHTMLDOM($rootURI); - $pages = $rootPage->find('div#tabmenu', 0); + $pages = $rootPage->find('div.flex-columns.inner_filter', 0); foreach ($pages->find('a') as $page) { self::collectDataInList($page->href, [$this, $customizeItemCall]); } From a61524bf776fb29aaf30451c972f1ddb90920fdf Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Sun, 17 Mar 2024 19:02:51 +0100 Subject: [PATCH 038/423] Update RedditBridge.php (#4019) prevent error htmlspecialchars_decode(): Passing null to parameter #1 --- bridges/RedditBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index e2f79b11..fbc6f678 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -189,7 +189,7 @@ class RedditBridge extends BridgeAbstract // Comment $item['content'] = htmlspecialchars_decode($data->body_html); - } elseif ($data->is_self) { + } elseif ($data->is_self && isset($data->selftext_html)) { // Text post $item['content'] = htmlspecialchars_decode($data->selftext_html); From 58e2b56d40e0cae1cd3689f8c946650e5caba4cf Mon Sep 17 00:00:00 2001 From: Patrick Date: Sun, 17 Mar 2024 19:03:09 +0100 Subject: [PATCH 039/423] Adjustment to new website layout (#4020) --- bridges/JohannesBlickBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/JohannesBlickBridge.php b/bridges/JohannesBlickBridge.php index 6c00feca..72583a53 100644 --- a/bridges/JohannesBlickBridge.php +++ b/bridges/JohannesBlickBridge.php @@ -13,7 +13,7 @@ class JohannesBlickBridge extends BridgeAbstract or returnServerError('Could not request: ' . self::URI); $html = defaultLinkTo($html, self::URI); - foreach ($html->find('td > a') as $index => $a) { + foreach ($html->find('ul[class=easyfolderlisting] > li > a') as $index => $a) { $item = []; // Create an empty item $articlePath = $a->href; $item['title'] = $a->innertext; From 3ed193eee2e873496ca3635561c61e0e8ba49edd Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 22 Mar 2024 09:44:42 +0100 Subject: [PATCH 040/423] [IdealoBridge] Update Bridge Meta data & (#4022) The bridge meta data has been updated to reflect that the bridge works for other international version of Idealo. The Price trend is displayed on every price in the the Feed element content. The same function is now used to show the price trend in the Feed element title, to remove some duplicate code.. --- bridges/IdealoBridge.php | 46 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php index fe13e13d..4eb66dcb 100644 --- a/bridges/IdealoBridge.php +++ b/bridges/IdealoBridge.php @@ -2,15 +2,15 @@ class IdealoBridge extends BridgeAbstract { - const NAME = 'Idealo.de Bridge'; + const NAME = 'idealo.de / idealo.fr / idealo.es Bridge'; const URI = 'https://www.idealo.de'; - const DESCRIPTION = 'Tracks the price for a product on idealo.de. Pricealarm if specific price is set'; + const DESCRIPTION = 'Tracks the price for a product on idealo.de / idealo.fr / idealo.es. Pricealarm if specific price is set'; const MAINTAINER = 'SebLaus'; const CACHE_TIMEOUT = 60 * 30; // 30 min const PARAMETERS = [ [ 'Link' => [ - 'name' => 'Idealo.de Link to productpage', + 'name' => 'idealo.de / idealo.fr / idealo.es Link to productpage', 'required' => true, 'exampleValue' => 'https://www.idealo.de/preisvergleich/OffersOfProduct/202007367_-s7-pro-ultra-roborock.html' ], @@ -81,6 +81,25 @@ class IdealoBridge extends BridgeAbstract return $title . ' - ' . $this::NAME; } + + /** + * Returns the Price Trend emoji + * @return string the Price Trend Emoji + */ + private function getPriceTrend($NewPrice, $OldPrice) + { + // In case there is no old PRice, then show no trend + if ($OldPrice === null) { + $trend = ''; + } else if ($NewPrice > $OldPrice) { + $trend = '↗'; + } else if ($NewPrice == $OldPrice) { + $trend = '➡'; + } else if ($NewPrice < $OldPrice) { + $trend = '↘'; + } + return $trend; + } public function collectData() { // Needs header with user-agent to function properly. @@ -127,9 +146,11 @@ class IdealoBridge extends BridgeAbstract // Get Product Image $image = $html->find('.datasheet-cover-image', 0)->src; + $content = ''; + // Generate Content if (isset($PriceNew) && $PriceNew > 1) { - $content = "

Price New:
$PriceNew

"; + $content .= sprintf('

Price New:
%s %s

', $PriceNew, $this->getPriceTrend($PriceNew, $OldPriceNew)); $content .= "

Price New before:
$OldPriceNew

"; } @@ -138,7 +159,7 @@ class IdealoBridge extends BridgeAbstract } if (isset($PriceUsed) && $PriceUsed > 1) { - $content .= "

Price Used:
$PriceUsed

"; + $content .= sprintf('

Price Used:
%s %s

', $PriceUsed, $this->getPriceTrend($PriceUsed, $OldPriceUsed)); $content .= "

Price Used before:
$OldPriceUsed

"; } @@ -191,22 +212,11 @@ class IdealoBridge extends BridgeAbstract $title = 'Priceupdate! '; if (!$this->getInput('ExcludeNew')) { - if (isset($PriceNew) && $PriceNew < $OldPriceNew) { - $title .= 'NEW:⬇ '; // Arrow Down Emoji - } - if (isset($PriceNew) && $PriceNew > $OldPriceNew) { - $title .= 'NEW:⬆ '; // Arrow Up Emoji - } + $title .= 'NEW' . $this->getPriceTrend($PriceNew, $OldPriceNew) . ' '; } - if (!$this->getInput('ExcludeUsed')) { - if (isset($PriceUsed) && $PriceUsed < $OldPriceUsed) { - $title .= 'USED:⬇ '; // Arrow Down Emoji - } - if (isset($PriceUsed) && $PriceUsed > $OldPriceUsed) { - $title .= 'USED:⬆ '; // Arrow Up Emoji - } + $title .= 'USED' . $this->getPriceTrend($PriceUsed, $OldPriceUsed) . ' '; } $title .= $Productname; $title .= ' '; From 2aace6c898c2d0f473b0a2071bf3e2d7e1f8e4d4 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Fri, 22 Mar 2024 21:01:16 +0100 Subject: [PATCH 041/423] Added Bridge for Anisearch.de (#4023) * Create AnisearchBridge.php * Update AnisearchBridge.php * Update AnisearchBridge.php --- bridges/AnisearchBridge.php | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 bridges/AnisearchBridge.php diff --git a/bridges/AnisearchBridge.php b/bridges/AnisearchBridge.php new file mode 100644 index 00000000..639d143f --- /dev/null +++ b/bridges/AnisearchBridge.php @@ -0,0 +1,54 @@ + [ + 'name' => 'Dub', + 'type' => 'list', + 'values' => [ + 'DE' + => 'https://www.anisearch.de/anime/index/page-1?char=all&synchro=de&sort=date&order=desc&view=4', + 'EN' + => 'https://www.anisearch.de/anime/index/page-1?char=all&synchro=en&sort=date&order=desc&view=4', + 'JP' + => 'https://www.anisearch.de/anime/index/page-1?char=all&synchro=ja&sort=date&order=desc&view=4' + ] + ] + ]]; + + public function collectData() + { + $baseurl = 'https://www.anisearch.de/'; + $limit = 10; + $dom = getSimpleHTMLDOM($this->getInput('category')); + foreach ($dom->find('li.btype0') as $key => $li) { + if ($key > $limit) { + break; + } + + $a = $li->find('a', 0); + $title = $a->find('span.title', 0); + $url = $baseurl . $a->href; + + //get article + $domarticle = getSimpleHTMLDOM($url); + $content = $domarticle->find('div.details-text', 0); + + //get header-image and set absolute src + $headerimage = $domarticle->find('img#details-cover', 0); + $src = $headerimage->src; + + $this->items[] = [ + 'title' => $title->plaintext, + 'uri' => $url, + 'content' => $headerimage . '
' . $content + ]; + } + } +} From fee5e269d0763aade289677d6685ebd8bd45c542 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Sun, 24 Mar 2024 16:38:51 +0100 Subject: [PATCH 042/423] Update CaschyBridge.php (#4027) without removing the video-container-div the embedded youtube videos work again --- bridges/CaschyBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/CaschyBridge.php b/bridges/CaschyBridge.php index 0e3a07bc..c25cdb08 100644 --- a/bridges/CaschyBridge.php +++ b/bridges/CaschyBridge.php @@ -54,7 +54,7 @@ class CaschyBridge extends FeedExpander { // remove unwanted stuff foreach ( - $article->find('div.video-container, div.aawp, p.aawp-disclaimer, iframe.wp-embedded-content, + $article->find('div.aawp, p.aawp-disclaimer, iframe.wp-embedded-content, div.wp-embed, p.wp-caption-text, script') as $element ) { $element->remove(); From 0c2099a8529faad54d848dd73cfc6bd839abb5ba Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Thu, 28 Mar 2024 19:41:56 +0100 Subject: [PATCH 043/423] [GolemBridge] fixed embedded youtube videos (#4033) * [GolemBridge] fixed embedded youtube videos embedded youtube-videos can be played directly from feed now * Update GolemBridge.php * Update GolemBridge.php * Update GolemBridge.php * Update GolemBridge.php --- bridges/GolemBridge.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index 599d713a..216e913f 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -106,10 +106,23 @@ class GolemBridge extends FeedExpander $article = $page->find('article', 0); + //built youtube iframes + foreach ($article->find('.embedcontent') as &$embedcontent) { + $ytscript = $embedcontent->find('script', 0); + if (preg_match('/www.youtube.com.*?\"/', $ytscript->innertext, $link)) { + $link = 'https://' . str_replace('\\', '', $link[0]); + $embedcontent->innertext .= <<'; + EOT; + } + } + // delete known bad elements foreach ( $article->find('div[id*="adtile"], #job-market, #seminars, iframe, - div.gbox_affiliate, div.toc, .embedcontent, script') as $bad + div.gbox_affiliate, div.toc') as $bad ) { $bad->remove(); } @@ -129,7 +142,7 @@ class GolemBridge extends FeedExpander $img->src = $img->getAttribute('data-src-full'); } - foreach ($content->find('p, h1, h2, h3, img[src*="."]') as $element) { + foreach ($content->find('p, h1, h2, h3, img[src*="."], iframe') as $element) { $item .= $element; } From e251e358ff757023f0e0863d47d1781c5196964c Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Thu, 28 Mar 2024 19:42:41 +0100 Subject: [PATCH 044/423] [HeiseBridge] fix for embedded youtube-videos (#4034) * [HeiseBridge] fix for embbedded youtube-videos with this the embedded youtube videos will work in the feed * Update HeiseBridge.php * Update HeiseBridge.php --- bridges/HeiseBridge.php | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index 504bcfb5..ab40e6d9 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -160,7 +160,7 @@ class HeiseBridge extends FeedExpander $article = defaultLinkTo($article, $item['uri']); // remove unwanted stuff - foreach ($article->find('figure.branding, a-ad, div.ho-text, a-img, .opt-in__content-container, .a-toc__list, a-collapse') as $element) { + foreach ($article->find('figure.branding, a-ad, div.ho-text, a-img, .a-toc__list, a-collapse, .opt-in__description, .opt-in__footnote') as $element) { $element->remove(); } // reload html, as remove() is buggy @@ -179,6 +179,30 @@ class HeiseBridge extends FeedExpander } } + //fix for embbedded youtube-videos + $oldlink = ''; + foreach ($article->find('div.video__yt-container') as &$ytvideo) { + if (preg_match('/www.youtube.*?\"/', $ytvideo->innertext, $link) && $link[0] != $oldlink) { + //save link to prevent duplicates + $oldlink = $link[0]; + $ytiframe = << + EOT; + //check if video is in header or article for correct possitioning + if (strpos($header->innertext, $link[0])) { + $item['content'] .= $ytiframe; + } else { + $ytvideo->innertext .= $ytiframe; + $reloadneeded = 1; + } + } + } + if (isset($reloadneeded)) { + $article = str_get_html($article->outertext); + } + $categories = $article->find('.article-footer__topics ul.topics li.topics__item a-topic a'); foreach ($categories as $category) { $item['categories'][] = trim($category->plaintext); From db984d8a8b98a985fe9a272794fa20081908ad13 Mon Sep 17 00:00:00 2001 From: July Date: Thu, 28 Mar 2024 14:43:17 -0400 Subject: [PATCH 045/423] AO3Bridge: move tags to categories and remove duplicate fic summary (#4031) * AO3Bridge: move tags to categories and remove duplicate fic summary * [AO3Bridge] Fix tag html entity encoding --- bridges/AO3Bridge.php | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php index 32bbb0a2..85f0f9f8 100644 --- a/bridges/AO3Bridge.php +++ b/bridges/AO3Bridge.php @@ -91,12 +91,26 @@ class AO3Bridge extends BridgeAbstract continue; // discard deleted works } $item['title'] = $title->plaintext; - $item['content'] = $element; $item['uri'] = $title->href; $strdate = $element->find('div p.datetime', 0)->plaintext; $item['timestamp'] = strtotime($strdate); + // detach from rest of page because remove() is buggy + $element = str_get_html($element->outertext()); + $tags = $element->find('ul.required-tags', 0); + foreach ($tags->childNodes() as $tag) { + $item['categories'][] = html_entity_decode($tag->plaintext); + } + $tags->remove(); + $tags = $element->find('ul.tags', 0); + foreach ($tags->childNodes() as $tag) { + $item['categories'][] = html_entity_decode($tag->plaintext); + } + $tags->remove(); + + $item['content'] = implode('', $element->childNodes()); + $chapters = $element->find('dl dd.chapters', 0); // bookmarked series and external works do not have a chapters count $chapters = (isset($chapters) ? $chapters->plaintext : 0); @@ -123,6 +137,10 @@ class AO3Bridge extends BridgeAbstract $response = $httpClient->request($url, $agent); $html = \str_get_html($response->getBody()); $html = defaultLinkTo($html, self::URI); + // remove duplicate fic summary + if ($ficsum = $html->find('#workskin > .preface > .summary', 0)) { + $ficsum->remove(); + } $item['content'] .= $html->find('#workskin', 0); } From be445759b631ea1ec954d70e8e1f30cfb9271f2b Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Thu, 28 Mar 2024 19:44:27 +0100 Subject: [PATCH 046/423] [PepperBridgeAbstract,DealabsBridge,HotUKDealsBridge,MydealsBridge] Move as much as possible to JSON (#4032) As the website use more and more JSON, and JSON is a machine readable format, I migrated as much as possible to the JSON. This simplifies the Abstract class a lot, and the Bridge classes need less language specifi strings. --- bridges/DealabsBridge.php | 42 +---- bridges/HotUKDealsBridge.php | 51 +----- bridges/MydealsBridge.php | 50 +----- bridges/PepperBridgeAbstract.php | 259 ++++++++----------------------- 4 files changed, 69 insertions(+), 333 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index c65f0c75..62d854f6 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -1915,9 +1915,6 @@ class DealabsBridge extends PepperBridgeAbstract 'thread-error' => 'Impossible de déterminer l\'ID de la discussion. Vérifiez l\'URL que vous avez entré', 'no-results' => 'Aucun résultat', 'currency' => '€', - 'relative-date-indicator' => [ - 'il y a', - ], 'price' => 'Prix', 'shipping' => 'Livraison', 'origin' => 'Origine', @@ -1925,42 +1922,7 @@ class DealabsBridge extends PepperBridgeAbstract 'title-keyword' => 'Recherche', 'title-group' => 'Groupe', 'title-talk' => 'Surveillance Discussion', - 'local-months' => [ - 'janvier', - 'février', - 'mars', - 'avril', - 'mai', - 'juin', - 'juillet', - 'août', - 'septembre', - 'octobre', - 'novembre', - 'décembre' - ], - 'local-time-relative' => [ - 'il y a ', - 'min', - 'h', - 'jour', - 'jours', - 'mois', - 'ans', - 'et ' - ], - 'date-prefixes' => [ - 'Actualisé ', - ], - 'relative-date-alt-prefixes' => [ - 'Actualisé ', - ], - 'relative-date-ignore-suffix' => [ - ], - - 'localdeal' => [ - 'Local', - 'Pays d\'expédition' - ], + 'deal-type' => 'Type de deal', + 'localdeal' => 'Deal Local', ]; } diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index 1f059123..b631db73 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3279,9 +3279,6 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'thread-error' => 'Unable to determine the thread ID. Check the URL you entered', 'no-results' => 'no results', 'currency' => '£', - 'relative-date-indicator' => [ - 'ago', - ], 'price' => 'Price', 'shipping' => 'Shipping', 'origin' => 'Origin', @@ -3289,51 +3286,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'title-keyword' => 'Search', 'title-group' => 'Group', 'title-talk' => 'Discussion Monitoring', - 'local-months' => [ - 'Jan', - 'Feb', - 'Mar', - 'Apr', - 'May', - 'Jun', - 'Jul', - 'Aug', - 'Sep', - 'Occ', - 'Nov', - 'Dec', - 'st', - 'nd', - 'rd', - 'th' - ], - 'local-time-relative' => [ - 'Posted ', - 'm', - 'h,', - 'day', - 'days', - 'month', - 'year', - 'and ' - ], - 'date-prefixes' => [ - 'Posted ', - 'Found ', - 'Refreshed ', - 'Made hot ' - ], - 'relative-date-alt-prefixes' => [ - 'Made hot ', - 'Refreshed ', - 'Last updated ' - ], - 'relative-date-ignore-suffix' => [ - '/by.*$/' - ], - 'localdeal' => [ - 'Local', - 'Expires' - ] + 'deal-type' => 'Deal Type', + 'localdeal' => 'Local deal', ]; } diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 08e32a0c..41bae46c 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2026,10 +2026,6 @@ class MydealsBridge extends PepperBridgeAbstract 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', 'no-results' => 'keine Ergebnisse', 'currency' => '€', - 'relative-date-indicator' => [ - 'vor', - 'seit' - ], 'price' => 'Preis', 'shipping' => 'Versand', 'origin' => 'Ursprung', @@ -2037,49 +2033,7 @@ class MydealsBridge extends PepperBridgeAbstract 'title-keyword' => 'Suche', 'title-group' => 'Gruppe', 'title-talk' => 'Überwachung Diskussion', - 'local-months' => [ - 'Jan', - 'Feb', - 'Mär', - 'Apr', - 'Mai', - 'Jun', - 'Jul', - 'Aug', - 'Sep', - 'Okt', - 'Nov', - 'Dez', - '.' - ], - 'local-time-relative' => [ - 'eingestellt vor ', - 'm', - 'h,', - 'day', - 'days', - 'month', - 'year', - 'and ' - ], - 'date-prefixes' => [ - 'eingestellt am ', - 'lokal ', - 'aktualisiert ', - ], - 'relative-date-alt-prefixes' => [ - 'aktualisiert vor ', - 'kommentiert vor ', - 'eingestellt vor ', - 'heiß seit ', - 'vor ' - ], - 'relative-date-ignore-suffix' => [ - '/von.*$/' - ], - 'localdeal' => [ - 'Lokal ', - 'Läuft bis ' - ] + 'deal-type' => 'Angebotsart', + 'localdeal' => 'Lokales Angebot', ]; } diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index d0e15238..7b40ea1c 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -46,35 +46,6 @@ class PepperBridgeAbstract extends BridgeAbstract $html = getSimpleHTMLDOM($url); $list = $html->find('article[id][class*=thread--deal]]'); - // Deal Image Link CSS Selector - $selectorImageLink = implode( - ' ', /* Notice this is a space! */ - [ - 'cept-thread-image-link', - 'imgFrame', - 'imgFrame--noBorder', - 'thread-listImgCell', - ] - ); - - // Deal Link CSS Selector - $selectorLink = implode( - ' ', /* Notice this is a space! */ - [ - 'cept-tt', - 'thread-link', - 'linkPlain', - ] - ); - - // Deal Hotness CSS Selector - $selectorHot = implode( - ' ', /* Notice this is a space! */ - [ - 'vote-box' - ] - ); - // Deal Description CSS Selector $selectorDescription = implode( ' ', /* Notice this is a space! */ @@ -83,65 +54,39 @@ class PepperBridgeAbstract extends BridgeAbstract ] ); - // Deal Date CSS Selector - $selectorDate = implode( - ' ', /* Notice this is a space! */ - [ - 'size--all-s', - 'flex', - 'boxAlign-jc--all-fe' - ] - ); - // If there is no results, we don't parse the content because it display some random deals $noresult = $html->find('h3[class*=text--b]', 0); if ($noresult != null && strpos($noresult->plaintext, $this->i8n('no-results')) !== false) { $this->items = []; } else { foreach ($list as $deal) { - $item = []; - $item['uri'] = $this->getDealURI($deal); - $item['title'] = $this->getTitle($deal); - $item['author'] = $deal->find('span.thread-username', 0)->plaintext; - // Get the JSON Data stored as vue $jsonDealData = $this->getDealJsonData($deal); + $dealMeta = Json::decode($deal->find('div[class=threadGrid-headerMeta]', 0)->find('div[class=js-vue2]', 1)->getAttribute('data-vue2')); + + $item = []; + $item['uri'] = $this->getDealURI($jsonDealData); + $item['title'] = $this->getTitle($jsonDealData); + $item['author'] = $this->getDealAuthor($jsonDealData); $item['content'] = '
' . $this->getImage($deal) . '' - . $this->getHTMLTitle($item) + . $this->getHTMLTitle($jsonDealData) . $this->getPrice($jsonDealData) . $this->getDiscount($jsonDealData) - . $this->getShipsFrom($deal) - . $this->getShippingCost($deal) + . $this->getShipsFrom($dealMeta) + . $this->getShippingCost($jsonDealData) . $this->getSource($jsonDealData) + . $this->getDealLocation($dealMeta) . $deal->find('div[class*=' . $selectorDescription . ']', 0)->innertext . '' . $this->getTemperature($jsonDealData) . '
'; - // Check if a clock icon is displayed on the deal - $clocks = $deal->find('svg[class*=icon--clock]'); - if ($clocks !== null && count($clocks) > 0) { - // Get the last clock, corresponding to the deal posting date - $clock = end($clocks); - - // Find the text corresponding to the clock - $spanDateDiv = $clock->next_sibling(); - $itemDate = $spanDateDiv->plaintext; - // In some case of a Local deal, there is no date, but we can use - // this case for other reason (like date not in the last field) - if ($this->contains($itemDate, $this->i8n('localdeal'))) { - $item['timestamp'] = time(); - } elseif ($this->contains($itemDate, $this->i8n('relative-date-indicator'))) { - $item['timestamp'] = $this->relativeDateToTimestamp($itemDate); - } else { - $item['timestamp'] = $this->parseDate($itemDate); - } - } + $item['timestamp'] = $this->getPublishedDate($jsonDealData); $this->items[] = $item; } } @@ -284,22 +229,31 @@ HEREDOC; } } + /** + * Get the Publish Date from a Deal if it exists + * @return integer Timestamp of the published date of the deal + */ + private function getPublishedDate($jsonDealData) + { + return $jsonDealData['props']['thread']['publishedAt']; + } + + /** + * Get the Deal Author from a Deal if it exists + * @return String Author of the deal + */ + private function getDealAuthor($jsonDealData) + { + return $jsonDealData['props']['thread']['user']['username']; + } + /** * Get the Title from a Deal if it exists * @return string String of the deal title */ - private function getTitle($deal) + private function getTitle($jsonDealData) { - $titleRoot = $deal->find('div[class*=threadGrid-title]', 0); - $titleA = $titleRoot->find('a[class*=thread-link]', 0); - $titleFirstChild = $titleRoot->first_child(); - if ($titleA !== null) { - $title = $titleA->plaintext; - } else { - // In some case, expired deals have a different format - $title = $titleRoot->find('span', 0)->plaintext; - } - + $title = $jsonDealData['props']['thread']['title']; return $title; } @@ -318,14 +272,10 @@ HEREDOC; * Get the HTML Title code from an item * @return string String of the deal title */ - private function getHTMLTitle($item) + private function getHTMLTitle($jsonDealData) { - if ($item['uri'] == '') { - $html = '

' . $item['title'] . '

'; - } else { - $html = '

' - . $item['title'] . '

'; - } + $html = '

' + . $this->getTitle($jsonDealData) . '

'; return $html; } @@ -334,10 +284,11 @@ HEREDOC; * Get the URI from a Deal if it exists * @return string String of the deal URI */ - private function getDealURI($deal) + private function getDealURI($jsonDealData) { - $dealId = $deal->attr['id']; - $uri = $this->i8n('bridge-uri') . $this->i8n('uri-deal') . str_replace('_', '-', $dealId); + $dealSlug = $jsonDealData['props']['thread']['titleSlug']; + $dealId = $jsonDealData['props']['thread']['threadId']; + $uri = $this->i8n('bridge-uri') . $this->i8n('uri-deal') . $dealSlug . '-' . $dealId; return $uri; } @@ -345,18 +296,14 @@ HEREDOC; * Get the Shipping costs from a Deal if it exists * @return string String of the deal shipping Cost */ - private function getShippingCost($deal) + private function getShippingCost($jsonDealData) { - if ($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0) != null) { - if ($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1) != null) { + $isFree = $jsonDealData['props']['thread']['shipping']['isFree']; + $price = $jsonDealData['props']['thread']['shipping']['price']; + if ($isFree !== null) { return '
' . $this->i8n('shipping') . ' : ' - . strip_tags($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1)->innertext) + . $price . ' ' . $this->i8n('currency') . '
'; - } else { - return '
' . $this->i8n('shipping') . ' : ' - . strip_tags($deal->find('span[class*=text--color-greyShade flex--inline]', 0)->innertext) - . '
'; - } } else { return ''; } @@ -422,6 +369,25 @@ HEREDOC; } } + /** + * Get the Deal location if it exists + * @return string String of the deal location + */ + private function getDealLocation($dealMeta) + { + $ribbons = $dealMeta['props']['metaRibbons']; + $isLocal = false; + foreach ($ribbons as $ribbon) { + $isLocal |= ($ribbon['type'] == 'local'); + } + if ($isLocal) { + $content = '
' . $this->i8n('deal-type') . ' : ' . $this->i8n('localdeal') . '
'; + } else { + $content = ''; + } + return $content; + } + /** * Get the Picture URL from a Deal if it exists * @return string String of the deal Picture URL @@ -437,9 +403,8 @@ HEREDOC; * Get the originating country from a Deal if it exists * @return string String of the deal originating country */ - private function getShipsFrom($deal) + private function getShipsFrom($dealMeta) { - $dealMeta = Json::decode($deal->find('div[class=threadGrid-headerMeta]', 0)->find('div[class=js-vue2]', 1)->getAttribute('data-vue2')); $metas = $dealMeta['props']['metaRibbons']; $shipsFrom = null; foreach ($metas as $meta) { @@ -453,104 +418,6 @@ HEREDOC; return ''; } - /** - * Transforms a local date into a timestamp - * @return int timestamp of the input date - */ - private function parseDate($string) - { - $month_local = $this->i8n('local-months'); - $month_en = [ - 'January', - 'February', - 'March', - 'April', - 'May', - 'June', - 'July', - 'August', - 'September', - 'October', - 'November', - 'December' - ]; - - // A date can be prfixed with some words, we remove theme - $string = $this->removeDatePrefixes($string); - // We translate the local months name in the english one - $date_str = trim(str_replace($month_local, $month_en, $string)); - - // If the date does not contain any year, we add the current year - if (!preg_match('/[0-9]{4}/', $string)) { - $date_str .= ' ' . date('Y'); - } - - // Add the Hour and minutes - $date_str .= ' 00:00'; - $date = DateTime::createFromFormat('j F Y H:i', $date_str); - // In some case, the date is not recognized : as a workaround the actual date is taken - if ($date === false) { - $date = new DateTime(); - } - return $date->getTimestamp(); - } - - /** - * Remove the prefix of a date if it has one - * @return the date without prefiux - */ - private function removeDatePrefixes($string) - { - $string = str_replace($this->i8n('date-prefixes'), [], $string); - return $string; - } - - /** - * Remove the suffix of a relative date if it has one - * @return the relative date without suffixes - */ - private function removeRelativeDateSuffixes($string) - { - if (count($this->i8n('relative-date-ignore-suffix')) > 0) { - $string = preg_replace($this->i8n('relative-date-ignore-suffix'), '', $string); - } - return $string; - } - - /** - * Transforms a relative local date into a timestamp - * @return int timestamp of the input date - */ - private function relativeDateToTimestamp($str) - { - $date = new DateTime(); - - // The minimal amount of time substracted is a minute : the seconds in the resulting date would be related to the execution time of the script. - // This make no sense, so we set the seconds manually to "00". - $date->setTime($date->format('H'), $date->format('i'), 0); - - // In case of update date, replace it by the regular relative date first word - $str = str_replace($this->i8n('relative-date-alt-prefixes'), $this->i8n('local-time-relative')[0], $str); - - $str = $this->removeRelativeDateSuffixes($str); - - $search = $this->i8n('local-time-relative'); - - $replace = [ - '-', - 'minute', - 'hour', - 'day', - 'month', - 'year', - '' - ]; - $date->modify(str_replace($search, $replace, $str)); - - - return $date->getTimestamp(); - } - /** * Returns the RSS Feed title according to the parameters * @return string the RSS feed Tiyle From e0be3662589fbc86b944906713cd6ebfaaf43ccc Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:37:43 +0100 Subject: [PATCH 047/423] Update AnisearchBridge.php (#4025) * Update AnisearchBridge.php added youtube trailer * made trailers optional and reduced scraping to 5 articles if selected * Update AnisearchBridge.php --- bridges/AnisearchBridge.php | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/bridges/AnisearchBridge.php b/bridges/AnisearchBridge.php index 639d143f..d5aad1c9 100644 --- a/bridges/AnisearchBridge.php +++ b/bridges/AnisearchBridge.php @@ -19,16 +19,29 @@ class AnisearchBridge extends BridgeAbstract 'JP' => 'https://www.anisearch.de/anime/index/page-1?char=all&synchro=ja&sort=date&order=desc&view=4' ] + ], + 'trailers' => [ + 'name' => 'Trailers', + 'type' => 'checkbox', + 'title' => 'Will include trailes', + 'defaultValue' => false ] ]]; public function collectData() { $baseurl = 'https://www.anisearch.de/'; + $trailers = false; + $trailers = $this->getInput('trailers'); $limit = 10; + if ($trailers) { + $limit = 5; + } + $dom = getSimpleHTMLDOM($this->getInput('category')); + foreach ($dom->find('li.btype0') as $key => $li) { - if ($key > $limit) { + if ($key >= $limit) { break; } @@ -44,10 +57,29 @@ class AnisearchBridge extends BridgeAbstract $headerimage = $domarticle->find('img#details-cover', 0); $src = $headerimage->src; + foreach ($content->find('.hidden') as $element) { + $element->remove(); + } + + //get trailer + $ytlink = ''; + if ($trailers) { + $trailerlink = $domarticle->find('section#trailers > div > div.swiper > ul.swiper-wrapper > li.swiper-slide > a', 0); + if (isset($trailerlink)) { + $trailersite = getSimpleHTMLDOM($baseurl . $trailerlink->href); + $trailer = $trailersite->find('div#player > iframe', 0); + $ytlink = <<'; + EOT; + } + } + $this->items[] = [ 'title' => $title->plaintext, 'uri' => $url, - 'content' => $headerimage . '
' . $content + 'content' => $headerimage . '
' . $content . $ytlink ]; } } From 24e429969f87574d5808098a27248f0e212ce86e Mon Sep 17 00:00:00 2001 From: Quentin de Longraye Date: Sat, 30 Mar 2024 16:11:57 +0100 Subject: [PATCH 048/423] specify system section for enabling bridges (#4036) --- docs/03_For_Hosts/05_Whitelisting.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/03_For_Hosts/05_Whitelisting.md b/docs/03_For_Hosts/05_Whitelisting.md index 113c4e3d..156174f0 100644 --- a/docs/03_For_Hosts/05_Whitelisting.md +++ b/docs/03_For_Hosts/05_Whitelisting.md @@ -1,14 +1,18 @@ -Modify `config.ini.php` to limit available bridges. +Modify `config.ini.php` to limit available bridges. Those changes should be applied in the `[system]` section. ## Enable all bridges ``` +[system] + enabled_bridges[] = * ``` ## Enable some bridges ``` +[system] + enabled_bridges[] = TwitchBridge enabled_bridges[] = GettrBridge ``` From 545dc969d35bc8c94a8c15875562690ee2fd6605 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 31 Mar 2024 03:38:42 +0200 Subject: [PATCH 049/423] refactor (#4037) --- actions/DisplayAction.php | 8 +++--- bridges/NintendoBridge.php | 1 - formats/HtmlFormat.php | 51 ++++++++++++++++------------------ lib/BridgeCard.php | 2 +- templates/html-format.html.php | 25 ++++++++++++----- 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index ed063825..24bdefe1 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -51,7 +51,6 @@ class DisplayAction implements ActionInterface return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400); } - if ( Configuration::getConfig('proxy', 'url') && Configuration::getConfig('proxy', 'by_bridge') @@ -62,8 +61,6 @@ class DisplayAction implements ActionInterface } $bridge = $bridgeFactory->create($bridgeClassName); - $formatFactory = new FormatFactory(); - $format = $formatFactory->create($format); $response = $this->createResponse($request, $bridge, $format); @@ -93,7 +90,7 @@ class DisplayAction implements ActionInterface return $response; } - private function createResponse(Request $request, BridgeAbstract $bridge, FormatAbstract $format) + private function createResponse(Request $request, BridgeAbstract $bridge, string $format) { $items = []; $feed = []; @@ -157,6 +154,9 @@ class DisplayAction implements ActionInterface } } + $formatFactory = new FormatFactory(); + $format = $formatFactory->create($format); + $format->setItems($items); $format->setFeed($feed); $now = time(); diff --git a/bridges/NintendoBridge.php b/bridges/NintendoBridge.php index 1c2ef71a..1c4ecf2b 100644 --- a/bridges/NintendoBridge.php +++ b/bridges/NintendoBridge.php @@ -4,7 +4,6 @@ class NintendoBridge extends XPathAbstract { const NAME = 'Nintendo Software Updates'; const URI = 'https://www.nintendo.co.uk/Support/Welcome-to-Nintendo-Support-11593.html'; - const DONATION_URI = ''; const DESCRIPTION = self::NAME; const MAINTAINER = 'Niehztog'; const PARAMETERS = [ diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index 93c824b3..1e2f60e6 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -6,34 +6,26 @@ class HtmlFormat extends FormatAbstract public function stringify() { + // This query string comes in already url decoded $queryString = $_SERVER['QUERY_STRING']; $feedArray = $this->getFeed(); $formatFactory = new FormatFactory(); - $buttons = []; - $linkTags = []; - foreach ($formatFactory->getFormatNames() as $formatName) { - // Dynamically build buttons for all formats (except HTML) + $formats = []; + + // Create all formats (except HTML) + $formatNames = $formatFactory->getFormatNames(); + foreach ($formatNames as $formatName) { if ($formatName === 'Html') { continue; } - $formatUrl = '?' . str_ireplace('format=Html', 'format=' . $formatName, htmlentities($queryString)); - $buttons[] = [ - 'href' => $formatUrl, - 'value' => $formatName, - ]; - $format = $formatFactory->create($formatName); - $linkTags[] = [ - 'href' => $formatUrl, - 'title' => $formatName, - 'type' => $format->getMimeType(), - ]; - } - - if (Configuration::getConfig('admin', 'donations') && $feedArray['donationUri']) { - $buttons[] = [ - 'href' => e($feedArray['donationUri']), - 'value' => 'Donate to maintainer', + // The format url is relative, but should be absolute in order to help feed readers. + $formatUrl = '?' . str_ireplace('format=Html', 'format=' . $formatName, $queryString); + $formatObject = $formatFactory->create($formatName); + $formats[] = [ + 'url' => $formatUrl, + 'name' => $formatName, + 'type' => $formatObject->getMimeType(), ]; } @@ -50,13 +42,18 @@ class HtmlFormat extends FormatAbstract ]; } + $donationUri = null; + if (Configuration::getConfig('admin', 'donations') && $feedArray['donationUri']) { + $donationUri = $feedArray['donationUri']; + } + $html = render_template(__DIR__ . '/../templates/html-format.html.php', [ - 'charset' => $this->getCharset(), - 'title' => $feedArray['name'], - 'linkTags' => $linkTags, - 'uri' => $feedArray['uri'], - 'buttons' => $buttons, - 'items' => $items, + 'charset' => $this->getCharset(), + 'title' => $feedArray['name'], + 'formats' => $formats, + 'uri' => $feedArray['uri'], + 'items' => $items, + 'donation_uri' => $donationUri, ]); // Remove invalid characters ini_set('mbstring.substitute_character', 'none'); diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index c4677b9d..d15ac865 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -78,7 +78,7 @@ final class BridgeCard $card .= sprintf('', $bridgeClassName); - if ($bridge->getDonationURI() !== '' && Configuration::getConfig('admin', 'donations')) { + if (Configuration::getConfig('admin', 'donations') && $bridge->getDonationURI()) { $card .= sprintf( '

%s ~ Donate

', $bridge->getMaintainer(), diff --git a/templates/html-format.html.php b/templates/html-format.html.php index 3b0fe6fe..bc95c5d0 100644 --- a/templates/html-format.html.php +++ b/templates/html-format.html.php @@ -8,12 +8,13 @@ - + + @@ -33,11 +34,21 @@ - - - + + + + + + + + + From b58d8b099b33ff030ac9004656a048b751ac2691 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Sun, 31 Mar 2024 03:44:10 +0200 Subject: [PATCH 050/423] docs: Complete helper function documentation (#3911) * docs: Complete helper function documentation Complete documentation of the Helper functions * docs: remove parameters and add a link to source - Parameters removed - Link to the file defining the function * docs: fix links Fix links to source files --- docs/06_Helper_functions/index.md | 81 +++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/docs/06_Helper_functions/index.md b/docs/06_Helper_functions/index.md index 31a13953..3aaeed89 100644 --- a/docs/06_Helper_functions/index.md +++ b/docs/06_Helper_functions/index.md @@ -233,3 +233,84 @@ $html = markdownToHtml($input); //
  • Translation improvements
    • // ``` + + +# e +The `e` function is used to convert special characters to HTML entities + +```PHP +e('0 < 1 and 2 > 1'); +``` + +`e` will return the content of the string escape that can be rendered as is in HTML + +[Defined in lib/html.php](/lib/html.php) + +# truncate +The `truncate` function is used to shorten a string if exceeds a certain length, and add a string indicating that the string has been shortened. + +```PHP +truncate('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed a neque nunc. Nam nibh sem.', 20 , '...'); +``` + +[Defined in lib/html.php](/lib/html.php) + +# sanitize +The `sanitize` function is used to remove some tags from a given HTML text. + +```PHP +$html = 'Sample Page +

    Lorem ipsum dolor sit amet, consectetur adipiscing elit...

    + + +'; +$tags_to_remove = ['script', 'iframe', 'input', 'form']; +$attributes_to_keep = ['title', 'href', 'src']; +$text_to_keep = []; +sanitize($html, $tags_to_remove, $attributes_to_keep, $text_to_keep); +``` + +This function returns a simplehtmldom object of the remaining contents. + +[Defined in lib/html.php](/lib/html.php) + +# convertLazyLoading +The `convertLazyLoading` function is used to convert onvert lazy-loading images and frames (video embeds) into static elements. It accepts the HTML content as HTML objects or string objects. It returns the HTML content with fixed image/frame URLs (same type as input). + +```PHP +$html = ' + +

    Hello world!

    + + +backgroundToImg($html); +``` + +[Defined in lib/html.php](/lib/html.php) + + +# Json::encode +The `Json::encode` function is used to encode a value as à JSON string. + +```PHP +$array = [ + "foo" => "bar", + "bar" => "foo", +]; +Json::encode($array, true, true); +``` + +[Defined in lib/utils.php](/lib/utils.php) + +# Json::decode +The `Json::decode` function is used to decode a JSON string into à PHP variable. + +```PHP +$json = '{ + "foo": "bar", + "bar": "foo" +}'; +Json::decode($json); +``` + +[Defined in lib/utils.php](/lib/utils.php) From d23fd2522ce71877bcfc942c13250ff3ec9d71ca Mon Sep 17 00:00:00 2001 From: Miika Launiainen Date: Sun, 31 Mar 2024 04:46:23 +0300 Subject: [PATCH 051/423] [GenshinImpactBridge] Fix bridge to use new API (#4011) * [GenshinImpactBridge] Fix bridge to use new API * Add category parameters back to not break existing feeds * Fix lint error * Remove whitespace --- bridges/GenshinImpactBridge.php | 55 +++++++++++++-------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/bridges/GenshinImpactBridge.php b/bridges/GenshinImpactBridge.php index 24bc39d8..0dc08a28 100644 --- a/bridges/GenshinImpactBridge.php +++ b/bridges/GenshinImpactBridge.php @@ -2,11 +2,11 @@ class GenshinImpactBridge extends BridgeAbstract { - const MAINTAINER = 'corenting'; const NAME = 'Genshin Impact'; - const URI = 'https://genshin.mihoyo.com/en/news'; - const CACHE_TIMEOUT = 7200; // 2h - const DESCRIPTION = 'News from the Genshin Impact website'; + const URI = 'https://genshin.hoyoverse.com/en/news'; + const CACHE_TIMEOUT = 18000; // 5h + const DESCRIPTION = 'Latest news from the Genshin Impact website'; + const MAINTAINER = 'Miicat_47'; const PARAMETERS = [ [ 'category' => [ @@ -25,37 +25,31 @@ class GenshinImpactBridge extends BridgeAbstract public function collectData() { - $category = $this->getInput('category'); - - $url = 'https://genshin.mihoyo.com/content/yuanshen/getContentList'; - $url = $url . '?pageSize=5&pageNum=1&channelId=' . $category; + $url = 'https://api-os-takumi-static.hoyoverse.com/content_v2_user/app/a1b1f9d3315447cc/getContentList?iAppId=32&iChanId=395&iPageSize=5&iPage=1&sLangKey=en-us'; $api_response = getContents($url); $json_list = json_decode($api_response, true); foreach ($json_list['data']['list'] as $json_item) { - $article_url = 'https://genshin.mihoyo.com/content/yuanshen/getContent'; - $article_url = $article_url . '?contentId=' . $json_item['contentId']; - $article_res = getContents($article_url); - $article_json = json_decode($article_res, true); - $article_time = $article_json['data']['start_time']; - $timezone = 'Asia/Shanghai'; - $article_timestamp = new DateTime($article_time, new DateTimeZone($timezone)); + $article_html = str_get_html($json_item['sContent']); + // Check if article contains a embed YouTube video + $exp_youtube = '/https:\/\/[w\.]+youtube\.com\/embed\/([\w]+)/m'; + if (preg_match($exp_youtube, $article_html, $matches)) { + // Replace the YouTube embed with a YouTube link + $yt_embed = $article_html->find('div[class="ttr-video-frame"]', 0); + $yt_link = sprintf('https://youtube.com/watch?v=%1$s', $matches[1]); + $article_html = str_replace($yt_embed, $yt_link, $article_html); + } $item = []; - - $item['title'] = $article_json['data']['title']; - $item['timestamp'] = $article_timestamp->format('U'); - $item['content'] = $article_json['data']['content']; - $item['uri'] = $this->getArticleUri($json_item); - $item['id'] = $json_item['contentId']; + $item['title'] = $json_item['sTitle']; + $item['timestamp'] = $json_item['dtStartTime']; + $item['content'] = $article_html; + $item['uri'] = 'https://genshin.hoyoverse.com/en/news/detail/' . $json_item['iInfoId']; + $item['id'] = $json_item['iInfoId']; // Picture - foreach ($article_json['data']['ext'] as $ext) { - if ($ext['arrtName'] == 'banner' && count($ext['value']) == 1) { - $item['enclosures'] = [$ext['value'][0]['url']]; - break; - } - } + $json_ext = json_decode($json_item['sExt'], true); + $item['enclosures'] = $json_ext['banner'][0]['url']; $this->items[] = $item; } @@ -63,11 +57,6 @@ class GenshinImpactBridge extends BridgeAbstract public function getIcon() { - return 'https://genshin.mihoyo.com/favicon.ico'; - } - - private function getArticleUri($json_item) - { - return 'https://genshin.mihoyo.com/en/news/detail/' . $json_item['contentId']; + return 'https://genshin.hoyoverse.com/favicon.ico'; } } From 1c3c85d8ff5a6d071f688ef09ca93f275b4995af Mon Sep 17 00:00:00 2001 From: Niehztog Date: Sun, 31 Mar 2024 18:46:07 +0200 Subject: [PATCH 052/423] [XPathBridge] Allow multiple categories (#4038) * [XPathAbstract] allow multiple categories * fix feed icons in two bridges * fix warning * fix linter errors --- bridges/BlizzardNewsBridge.php | 7 ++++ bridges/NiusBridge.php | 5 +++ lib/XPathAbstract.php | 74 +++++++++++++++++++++++++--------- 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/bridges/BlizzardNewsBridge.php b/bridges/BlizzardNewsBridge.php index 3930e0a4..19c38152 100644 --- a/bridges/BlizzardNewsBridge.php +++ b/bridges/BlizzardNewsBridge.php @@ -57,4 +57,11 @@ class BlizzardNewsBridge extends XPathAbstract } return 'https://news.blizzard.com/' . $locale; } + + public function getIcon() + { + return <<getItemValueOrNodeValue($typedResult, $isContent, $isContent && !$this->getSettingUseRawItemContent()); + $isCategories = 'categories' === $param; + $value = $this->getItemValueOrNodeValue($typedResult, $isContent, $isContent && !$this->getSettingUseRawItemContent(), $isCategories); $item->__set($param, $this->formatParamValue($param, $value)); } @@ -459,7 +460,7 @@ abstract class XPathAbstract extends BridgeAbstract */ protected function formatParamValue($param, $value) { - $value = $this->fixEncoding($value); + $value = is_array($value) ? array_map([$this, 'fixEncoding'], $value) : $this->fixEncoding($value); switch ($param) { case 'title': return $this->formatItemTitle($value); @@ -572,12 +573,12 @@ abstract class XPathAbstract extends BridgeAbstract * formatted as array. * Can be easily overwritten for in case the values need to be transformed into something * else. - * @param string $value + * @param string|array $value * @return array */ protected function formatItemCategories($value) { - return [$value]; + return is_array($value) ? $value : [$value]; } /** @@ -596,22 +597,21 @@ abstract class XPathAbstract extends BridgeAbstract /** * @param $typedResult - * @return string + * @param bool $returnXML + * @param bool $escapeHtml + * @param bool $allowMultiple + * @return string|array + * @throws Exception */ - protected function getItemValueOrNodeValue($typedResult, $returnXML = false, $escapeHtml = false) + protected function getItemValueOrNodeValue($typedResult, $returnXML = false, $escapeHtml = false, $allowMultiple = false) { - if ($typedResult instanceof \DOMNodeList) { + if ($typedResult instanceof \DOMNodeList && !$allowMultiple) { $item = $typedResult->item(0); - if ($item instanceof \DOMElement) { - // Don't escape XML - if ($returnXML) { - return ($item->ownerDocument ?? $item)->saveXML($item); - } - $text = $item->nodeValue; - } elseif ($item instanceof \DOMAttr) { - $text = $item->value; - } elseif ($item instanceof \DOMText) { - $text = $item->wholeText; + $text = $this->extractNodeListContent($item, $returnXML); + } elseif ($typedResult instanceof \DOMNodeList && $allowMultiple) { + $text = []; + foreach ($typedResult as $item) { + $text[] = $this->extractNodeListContent($item, $returnXML); } } elseif (is_string($typedResult) && strlen($typedResult) > 0) { $text = $typedResult; @@ -619,10 +619,46 @@ abstract class XPathAbstract extends BridgeAbstract throw new \Exception('Unknown type of XPath expression result.'); } + if (is_array($text)) { + foreach ($text as &$element) { + $element = $this->cleanExtractedText($element, $escapeHtml, $returnXML); + } + } else { + $text = $this->cleanExtractedText($text, $escapeHtml, $returnXML); + } + return $text; + } + + /** + * @param $item + * @param $returnXML + * @return false|string + * @throws Exception + */ + protected function extractNodeListContent($item, $returnXML) + { + if ($item instanceof \DOMElement) { + return $returnXML ? ($item->ownerDocument ?? $item)->saveXML($item) : $item->nodeValue; + } elseif ($item instanceof \DOMAttr) { + return $item->value; + } elseif ($item instanceof \DOMText) { + return $item->wholeText; + } + throw new \Exception('Unknown type of XPath expression result.'); + } + + /** + * @param $text + * @param $escapeHtml + * @param $returnXML + * @return string + */ + protected function cleanExtractedText($text, $escapeHtml, $returnXML) + { $text = trim($text); - if ($escapeHtml) { - return htmlspecialchars($text); + if ($escapeHtml && !$returnXML) { + $text = htmlspecialchars($text); } return $text; } From 8ca1b908400d2965c3ca6aa76b821b7bca7c50e0 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 31 Mar 2024 20:07:14 +0200 Subject: [PATCH 053/423] fix(NationalGeographicBridge) (#4039) --- bridges/NationalGeographicBridge.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bridges/NationalGeographicBridge.php b/bridges/NationalGeographicBridge.php index f7572240..7f8f4fa2 100644 --- a/bridges/NationalGeographicBridge.php +++ b/bridges/NationalGeographicBridge.php @@ -168,7 +168,7 @@ class NationalGeographicBridge extends BridgeAbstract } $image = $story['img']; - $item['enclosures'][] = $image['src']; + $item['enclosures'][] = str_replace(' ', '%20', $image['src']); foreach ($story['tags'] as $tag) { $item['categories'][] = $tag['name'] ?? $tag; @@ -218,7 +218,10 @@ class NationalGeographicBridge extends BridgeAbstract switch ($image_type) { case 'image': case 'imagegroup': - $image = $image_module['image']; + $image = $image_module['image'] ?? null; + if (!$image) { + return ''; + } $image_src = $image['src']; if (isset($image_module['alt'])) { $image_alt = $image_module['alt']; @@ -266,7 +269,11 @@ EOD; $json = json_decode($matches[1][0], true); - $unfiltered_data = $json['page']['content']['article']['frms']; + if (isset($json['page']['content']['article']['frms'])) { + $unfiltered_data = $json['page']['content']['article']['frms']; + } else { + $unfiltered_data = $json['page']['content']['prismarticle']['frms']; + } $filtered_data = $this->filterArticleData($unfiltered_data); $article = $filtered_data['edgs'][0]; @@ -288,7 +295,7 @@ EOD; } } - $published_date = $article['pbDt']; + $published_date = $article['pbDt'] ?? $article['dt']; $article_body = $article['bdy']; $content = ''; From 73289324bd39a31e225bd8f8048a1081bb771c67 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 31 Mar 2024 21:02:55 +0200 Subject: [PATCH 054/423] feat: add vendor http header to cached responses (#4040) --- actions/DisplayAction.php | 2 +- bridges/MediapartBlogsBridge.php | 7 ++++++- lib/FeedItem.php | 2 -- lib/http.php | 9 ++++++++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 24bdefe1..93813004 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -32,7 +32,7 @@ class DisplayAction implements ActionInterface return new Response('', 304, ['last-modified' => $modificationTimeGMT . 'GMT']); } } - return $cachedResponse; + return $cachedResponse->withHeader('rss-bridge', 'This is a cached response'); } if (!$bridgeName) { diff --git a/bridges/MediapartBlogsBridge.php b/bridges/MediapartBlogsBridge.php index fa8c3d5f..d1e1c3c9 100644 --- a/bridges/MediapartBlogsBridge.php +++ b/bridges/MediapartBlogsBridge.php @@ -35,7 +35,12 @@ class MediapartBlogsBridge extends BridgeAbstract $item['title'] = $item_title->innertext; $item['uri'] = self::BASE_URI . trim($item_title->href); - $item['author'] = $element->find('.author .subscriber', 0)->innertext; + + $author = $element->find('.author .subscriber', 0); + if ($author) { + $item['author'] = $author->innertext; + } + $item['content'] = $item_divs[count($item_divs) - 2] . $item_divs[count($item_divs) - 1]; $item['timestamp'] = strtotime($element->find('.author time', 0)->datetime); diff --git a/lib/FeedItem.php b/lib/FeedItem.php index bd37f119..fc4549a7 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -178,7 +178,6 @@ class FeedItem } else { $this->author = $author; } - return $this; } public function getContent(): ?string @@ -284,7 +283,6 @@ class FeedItem } else { $this->misc[$name] = $value; } - return $this; } public function toArray(): array diff --git a/lib/http.php b/lib/http.php index e4f9bf48..39f0c727 100644 --- a/lib/http.php +++ b/lib/http.php @@ -331,7 +331,14 @@ final class Response return array_pop($header); } - public function withBody(string $body): Response + public function withHeader(string $name, string $value): self + { + $clone = clone $this; + $clone->headers[$name] = [$value]; + return $clone; + } + + public function withBody(string $body): self { $clone = clone $this; $clone->body = $body; From 17a3b4c9d871208896e6c50a64b539d8689be128 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 31 Mar 2024 21:32:27 +0200 Subject: [PATCH 055/423] Fix 198 (#4041) * fix(twitch): log instead of exception * typo --- bridges/RedditBridge.php | 2 +- bridges/TwitchBridge.php | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index fbc6f678..7ece0e15 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -121,7 +121,7 @@ class RedditBridge extends BridgeAbstract $comments = false; $frontend = $this->getInput('frontend'); if ($frontend == '') { - $frontend = 'https://old.reddit.com'; + $frontend = 'https://old.reddit.com'; } $section = $this->getInput('d'); diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index f408f885..9e70944e 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -95,10 +95,14 @@ EOD; if ($data->user === null) { throw new \Exception(sprintf('Unable to find channel `%s`', $channel)); } + $user = $data->user; if ($user->videos === null) { - throw new HttpException('Service Unavailable', 503); + // twitch regularly does this for unknown reasons + $this->logger->info('Twitch returned empty set of videos', ['data' => $data]); + return; } + foreach ($user->videos->edges as $edge) { $video = $edge->node; From 9682f74fc569214186a504c03cf52dc78ed73d5f Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 31 Mar 2024 21:37:51 +0200 Subject: [PATCH 056/423] fix(cnet): author typo (#4042) --- bridges/CNETBridge.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bridges/CNETBridge.php b/bridges/CNETBridge.php index 4a63c847..17c05e9b 100644 --- a/bridges/CNETBridge.php +++ b/bridges/CNETBridge.php @@ -56,7 +56,7 @@ class CNETBridge extends SitemapBridge foreach ($links as $article_uri) { $article_dom = convertLazyLoading(getSimpleHTMLDOMCached($article_uri)); $title = trim($article_dom->find('h1', 0)->plaintext); - $author = $article_dom->find('span.c-assetAuthor_name', 0)->plaintext; + $author = $article_dom->find('span.c-assetAuthor_name', 0); $headline = $article_dom->find('p.c-contentHeader_description', 0); $content = $article_dom->find('div.c-pageArticle_content, div.single-article__content, div.article-main-body', 0); $date = null; @@ -97,7 +97,11 @@ class CNETBridge extends SitemapBridge $item = []; $item['uri'] = $article_uri; $item['title'] = $title; - $item['author'] = $author; + + if ($author) { + $item['author'] = $author->plaintext; + } + $item['content'] = $content; if (!is_null($date)) { From 182567e4341db19eb91df107216e6667c5d6e5de Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 31 Mar 2024 21:52:53 +0200 Subject: [PATCH 057/423] fix(bridges/DavesTrailerPageBridge): remove (#4043) --- bridges/DavesTrailerPageBridge.php | 40 ------------------------------ 1 file changed, 40 deletions(-) delete mode 100644 bridges/DavesTrailerPageBridge.php diff --git a/bridges/DavesTrailerPageBridge.php b/bridges/DavesTrailerPageBridge.php deleted file mode 100644 index 965f7e59..00000000 --- a/bridges/DavesTrailerPageBridge.php +++ /dev/null @@ -1,40 +0,0 @@ -find('tr') as $tr) { - // If it's a date row, update the current date - if ($tr->align == 'center') { - $curr_date = $tr->plaintext; - continue; - } - - $item = []; - - // title - $item['title'] = $tr->find('td', 0)->find('b', 0)->plaintext; - - // content - $item['content'] = $tr->find('ul', 1); - - // uri - $item['uri'] = $tr->find('a', 3)->getAttribute('href'); - - // date: parsed by FeedItem using strtotime - $item['timestamp'] = $curr_date; - - $this->items[] = $item; - } - } -} From d5d470cbc2195472fc1387ab55befd09d9ce21fc Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 31 Mar 2024 22:10:59 +0200 Subject: [PATCH 058/423] fix(dribble) (#4044) --- bridges/DribbbleBridge.php | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/bridges/DribbbleBridge.php b/bridges/DribbbleBridge.php index 3957c9de..539127b3 100644 --- a/bridges/DribbbleBridge.php +++ b/bridges/DribbbleBridge.php @@ -18,12 +18,12 @@ favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico'; { $html = getSimpleHTMLDOM(self::URI); - $json = $this->loadEmbeddedJsonData($html); + $data = $this->fetchData($html); foreach ($html->find('li[id^="screenshot-"]') as $shot) { $item = []; - $additional_data = $this->findJsonForShot($shot, $json); + $additional_data = $this->findJsonForShot($shot, $data); if ($additional_data === null) { $item['uri'] = self::URI . $shot->find('a', 0)->href; $item['title'] = $shot->find('.shot-title', 0)->plaintext; @@ -46,9 +46,8 @@ favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico'; } } - private function loadEmbeddedJsonData($html) + private function fetchData($html) { - $json = []; $scripts = $html->find('script'); foreach ($scripts as $script) { @@ -69,12 +68,17 @@ favicon-63b2904a073c89b52b19aa08cebc16a154bcf83fee8ecc6439968b1e6db569c7.ico'; $end = strpos($script->innertext, '];') + 1; // convert JSON to PHP array - $json = json_decode(substr($script->innertext, $start, $end - $start), true); - break; + $json = substr($script->innertext, $start, $end - $start); + + try { + // TODO: fix broken json + return Json::decode($json); + } catch (\JsonException $e) { + return []; + } } } - - return $json; + return []; } private function findJsonForShot($shot, $json) From 7001fbaf49a801226e819919cff913909242e7ac Mon Sep 17 00:00:00 2001 From: July Date: Sun, 31 Mar 2024 16:41:58 -0400 Subject: [PATCH 059/423] [AO3Bridge] Fix bad heading selector (#4045) --- bridges/AO3Bridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php index 85f0f9f8..4c09c28c 100644 --- a/bridges/AO3Bridge.php +++ b/bridges/AO3Bridge.php @@ -77,7 +77,7 @@ class AO3Bridge extends BridgeAbstract $html = defaultLinkTo($html, self::URI); // Get list title. Will include page range + count in some cases - $heading = ($html->find('#main > h2', 0)); + $heading = ($html->find('#main h2', 0)); if ($heading->find('a.tag')) { $heading = $heading->find('a.tag', 0); } From b4659786cb682096a46c0b6ca8a54505aa0ba49d Mon Sep 17 00:00:00 2001 From: Miika Launiainen Date: Mon, 1 Apr 2024 22:16:32 +0300 Subject: [PATCH 060/423] [GenshinImpactBridge] Small fixes (#4046) * Switch json_decode to Json::decode * Change regex delimeter from / to # * Save item enclosures as list --- bridges/GenshinImpactBridge.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridges/GenshinImpactBridge.php b/bridges/GenshinImpactBridge.php index 0dc08a28..924155d9 100644 --- a/bridges/GenshinImpactBridge.php +++ b/bridges/GenshinImpactBridge.php @@ -27,13 +27,13 @@ class GenshinImpactBridge extends BridgeAbstract { $url = 'https://api-os-takumi-static.hoyoverse.com/content_v2_user/app/a1b1f9d3315447cc/getContentList?iAppId=32&iChanId=395&iPageSize=5&iPage=1&sLangKey=en-us'; $api_response = getContents($url); - $json_list = json_decode($api_response, true); + $json_list = Json::decode($api_response); foreach ($json_list['data']['list'] as $json_item) { $article_html = str_get_html($json_item['sContent']); // Check if article contains a embed YouTube video - $exp_youtube = '/https:\/\/[w\.]+youtube\.com\/embed\/([\w]+)/m'; + $exp_youtube = '#https://[w\.]+youtube\.com/embed/([\w]+)#m'; if (preg_match($exp_youtube, $article_html, $matches)) { // Replace the YouTube embed with a YouTube link $yt_embed = $article_html->find('div[class="ttr-video-frame"]', 0); @@ -48,8 +48,8 @@ class GenshinImpactBridge extends BridgeAbstract $item['id'] = $json_item['iInfoId']; // Picture - $json_ext = json_decode($json_item['sExt'], true); - $item['enclosures'] = $json_ext['banner'][0]['url']; + $json_ext = Json::decode($json_item['sExt']); + $item['enclosures'] = [$json_ext['banner'][0]['url']]; $this->items[] = $item; } From a12bab9eedcf149778ad47e2ee84346e7b0d2016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wr=C3=B3bel?= Date: Mon, 1 Apr 2024 23:44:45 +0200 Subject: [PATCH 061/423] [AllegroBridge] ask for a complete cookie string, mere wcdx works no more (#4048) --- bridges/AllegroBridge.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/bridges/AllegroBridge.php b/bridges/AllegroBridge.php index 7cad11f1..04121257 100644 --- a/bridges/AllegroBridge.php +++ b/bridges/AllegroBridge.php @@ -13,12 +13,9 @@ class AllegroBridge extends BridgeAbstract 'exampleValue' => 'https://allegro.pl/kategoria/swieze-warzywa-cebula-318660', 'required' => true, ], - 'sessioncookie' => [ - 'name' => 'The \'wdctx\' session cookie', - 'title' => 'Paste the value of the \'wdctx\' cookie from your browser if you want to prevent Allegro imposing rate limits', - 'pattern' => '^.{70,};?$', - // phpcs:ignore - 'exampleValue' => 'v4.1-oCrmXTMqv2ppC21GTUCKLmUwRPP1ssQVALKuqwsZ1VXjcKgL2vO5TTRM5xMxS9GiyqxF1gAeyc-63dl0coUoBKXCXi_nAmr95yyqGpq2RAFoneZ4L399E8n6iYyemcuGARjAoSfjvLHJCEwvvHHynSgaxlFBu7hUnKfuy39zo9sSQdyTUjotJg3CAZ53q9v2raAnPCyGOAR4ytRILd9p24EJnxp7_oR0XbVPIo1hDa4WmjXFOxph8rHaO5tWd', + 'cookie' => [ + 'name' => 'The complete cookie value', + 'title' => 'Paste the value of the cookie value from your browser if you want to prevent Allegro imposing rate limits', 'required' => false, ], 'includeSponsoredOffers' => [ @@ -70,9 +67,9 @@ class AllegroBridge extends BridgeAbstract $opts = []; - // If a session cookie is provided - if ($sessioncookie = $this->getInput('sessioncookie')) { - $opts[CURLOPT_COOKIE] = 'wdctx=' . $sessioncookie; + // If a cookie is provided + if ($cookie = $this->getInput('cookie')) { + $opts[CURLOPT_COOKIE] = $cookie; } $html = getSimpleHTMLDOM($url, [], $opts); From bb979e9e0865991806f0d2cf1c55044168bcf586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wr=C3=B3bel?= Date: Tue, 2 Apr 2024 00:06:15 +0200 Subject: [PATCH 062/423] [AllegroBridge] fix logical condition on parameters (#4049) --- bridges/AllegroBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/AllegroBridge.php b/bridges/AllegroBridge.php index 04121257..55e9f116 100644 --- a/bridges/AllegroBridge.php +++ b/bridges/AllegroBridge.php @@ -81,11 +81,11 @@ class AllegroBridge extends BridgeAbstract $results = $html->find('article[data-analytics-view-custom-context="REGULAR"]'); - if (!$this->getInput('includeSponsoredOffers')) { + if ($this->getInput('includeSponsoredOffers')) { $results = array_merge($results, $html->find('article[data-analytics-view-custom-context="SPONSORED"]')); } - if (!$this->getInput('includePromotedOffers')) { + if ($this->getInput('includePromotedOffers')) { $results = array_merge($results, $html->find('article[data-analytics-view-custom-context="PROMOTED"]')); } From 8f962383c269aa181c3fc7025c478ae712cd38a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wr=C3=B3bel?= Date: Tue, 2 Apr 2024 01:01:23 +0200 Subject: [PATCH 063/423] [eBayBridge] fix Belgian eBay URL handling (#4050) Fixes #3918 --- bridges/EBayBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/EBayBridge.php b/bridges/EBayBridge.php index 507930ea..87958164 100644 --- a/bridges/EBayBridge.php +++ b/bridges/EBayBridge.php @@ -10,7 +10,7 @@ class EBayBridge extends BridgeAbstract 'url' => [ 'name' => 'Search URL', 'title' => 'Copy the URL from your browser\'s address bar after searching for your items and paste it here', - 'pattern' => '^(https:\/\/)?(www.)?ebay\.(com|com\.au|at|be|ca|ch|cn|es|fr|de|com\.hk|ie|it|com\.my|nl|ph|pl|com\.sg|co\.uk).*$', + 'pattern' => '^(https:\/\/)?(www\.)?(befr\.|benl\.)?ebay\.(com|com\.au|at|be|ca|ch|cn|es|fr|de|com\.hk|ie|it|com\.my|nl|ph|pl|com\.sg|co\.uk).*$', 'exampleValue' => 'https://www.ebay.com/sch/i.html?_nkw=atom+rss', 'required' => true, ] From fb66775ecee9db86c6be55c28eeae1c8f50e4cee Mon Sep 17 00:00:00 2001 From: Niehztog Date: Tue, 2 Apr 2024 23:14:25 +0200 Subject: [PATCH 064/423] [XPathAbstract] Refactor xpath abstract (#4047) * refactor XPathAbstract, keep all functionality intact * fix linter errors * further simplify code * set default value for raw item content to true, avoiding escaping of html tags in feed item contents by default --- bridges/BlizzardNewsBridge.php | 2 +- lib/XPathAbstract.php | 131 ++++++++++++--------------------- 2 files changed, 50 insertions(+), 83 deletions(-) diff --git a/bridges/BlizzardNewsBridge.php b/bridges/BlizzardNewsBridge.php index 19c38152..993492d4 100644 --- a/bridges/BlizzardNewsBridge.php +++ b/bridges/BlizzardNewsBridge.php @@ -37,7 +37,7 @@ class BlizzardNewsBridge extends XPathAbstract const XPATH_EXPRESSION_ITEM = '/html/body/div/div[4]/div[2]/div[2]/div/div/section/ol/li/article'; const XPATH_EXPRESSION_ITEM_TITLE = './/div/div[2]/h2'; - const XPATH_EXPRESSION_ITEM_CONTENT = './/div[@class="ArticleListItem-description"]/div[@class="h6"]'; + const XPATH_EXPRESSION_ITEM_CONTENT = './/div[@class="ArticleListItem-description"]/div[@class="h6"]/text()'; const XPATH_EXPRESSION_ITEM_URI = './/a[@class="ArticleLink ArticleLink"]/@href'; const XPATH_EXPRESSION_ITEM_AUTHOR = ''; const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/time[@class="ArticleListItem-footerTimestamp"]/@timestamp'; diff --git a/lib/XPathAbstract.php b/lib/XPathAbstract.php index 224d8e87..6163ca13 100644 --- a/lib/XPathAbstract.php +++ b/lib/XPathAbstract.php @@ -76,15 +76,6 @@ abstract class XPathAbstract extends BridgeAbstract */ const XPATH_EXPRESSION_ITEM_CONTENT = ''; - /** - * Use raw item content - * Whether to use the raw item content or to replace certain characters with - * special significance in HTML by HTML entities (using the PHP function htmlspecialchars). - * - * Use {@see XPathAbstract::getSettingUseRawItemContent()} to read this parameter - */ - const SETTING_USE_RAW_ITEM_CONTENT = false; - /** * XPath expression for extracting an item link from the item context * This expression should match a node's attribute containing the article URL @@ -158,6 +149,15 @@ abstract class XPathAbstract extends BridgeAbstract */ const SETTING_FIX_ENCODING = false; + /** + * Use raw item content + * Whether to use the raw item content or to replace certain characters with + * special significance in HTML by HTML entities (using the PHP function htmlspecialchars). + * + * Use {@see XPathAbstract::getSettingUseRawItemContent()} to read this parameter + */ + const SETTING_USE_RAW_ITEM_CONTENT = true; + /** * Internal storage for resulting feed name, automatically detected * @var string @@ -245,15 +245,6 @@ abstract class XPathAbstract extends BridgeAbstract return static::XPATH_EXPRESSION_ITEM_CONTENT; } - /** - * Use raw item content - * @return bool - */ - protected function getSettingUseRawItemContent(): bool - { - return static::SETTING_USE_RAW_ITEM_CONTENT; - } - /** * XPath expression for extracting an item link from the item context * @return string @@ -309,6 +300,15 @@ abstract class XPathAbstract extends BridgeAbstract return static::SETTING_FIX_ENCODING; } + /** + * Use raw item content + * @return bool + */ + protected function getSettingUseRawItemContent(): bool + { + return static::SETTING_USE_RAW_ITEM_CONTENT; + } + /** * Internal helper method for quickly accessing all the user defined constants * in derived classes @@ -331,8 +331,6 @@ abstract class XPathAbstract extends BridgeAbstract return $this->getExpressionItemTitle(); case 'content': return $this->getExpressionItemContent(); - case 'raw_content': - return $this->getSettingUseRawItemContent(); case 'uri': return $this->getExpressionItemUri(); case 'author': @@ -345,6 +343,8 @@ abstract class XPathAbstract extends BridgeAbstract return $this->getExpressionItemCategories(); case 'fix_encoding': return $this->getSettingFixEncoding(); + case 'raw_content': + return $this->getSettingUseRawItemContent(); } } @@ -438,9 +438,15 @@ abstract class XPathAbstract extends BridgeAbstract continue; } - $isContent = $param === 'content'; - $isCategories = 'categories' === $param; - $value = $this->getItemValueOrNodeValue($typedResult, $isContent, $isContent && !$this->getSettingUseRawItemContent(), $isCategories); + if ('categories' === $param && $typedResult instanceof \DOMNodeList) { + $value = []; + foreach ($typedResult as $domNode) { + $value[] = $this->getItemValueOrNodeValue($domNode, false); + } + } else { + $value = $this->getItemValueOrNodeValue($typedResult, 'content' === $param); + } + $item->__set($param, $this->formatParamValue($param, $value)); } @@ -460,6 +466,7 @@ abstract class XPathAbstract extends BridgeAbstract */ protected function formatParamValue($param, $value) { + $value = is_array($value) ? array_map('trim', $value) : trim($value); $value = is_array($value) ? array_map([$this, 'fixEncoding'], $value) : $this->fixEncoding($value); switch ($param) { case 'title': @@ -503,7 +510,7 @@ abstract class XPathAbstract extends BridgeAbstract */ protected function formatItemContent($value) { - return $value; + return $this->getParam('raw_content') ? $value : htmlspecialchars($value); } /** @@ -599,68 +606,28 @@ abstract class XPathAbstract extends BridgeAbstract * @param $typedResult * @param bool $returnXML * @param bool $escapeHtml - * @param bool $allowMultiple - * @return string|array - * @throws Exception - */ - protected function getItemValueOrNodeValue($typedResult, $returnXML = false, $escapeHtml = false, $allowMultiple = false) - { - if ($typedResult instanceof \DOMNodeList && !$allowMultiple) { - $item = $typedResult->item(0); - $text = $this->extractNodeListContent($item, $returnXML); - } elseif ($typedResult instanceof \DOMNodeList && $allowMultiple) { - $text = []; - foreach ($typedResult as $item) { - $text[] = $this->extractNodeListContent($item, $returnXML); - } - } elseif (is_string($typedResult) && strlen($typedResult) > 0) { - $text = $typedResult; - } else { - throw new \Exception('Unknown type of XPath expression result.'); - } - - if (is_array($text)) { - foreach ($text as &$element) { - $element = $this->cleanExtractedText($element, $escapeHtml, $returnXML); - } - } else { - $text = $this->cleanExtractedText($text, $escapeHtml, $returnXML); - } - return $text; - } - - /** - * @param $item - * @param $returnXML - * @return false|string - * @throws Exception - */ - protected function extractNodeListContent($item, $returnXML) - { - if ($item instanceof \DOMElement) { - return $returnXML ? ($item->ownerDocument ?? $item)->saveXML($item) : $item->nodeValue; - } elseif ($item instanceof \DOMAttr) { - return $item->value; - } elseif ($item instanceof \DOMText) { - return $item->wholeText; - } - throw new \Exception('Unknown type of XPath expression result.'); - } - - /** - * @param $text - * @param $escapeHtml - * @param $returnXML * @return string + * @throws Exception */ - protected function cleanExtractedText($text, $escapeHtml, $returnXML) + protected function getItemValueOrNodeValue($typedResult, $returnXML = false) { - $text = trim($text); - - if ($escapeHtml && !$returnXML) { - $text = htmlspecialchars($text); + if ($typedResult instanceof \DOMNodeList) { + $typedResult = $typedResult->item(0); } - return $text; + + if ($typedResult instanceof \DOMElement) { + return $returnXML ? ($typedResult->ownerDocument ?? $typedResult)->saveXML($typedResult) : $typedResult->nodeValue; + } elseif ($typedResult instanceof \DOMAttr) { + return $typedResult->value; + } elseif ($typedResult instanceof \DOMText) { + return $typedResult->wholeText; + } elseif (is_string($typedResult)) { + return $typedResult; + } elseif (null === $typedResult) { + return ''; + } + + throw new \Exception('Unknown type of XPath expression result: ' . gettype($typedResult)); } /** From f736da6faebecf9215c11674c4dce12497136281 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:23:52 +0200 Subject: [PATCH 065/423] [GolemBridge] fix for internal videos (#4051) * [GolemBridge] fix for internal videos with this internal golem-videos can be played directly from feed * Update GolemBridge.php --- bridges/GolemBridge.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index 216e913f..b52d3c2f 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -119,6 +119,16 @@ class GolemBridge extends FeedExpander } } + //built golem videos + foreach ($article->find('.gvideofig') as &$embedcontent) { + if (preg_match('/gvideo_(.*)/', $embedcontent->id, $videoid)) { + $embedcontent->innertext .= << + EOT; + } + } + // delete known bad elements foreach ( $article->find('div[id*="adtile"], #job-market, #seminars, iframe, @@ -142,7 +152,7 @@ class GolemBridge extends FeedExpander $img->src = $img->getAttribute('data-src-full'); } - foreach ($content->find('p, h1, h2, h3, img[src*="."], iframe') as $element) { + foreach ($content->find('p, h1, h2, h3, img[src*="."], iframe, video') as $element) { $item .= $element; } From 94292af51b1dd662cc9513df0f2bfacfa70dad35 Mon Sep 17 00:00:00 2001 From: User123698745 Date: Thu, 4 Apr 2024 04:07:16 +0200 Subject: [PATCH 066/423] [prtester.py] fix url parameter encoding (#4052) this will (at least) fix the pr preview of: bridges/AnisearchBridge.php bridges/BakaUpdatesMangaReleasesBridge.php bridges/DesoutterBridge.php bridges/IndiegogoBridge.php --- .github/prtester.py | 46 +++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/.github/prtester.py b/.github/prtester.py index 30a9f43b..3d7dae99 100644 --- a/.github/prtester.py +++ b/.github/prtester.py @@ -5,6 +5,7 @@ from bs4 import BeautifulSoup from datetime import datetime from typing import Iterable import os.path +import urllib # This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge # @@ -45,15 +46,14 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w bridgeid = bridge_card.get('id') bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata print(f'{bridgeid}{instance_suffix}') - bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html' bridge_name = bridgeid.replace('Bridge', '') context_forms = bridge_card.find_all("form") form_number = 1 for context_form in context_forms: # a bridge can have multiple contexts, named 'forms' in html - # this code will produce a fully working formstring that should create a working feed when called + # this code will produce a fully working url that should create a working feed when called # this will create an example feed for every single context, to test them all - formstring = '' + context_parameters = {} error_messages = [] context_name = '*untitled*' context_name_element = context_form.find_previous_sibling('h5') @@ -62,27 +62,27 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w parameters = context_form.find_all("input") lists = context_form.find_all("select") # this for/if mess cycles through all available input parameters, checks if it required, then pulls - # the default or examplevalue and then combines it all together into the formstring + # the default or examplevalue and then combines it all together into the url parameters # if an example or default value is missing for a required attribute, it will throw an error # any non-required fields are not tested!!! for parameter in parameters: - if parameter.get('type') == 'hidden' and parameter.get('name') == 'context': - cleanvalue = parameter.get('value').replace(" ","+") - formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue - if parameter.get('type') == 'number' or parameter.get('type') == 'text': + parameter_type = parameter.get('type') + parameter_name = parameter.get('name') + if parameter_type == 'hidden': + context_parameters[parameter_name] = parameter.get('value') + if parameter_type == 'number' or parameter_type == 'text': if parameter.has_attr('required'): if parameter.get('placeholder') == '': if parameter.get('value') == '': - name_value = parameter.get('name') - error_messages.append(f'Missing example or default value for parameter "{name_value}"') + error_messages.append(f'Missing example or default value for parameter "{parameter_name}"') else: - formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value') + context_parameters[parameter_name] = parameter.get('value') else: - formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder') - # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring - if parameter.get('type') == 'checkbox': + context_parameters[parameter_name] = parameter.get('placeholder') + # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the url parameters + if parameter_type == 'checkbox': if parameter.has_attr('checked'): - formstring = formstring + '&' + parameter.get('name') + '=on' + context_parameters[parameter_name] = 'on' for listing in lists: selectionvalue = '' listname = listing.get('name') @@ -102,15 +102,21 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w if 'selected' in selectionentry.attrs: selectionvalue = selectionentry.get('value') break - formstring = formstring + '&' + listname + '=' + selectionvalue + context_parameters[listname] = selectionvalue termpad_url = 'about:blank' if error_messages: status = '
    '.join(map(lambda m: f'❌ `{m}`', error_messages)) else: - # if all example/default values are present, form the full request string, run the request, add a tag with + # if all example/default values are present, form the full request url, run the request, add a tag with # the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and # then upload it to termpad.com, a pastebin-like-site. - response = requests.get(instance.url + bridgestring + formstring) + context_parameters.update({ + 'action': 'display', + 'bridge': bridgeid, + 'format': 'Html', + }) + request_url = f'{instance.url}/?{urllib.parse.urlencode(context_parameters)}' + response = requests.get(request_url) page_text = response.text.replace('','') page_text = page_text.encode("utf_8") soup = BeautifulSoup(page_text, "html.parser") @@ -163,8 +169,8 @@ if __name__ == '__main__': for instance_arg in args.instances: instance_arg_parts = instance_arg.split('::') instance = Instance() - instance.name = instance_arg_parts[1] if len(instance_arg_parts) >= 2 else '' - instance.url = instance_arg_parts[0] + instance.name = instance_arg_parts[1].strip() if len(instance_arg_parts) >= 2 else '' + instance.url = instance_arg_parts[0].strip().rstrip("/") instances.append(instance) else: instance = Instance() From 82606a479a75cd4eff4740ba49715b3135c89c49 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Thu, 4 Apr 2024 04:08:29 +0200 Subject: [PATCH 067/423] [PepperBridgeAbstract,DealabsBridge,HotUKDealsBridge,MydealsBridge] Fix search URL, No results handling fixed, Thread title and Message URL handling (#4053) * [PepperBridgeAbstract,DealabsBridge,HotUKDealsBridge,MydealsBridge] Fix search URL, No results handling fixed, Thread title and Message URL handling Search URL has been updated according to the website. If a search doesn't return any results, the HTML won't contain any specific text now : the HTML structure is slightly different, so the bridge has been updated. The unnneded 'no-results' text is now removed from the specific bridges. The board thread title has been removed from the content, so now we use the page element. In case a board message is empty, there was an exception during the filtering of message without URL. * [PepperBridgeAbstract,DealabsBridge,HotUKDealsBridge,MydealsBridge] Fix search URL, No results handling fixed, Thread title and Message URL handling Coding policy fixes --- bridges/DealabsBridge.php | 1 - bridges/HotUKDealsBridge.php | 1 - bridges/MydealsBridge.php | 1 - bridges/PepperBridgeAbstract.php | 25 ++++++++++++++----------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index 62d854f6..3ee1c6f5 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -1913,7 +1913,6 @@ class DealabsBridge extends PepperBridgeAbstract 'uri-merchant' => 'search/bons-plans?merchant-id=', 'request-error' => 'Impossible de joindre Dealabs', 'thread-error' => 'Impossible de déterminer l\'ID de la discussion. Vérifiez l\'URL que vous avez entré', - 'no-results' => 'Aucun résultat', 'currency' => '€', 'price' => 'Prix', 'shipping' => 'Livraison', diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index b631db73..6958220e 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3277,7 +3277,6 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'uri-merchant' => 'search/deals?merchant-id=', 'request-error' => 'Could not request HotUKDeals', 'thread-error' => 'Unable to determine the thread ID. Check the URL you entered', - 'no-results' => 'no results', 'currency' => '£', 'price' => 'Price', 'shipping' => 'Shipping', diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 41bae46c..7b23f263 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2024,7 +2024,6 @@ class MydealsBridge extends PepperBridgeAbstract 'uri-merchant' => 'search/gutscheine?merchant-id=', 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', - 'no-results' => 'keine Ergebnisse', 'currency' => '€', 'price' => 'Preis', 'shipping' => 'Versand', diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 7b40ea1c..33d427bc 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -55,8 +55,8 @@ class PepperBridgeAbstract extends BridgeAbstract ); // If there is no results, we don't parse the content because it display some random deals - $noresult = $html->find('h3[class*=text--b]', 0); - if ($noresult != null && strpos($noresult->plaintext, $this->i8n('no-results')) !== false) { + $noresult = $html->find('section[class=subNav]', 0)->find('div[class*=page-center listLayout aGrid]', 0); + if ($noresult === null) { $this->items = []; } else { foreach ($list as $deal) { @@ -174,13 +174,16 @@ HEREDOC; $item['uid'] = $comment->commentId; // Timestamp handling needs a new parsing function if ($onlyWithUrl == true) { - // Count Links and Quote Links - $content = str_get_html($item['content']); - $countLinks = count($content->find('a[href]')); - $countQuoteLinks = count($content->find('a[href][class=userHtml-quote-source]')); - // Only add element if there are Links ans more links tant Quote links - if ($countLinks > 0 && $countLinks > $countQuoteLinks) { - $this->items[] = $item; + // Only parse the comment if it is not empry + if ($item['content'] != '') { + // Count Links and Quote Links + $content = str_get_html($item['content']); + $countLinks = count($content->find('a[href]')); + $countQuoteLinks = count($content->find('a[href][class=userHtml-quote-source]')); + // Only add element if there are Links and more links tant Quote links + if ($countLinks > 0 && $countLinks > $countQuoteLinks) { + $this->items[] = $item; + } } } else { $this->items[] = $item; @@ -264,7 +267,7 @@ HEREDOC; private function getTalkTitle() { $html = getSimpleHTMLDOMCached($this->getInput('url')); - $title = $html->find('.thread-title', 0)->plaintext; + $title = $html->find('title', 0)->plaintext; return $title; } @@ -472,7 +475,7 @@ HEREDOC; $priceFrom = $this->getInput('priceFrom'); $priceTo = $this->getInput('priceTo'); $url = $this->i8n('bridge-uri') - . 'search/advanced?q=' + . 'search?q=' . urlencode($q) . '&hide_expired=' . $hide_expired . '&hide_local=' . $hide_local From 3cba984d22d6b1e045781f208211c4fb336c2bf0 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 4 Apr 2024 17:43:07 +0200 Subject: [PATCH 068/423] fix(FDroidRepoBridge): unlink when json file is absent from archive (#4056) --- bridges/FDroidRepoBridge.php | 110 +++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/bridges/FDroidRepoBridge.php b/bridges/FDroidRepoBridge.php index 7ce41baf..286ada1b 100644 --- a/bridges/FDroidRepoBridge.php +++ b/bridges/FDroidRepoBridge.php @@ -49,7 +49,7 @@ class FDroidRepoBridge extends BridgeAbstract throw new \Exception('FDroidRepoBridge requires the php-zip extension'); } - $this->repo = $this->getRepo(); + $this->repo = $this->fetchData(); switch ($this->queriedContext) { case 'Latest Updates': $this->getAllUpdates(); @@ -58,63 +58,40 @@ class FDroidRepoBridge extends BridgeAbstract $this->getPackage($this->getInput('package')); break; default: - returnServerError('Unimplemented Context (collectData)'); + throw new \Exception('Unimplemented Context (collectData)'); } } - public function getURI() - { - if (empty($this->queriedContext)) { - return parent::getURI(); - } - - $url = rtrim($this->GetInput('url'), '/'); - return strstr($url, '?', true) ?: $url; - } - - public function getName() - { - if (empty($this->queriedContext)) { - return parent::getName(); - } - - $name = $this->repo['repo']['name']; - switch ($this->queriedContext) { - case 'Latest Updates': - return $name; - case 'Follow Package': - return $this->getInput('package') . ' - ' . $name; - default: - returnServerError('Unimplemented Context (getName)'); - } - } - - private function getRepo() + /** + * This method fetches data from arbitrary url and writes to os temp file. + * I don't think there's any security problem here but might be DOS problems. + */ + private function fetchData() { $url = $this->getURI(); - // Get repo information (only available as JAR) - $jar = getContents($url . '/index-v1.jar'); - $jar_loc = tempnam(sys_get_temp_dir(), ''); - file_put_contents($jar_loc, $jar); + $zipFile = getContents($url . '/index-v1.jar'); + // On linux this creates a temp file in /tmp/ + $temporaryFile = tempnam(sys_get_temp_dir(), 'rssbridge_'); + file_put_contents($temporaryFile, $zipFile); - // JAR files are specially formatted ZIP files - $jar = new \ZipArchive(); - if ($jar->open($jar_loc) !== true) { - unlink($jar_loc); + $archive = new \ZipArchive(); + if ($archive->open($temporaryFile) !== true) { + unlink($temporaryFile); throw new \Exception('Failed to extract archive'); } - // Get file pointer to the relevant JSON inside - $fp = $jar->getStream('index-v1.json'); + $fp = $archive->getStream('index-v1.json'); if (!$fp) { - returnServerError('Failed to get file pointer'); + unlink($temporaryFile); + throw new \Exception('Failed to get file pointer'); } - $data = json_decode(stream_get_contents($fp), true); + $json = stream_get_contents($fp); fclose($fp); - $jar->close(); - unlink($jar_loc); + $data = Json::decode($json); + $archive->close(); + unlink($temporaryFile); return $data; } @@ -158,9 +135,9 @@ class FDroidRepoBridge extends BridgeAbstract $summary = $lang['summary'] ?? $app['summary'] ?? ''; $description = markdownToHtml(trim($lang['description'] ?? $app['description'] ?? 'None')); $whatsNew = markdownToHtml(trim($lang['whatsNew'] ?? 'None')); - $website = $this->link($lang['webSite'] ?? $app['webSite'] ?? $app['authorWebSite'] ?? null); - $source = $this->link($app['sourceCode'] ?? null); - $issueTracker = $this->link($app['issueTracker'] ?? null); + $website = $this->createAnchor($lang['webSite'] ?? $app['webSite'] ?? $app['authorWebSite'] ?? null); + $source = $this->createAnchor($app['sourceCode'] ?? null); + $issueTracker = $this->createAnchor($app['issueTracker'] ?? null); $license = $app['license'] ?? 'None'; $item['content'] = <<<EOD {$icon} @@ -182,7 +159,7 @@ EOD; private function getPackage($package) { if (!isset($this->repo['packages'][$package])) { - returnClientError('Invalid Package Name'); + throw new \Exception('Invalid Package Name'); } $package = $this->repo['packages'][$package]; @@ -192,7 +169,7 @@ EOD; $item['uri'] = $this->getURI() . '/' . $version['apkName']; $item['title'] = $version['versionName']; $item['timestamp'] = date(DateTime::ISO8601, (int) ($version['added'] / 1000)); - $item['uid'] = $version['versionCode']; + $item['uid'] = (string) $version['versionCode']; $size = round($version['size'] / 1048576, 1); // Bytes -> MB $sdk_link = 'https://developer.android.com/studio/releases/platforms'; $item['content'] = <<<EOD @@ -208,11 +185,42 @@ EOD; } } - private function link($url) + public function getURI() + { + if (empty($this->queriedContext)) { + return parent::getURI(); + } + + $url = rtrim($this->getInput('url'), '/'); + if (strstr($url, '?', true)) { + return strstr($url, '?', true); + } else { + return $url; + } + } + + public function getName() + { + if (empty($this->queriedContext)) { + return parent::getName(); + } + + $name = $this->repo['repo']['name']; + switch ($this->queriedContext) { + case 'Latest Updates': + return $name; + case 'Follow Package': + return $this->getInput('package') . ' - ' . $name; + default: + throw new \Exception('Unimplemented Context (getName)'); + } + } + + private function createAnchor($url) { if (empty($url)) { return null; } - return '<a href="' . $url . '">' . $url . '</a>'; + return sprintf('<a href="%s">%s</a>', $url, $url); } } From 001dd47439339672b6e84979fd0e1a0118bc27b6 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 4 Apr 2024 19:12:04 +0200 Subject: [PATCH 069/423] fix: small tweaks (#4057) --- bridges/FeedMergeBridge.php | 5 +++- bridges/GatesNotesBridge.php | 4 +++ bridges/PixivBridge.php | 47 +++--------------------------------- lib/FeedExpander.php | 1 + 4 files changed, 13 insertions(+), 44 deletions(-) diff --git a/bridges/FeedMergeBridge.php b/bridges/FeedMergeBridge.php index f2c1d9d5..37b574b6 100644 --- a/bridges/FeedMergeBridge.php +++ b/bridges/FeedMergeBridge.php @@ -64,6 +64,7 @@ TEXT; $this->collectExpandableDatas($feed); } catch (HttpException $e) { $this->logger->warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e))); + // This feed item might be spammy. Considering dropping it. $this->items[] = [ 'title' => 'RSS-Bridge: ' . $e->getMessage(), // Give current time so it sorts to the top @@ -71,7 +72,7 @@ TEXT; ]; continue; } catch (\Exception $e) { - if (str_starts_with($e->getMessage(), 'Unable to parse xml')) { + if (str_starts_with($e->getMessage(), 'Failed to parse xml')) { // Allow this particular exception from FeedExpander $this->logger->warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e))); continue; @@ -83,6 +84,8 @@ TEXT; } } + // If $this->items is empty we should consider throw exception here + // Sort by timestamp descending usort($this->items, function ($a, $b) { $t1 = $a['timestamp'] ?? $a['uri'] ?? $a['title']; diff --git a/bridges/GatesNotesBridge.php b/bridges/GatesNotesBridge.php index 3381e096..b46b3ce6 100644 --- a/bridges/GatesNotesBridge.php +++ b/bridges/GatesNotesBridge.php @@ -21,6 +21,10 @@ class GatesNotesBridge extends BridgeAbstract $rawContent = getContents($apiUrl); $cleanedContent = trim($rawContent, '"'); + $cleanedContent = str_replace([ + '<string xmlns="http://schemas.microsoft.com/2003/10/Serialization/">', + '</string>' + ], '', $cleanedContent); $cleanedContent = str_replace('\r\n', "\n", $cleanedContent); $cleanedContent = stripslashes($cleanedContent); diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php index fc4443ed..604b5d4b 100644 --- a/bridges/PixivBridge.php +++ b/bridges/PixivBridge.php @@ -160,7 +160,8 @@ class PixivBridge extends BridgeAbstract $json = array_reduce($json, function ($acc, $i) { if ($i['illustType'] === 0) { $acc[] = $i; - }return $acc; + } + return $acc; }, []); break; case 'manga': @@ -235,8 +236,10 @@ class PixivBridge extends BridgeAbstract $item = []; $item['uid'] = $result['id']; + $subpath = array_key_exists('illustType', $result) ? 'artworks/' : 'novel/show.php?id='; $item['uri'] = static::URI . $subpath . $result['id']; + $item['title'] = $result['title']; $item['author'] = $result['userName']; $item['timestamp'] = $result['updateDate']; @@ -253,8 +256,6 @@ class PixivBridge extends BridgeAbstract } } else { $img_url = $result['url']; - // Temporarily disabling caching of the image - //$img_url = $this->cacheImage($result['url'], $result['id'], array_key_exists('illustType', $result)); } // Currently, this might result in broken image due to their strict referrer check @@ -271,46 +272,6 @@ class PixivBridge extends BridgeAbstract } } - /** - * todo: remove manual file cache - * See bridge specific documentation for alternative option. - */ - private function cacheImage($url, $illustId, $isImage) - { - $illustId = preg_replace('/[^0-9]/', '', $illustId); - $thumbnailurl = $url; - - $path = PATH_CACHE . 'pixiv_img/'; - if (!is_dir($path)) { - mkdir($path, 0755, true); - } - - $path .= $illustId; - if ($this->getInput('fullsize')) { - $path .= '_fullsize'; - } - $path .= '.jpg'; - - if (!is_file($path)) { - // Get fullsize URL - if ($isImage && $this->getInput('fullsize')) { - $ajax_uri = static::URI . 'ajax/illust/' . $illustId; - $imagejson = $this->getData($ajax_uri, true, true); - $url = $imagejson['body']['urls']['original']; - } - - $headers = ['Referer: ' . static::URI]; - try { - $illust = $this->getData($url, true, false, $headers); - } catch (Exception $e) { - $illust = $this->getData($thumbnailurl, true, false, $headers); // Original thumbnail - } - file_put_contents($path, $illust); - } - - return get_home_page_url() . 'cache/pixiv_img/' . preg_replace('/.*\//', '', $path); - } - private function checkOptions() { $proxy = $this->getOption('proxy_url'); diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index abe964e1..fe809bc2 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -34,6 +34,7 @@ abstract class FeedExpander extends BridgeAbstract try { $this->feed = $feedParser->parseFeed($xmlString); } catch (\Exception $e) { + // FeedMergeBridge relies on this string throw new \Exception(sprintf('Failed to parse xml from %s: %s', $url, create_sane_exception_message($e))); } From 3ff2ef94e00d2edd52afb6795e9060538e63e4d5 Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Thu, 4 Apr 2024 19:28:56 +0200 Subject: [PATCH 070/423] Fix docs : Replace relative links to files with full URL (#4059) --- docs/06_Helper_functions/index.md | 39 +++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/06_Helper_functions/index.md b/docs/06_Helper_functions/index.md index 3aaeed89..2b675ca3 100644 --- a/docs/06_Helper_functions/index.md +++ b/docs/06_Helper_functions/index.md @@ -8,6 +8,8 @@ $this->getInput('your input name here'); `getInput` will either return the value for your parameter or `null` if the parameter is unknown or not specified. +[Defined in lib/BridgeAbstract.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/BridgeAbstract.php) + # getKey The `getKey` function is used to receive the key name to a selected list value given the name of the list, specified in `const PARAMETERS` @@ -39,6 +41,8 @@ $this->getKey('country'); `getKey` will either return the key name for your parameter or `null` if the parameter is unknown or not specified. +[Defined in lib/BridgeAbstract.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/BridgeAbstract.php) + # getContents The `getContents` function uses [cURL](https://secure.php.net/manual/en/book.curl.php) to acquire data from the specified URI while respecting the various settings defined at a global level by RSS-Bridge (i.e., proxy host, user agent, etc.). This function accepts a few parameters: @@ -55,6 +59,8 @@ $opts = array(CURLOPT_POST => 1); $html = getContents($url, $header, $opts); ``` +[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php) + # getSimpleHTMLDOM The `getSimpleHTMLDOM` function is a wrapper for the [simple_html_dom](https://simplehtmldom.sourceforge.io/) [file_get_html](https://simplehtmldom.sourceforge.io/docs/1.9/api/file_get_html/) function in order to provide context by design. @@ -62,6 +68,9 @@ The `getSimpleHTMLDOM` function is a wrapper for the ```PHP $html = getSimpleHTMLDOM('your URI'); ``` + +[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php) + # getSimpleHTMLDOMCached The `getSimpleHTMLDOMCached` function does the same as the [`getSimpleHTMLDOM`](#getsimplehtmldom) function, @@ -76,6 +85,8 @@ This function allows to specify the cache duration with the second parameter. $html = getSimpleHTMLDOMCached('your URI', 86400); // Duration 24h ``` +[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php) + # returnClientError The `returnClientError` function aborts execution of the current bridge and returns the given error message with error code **400**: @@ -86,6 +97,8 @@ returnClientError('Your error message') Use this function when the user provided invalid parameter or a required parameter is missing. +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) + # returnServerError The `returnServerError` function aborts execution of the current bridge and returns the given error message with error code **500**: @@ -96,6 +109,8 @@ returnServerError('Your error message') Use this function when a problem occurs that has nothing to do with the parameters provided by the user. (like: Host service gone missing, empty data received, etc...) +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) + # defaultLinkTo Automatically replaces any relative URL in a given string or DOM object (i.e. the one returned by [getSimpleHTMLDOM](#getsimplehtmldom)) with an absolute URL. @@ -122,6 +137,8 @@ $html = defaultLinkTo($html, $this->getURI()); // Using bridge URL // <img src="https://www.github.com/rss-bridge/rss-bridge/blob/master/README.md"> ``` +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) + # backgroundToImg Replaces tags with styles of `backgroud-image` by `<img />` tags. @@ -131,6 +148,8 @@ backgroundToImg(mixed $htmlContent) : object Returns a DOM object (even if provided a string). +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) + # extractFromDelimiters Extract the first part of a string matching the specified start and end delimiters. ```php @@ -151,6 +170,8 @@ $extracted = extractFromDelimiters($string, $start, $end); // 'John Doe' ``` +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) + # stripWithDelimiters Remove one or more part(s) of a string using a start and end delimiter. It is the inverse of `extractFromDelimiters`. @@ -173,6 +194,8 @@ $cleaned = stripWithDelimiters($string, $start, $end); // 'foobar' ``` +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) + # stripRecursiveHTMLSection Remove HTML sections containing one or more sections using the same HTML tag. @@ -192,6 +215,8 @@ $cleaned = stripRecursiveHTMLSection($string, $tag_name, $tag_start); // 'foobar' ``` +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) + # markdownToHtml Converts markdown input to HTML using [Parsedown](https://parsedown.org/). @@ -234,6 +259,7 @@ $html = markdownToHtml($input); // </ul> ``` +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) # e The `e` function is used to convert special characters to HTML entities @@ -244,7 +270,7 @@ e('0 < 1 and 2 > 1'); `e` will return the content of the string escape that can be rendered as is in HTML -[Defined in lib/html.php](/lib/html.php) +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) # truncate The `truncate` function is used to shorten a string if exceeds a certain length, and add a string indicating that the string has been shortened. @@ -253,7 +279,7 @@ The `truncate` function is used to shorten a string if exceeds a certain length, truncate('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed a neque nunc. Nam nibh sem.', 20 , '...'); ``` -[Defined in lib/html.php](/lib/html.php) +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) # sanitize The `sanitize` function is used to remove some tags from a given HTML text. @@ -272,7 +298,7 @@ sanitize($html, $tags_to_remove, $attributes_to_keep, $text_to_keep); This function returns a simplehtmldom object of the remaining contents. -[Defined in lib/html.php](/lib/html.php) +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) # convertLazyLoading The `convertLazyLoading` function is used to convert onvert lazy-loading images and frames (video embeds) into static elements. It accepts the HTML content as HTML objects or string objects. It returns the HTML content with fixed image/frame URLs (same type as input). @@ -286,8 +312,7 @@ $html = '<html> backgroundToImg($html); ``` -[Defined in lib/html.php](/lib/html.php) - +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) # Json::encode The `Json::encode` function is used to encode a value as à JSON string. @@ -300,7 +325,7 @@ $array = [ Json::encode($array, true, true); ``` -[Defined in lib/utils.php](/lib/utils.php) +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) # Json::decode The `Json::decode` function is used to decode a JSON string into à PHP variable. @@ -313,4 +338,4 @@ $json = '{ Json::decode($json); ``` -[Defined in lib/utils.php](/lib/utils.php) +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) From d5aa3aef699b400d5b40d61ee45c4476f0fcb38e Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Fri, 5 Apr 2024 11:31:30 +0200 Subject: [PATCH 071/423] [FDroidRepoBridge] Fix example repo The ttrss example/placeholder repo is offline, which fails CI jobs. Replace it with a healthy repo and package to get working CI tests and comparisons. --- bridges/FDroidRepoBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/FDroidRepoBridge.php b/bridges/FDroidRepoBridge.php index 286ada1b..b3fd146e 100644 --- a/bridges/FDroidRepoBridge.php +++ b/bridges/FDroidRepoBridge.php @@ -14,7 +14,7 @@ class FDroidRepoBridge extends BridgeAbstract 'name' => 'Repository URL', 'title' => 'Usually ends with /repo/', 'required' => true, - 'exampleValue' => 'https://srv.tt-rss.org/fdroid/repo' + 'exampleValue' => 'https://molly.im/fdroid/foss/fdroid/repo' ] ], 'Latest Updates' => [ @@ -35,7 +35,7 @@ class FDroidRepoBridge extends BridgeAbstract 'package' => [ 'name' => 'Package Identifier', 'required' => true, - 'exampleValue' => 'org.fox.ttrss' + 'exampleValue' => 'im.molly.app' ] ] ]; From b3ac1d176ce9f4778986190702216f0f25918a70 Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Fri, 5 Apr 2024 17:39:38 +0200 Subject: [PATCH 072/423] [FDroidRepoBridge] Simplify json retrieval (#4063) * [FDroidRepoBridge] Simplify json retrieval I looked into avoiding the writing-to-file and then reading-from-file altogether. Using a special file path that leaves the data in memory probably wouldn't work. But I'm unsure why we use the `index-v1.jar` file altogether. The main F-Droid repo [lists](https://f-droid.org/en/docs/All_our_APIs/#the-repo-index) not only `index-v1.jar` (which only makes sense if we were to use the contained signature, which we don't), but also `index-v1.json` and `index-v2.json`. These json files can be fetched with `getContents`, optionally cached, and directly fed into `Json::decode` without using a temporary file. The HTTP transfer encoding can compress the file to a similar degree the jar (=zip) can. That's exactly what this commit uses. Now the question is whether all the F-Droid repositories out there have this file. I went through the whole [list of known repositories](https://forum.f-droid.org/t/known-repositories/721) and only one repo misses the `index-v1.json` file: [Bromite](https://fdroid.bromite.org/fdroid/repo/index-v1.json). Under these circumstances we can depend on the availability of the `index-v1.json` file. Closes #4062 * [FDroidRepoBridge] Cleanup not requiring Zip With the last commit 1152386678151aeafd984061d34248023378bf64, the zip extension is not required anymore. Don't fail if it's not available. --- bridges/FDroidRepoBridge.php | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/bridges/FDroidRepoBridge.php b/bridges/FDroidRepoBridge.php index b3fd146e..844f6abb 100644 --- a/bridges/FDroidRepoBridge.php +++ b/bridges/FDroidRepoBridge.php @@ -45,10 +45,6 @@ class FDroidRepoBridge extends BridgeAbstract public function collectData() { - if (!extension_loaded('zip')) { - throw new \Exception('FDroidRepoBridge requires the php-zip extension'); - } - $this->repo = $this->fetchData(); switch ($this->queriedContext) { case 'Latest Updates': @@ -62,36 +58,11 @@ class FDroidRepoBridge extends BridgeAbstract } } - /** - * This method fetches data from arbitrary url and writes to os temp file. - * I don't think there's any security problem here but might be DOS problems. - */ private function fetchData() { $url = $this->getURI(); - - $zipFile = getContents($url . '/index-v1.jar'); - // On linux this creates a temp file in /tmp/ - $temporaryFile = tempnam(sys_get_temp_dir(), 'rssbridge_'); - file_put_contents($temporaryFile, $zipFile); - - $archive = new \ZipArchive(); - if ($archive->open($temporaryFile) !== true) { - unlink($temporaryFile); - throw new \Exception('Failed to extract archive'); - } - - $fp = $archive->getStream('index-v1.json'); - if (!$fp) { - unlink($temporaryFile); - throw new \Exception('Failed to get file pointer'); - } - - $json = stream_get_contents($fp); - fclose($fp); + $json = getContents($url . '/index-v1.json'); $data = Json::decode($json); - $archive->close(); - unlink($temporaryFile); return $data; } From 4602f4f475d938202de6c65214b321d41d0a93e8 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 6 Apr 2024 18:07:45 +0200 Subject: [PATCH 073/423] tweaks (#4065) --- bridges/FilterBridge.php | 2 +- bridges/TrelloBridge.php | 34 ++++++++++------------------------ bridges/TwitchBridge.php | 2 +- formats/HtmlFormat.php | 2 +- lib/BridgeAbstract.php | 9 +++++++++ lib/contents.php | 2 ++ 6 files changed, 24 insertions(+), 27 deletions(-) diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index 1add47f4..3448a8c7 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -77,7 +77,7 @@ class FilterBridge extends FeedExpander { $url = $this->getInput('url'); if (!Url::validate($url)) { - returnClientError('The url parameter must either refer to http or https protocol.'); + throw new \Exception('The url parameter must either refer to http or https protocol.'); } $this->collectExpandableDatas($this->getURI()); } diff --git a/bridges/TrelloBridge.php b/bridges/TrelloBridge.php index cab2bde2..42651fd1 100644 --- a/bridges/TrelloBridge.php +++ b/bridges/TrelloBridge.php @@ -553,10 +553,8 @@ class TrelloBridge extends BridgeAbstract private function queryAPI($path, $params = []) { - $data = json_decode(getContents('https://trello.com/1/' - . $path - . '?' - . http_build_query($params))); + $url = 'https://trello.com/1/' . $path . '?' . http_build_query($params); + $data = json_decode(getContents($url)); return $data; } @@ -576,33 +574,21 @@ class TrelloBridge extends BridgeAbstract && !$textOnly && isset($entity->originalUrl) ) { - $string = '<p><a href="' - . $entity->originalUrl - . '"><img src="' - . $entity->previewUrl - . '"></a></p>'; + $string = sprintf( + '<p><a href="%s"><img src="%s"></a></p>', + $entity->originalUrl, + $entity->previewUrl ?? '' + ); } elseif ($type === 'card' && !$textOnly) { - $string = '<a href="https://trello.com/c/' - . $entity->shortLink - . '">' - . $entity->text - . '</a>'; + $string = sprintf('<a href="https://trello.com/c/%s">%s</a>', $entity->shortLink, $entity->text); } elseif ($type === 'member' && !$textOnly) { - $string = '<a href="https://trello.com/' - . $entity->username - . '">' - . $entity->text - . '</a>'; + $string = sprintf('<a href="https://trello.com/%s">%s</a>', $entity->username, $entity->text); } elseif ($type === 'date') { $string = gmdate('M j, Y \a\t g:i A T', strtotime($entity->date)); } elseif ($type === 'translatable') { $string = self::ACTION_TEXTS[$entity->translationKey]; } else { - if (isset($entity->text)) { - $string = $entity->text; - } else { - $string = ''; - } + $string = $entity->text ?? ''; } $strings['{' . $entity_name . '}'] = $string; } diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index 9e70944e..424cd6e3 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -99,7 +99,7 @@ EOD; $user = $data->user; if ($user->videos === null) { // twitch regularly does this for unknown reasons - $this->logger->info('Twitch returned empty set of videos', ['data' => $data]); + //$this->logger->info('Twitch returned empty set of videos', ['data' => $data]); return; } diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index 1e2f60e6..37ef3a93 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -6,7 +6,7 @@ class HtmlFormat extends FormatAbstract public function stringify() { - // This query string comes in already url decoded + // This query string is url encoded $queryString = $_SERVER['QUERY_STRING']; $feedArray = $this->getFeed(); diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 1456e1c3..2467dec6 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -6,8 +6,17 @@ abstract class BridgeAbstract const URI = ''; const DONATION_URI = ''; const DESCRIPTION = 'No description provided'; + + /** + * Preferably a github username + */ const MAINTAINER = 'No maintainer'; + + /** + * Cache TTL in seconds + */ const CACHE_TIMEOUT = 3600; + const CONFIGURATION = []; const PARAMETERS = []; const TEST_DETECT_PARAMETERS = []; diff --git a/lib/contents.php b/lib/contents.php index 43db8c03..ba6dd531 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -17,6 +17,8 @@ function getContents( $httpClient = RssBridge::getHttpClient(); $cache = RssBridge::getCache(); + // TODO: consider url validation at this point + $httpHeadersNormalized = []; foreach ($httpHeaders as $httpHeader) { $parts = explode(':', $httpHeader); From 7d6881732dababa6a3188128e40227421cd8c972 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Sun, 7 Apr 2024 17:02:36 -0400 Subject: [PATCH 074/423] [ScribbleHubBridge] Add list page feed creation (#4012) * [ScribbleHubBridge] Add list page feed creation * [ScribbleHubBridge] Add list title handling * [ScribbleHubBridge] Don't include timestamp in List GUIDs * [ScribbleHubBridge] Fix usage of dynamic property --- bridges/ScribbleHubBridge.php | 60 ++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/bridges/ScribbleHubBridge.php b/bridges/ScribbleHubBridge.php index 0f7c7a6c..60add802 100644 --- a/bridges/ScribbleHubBridge.php +++ b/bridges/ScribbleHubBridge.php @@ -23,6 +23,14 @@ class ScribbleHubBridge extends FeedExpander // Example: latest chapters from Uskweirs 'exampleValue' => '965299', ], + ], + 'List' => [ + 'url' => [ + 'name' => 'url', + 'required' => true, + // Example: latest stories with the 'Transgender' tag + 'exampleValue' => 'https://www.scribblehub.com/series-finder/?sf=1&gi=6&tgi=1088&sort=dateadded', + ], ] ]; @@ -34,6 +42,10 @@ class ScribbleHubBridge extends FeedExpander public function collectData() { $url = 'https://rssscribblehub.com/rssfeed.php?type='; + if ($this->queriedContext === 'List') { + $this->collectList($this->getURI()); + return; + } if ($this->queriedContext === 'Author') { $url = $url . 'author&uid=' . $this->getInput('uid'); } else { //All and Series use the same source feed @@ -42,6 +54,44 @@ class ScribbleHubBridge extends FeedExpander $this->collectExpandableDatas($url); } + protected $author = ''; + + private function collectList($url) + { + $html = getSimpleHTMLDOMCached($url); + foreach ($html->find('.search_main_box') as $element) { + $item = []; + + $title = $element->find('.search_title a', 0); + $item['title'] = $title->plaintext; + $item['uri'] = $title->href; + + $strdate = $element->find('[title="Last Updated"]', 0)->plaintext; + $item['timestamp'] = strtotime($strdate); + $item['uid'] = $item['uri']; + + $details = getSimpleHTMLDOMCached($item['uri']); + $item['enclosures'][] = $details->find('.fic_image img', 0)->src; + $item['content'] = $details->find('.wi_fic_desc', 0); + + foreach ($details->find('.fic_genre') as $tag) { + $item['categories'][] = $tag->plaintext; + } + foreach ($details->find('.stag') as $tag) { + $item['categories'][] = $tag->plaintext; + } + + $read_url = $details->find('.read_buttons a', 0)->href; + $read_html = getSimpleHTMLDOMCached($read_url); + $item['content'] .= '<hr><h3>'; + $item['content'] .= $read_html->find('.chapter-title', 0); + $item['content'] .= '</h3>'; + $item['content'] .= $read_html->find('#chp_raw', 0); + + $this->items[] = $item; + } + } + protected function parseItem(array $item) { //For series, filter out other series from 'All' feed @@ -102,12 +152,17 @@ class ScribbleHubBridge extends FeedExpander } catch (HttpException $e) { // 403 Forbidden, This means we got anti-bot response if ($e->getCode() === 403) { - return $item; + return $name; } throw $e; } $title = html_entity_decode($page->find('.fic_title', 0)->plaintext); break; + case 'List': + $page = getSimpleHTMLDOMCached($this->getURI()); + $title = $page->find('head > title', 0)->plaintext; + $title = explode(' |', $title)[0]; + break; } if (isset($title)) { $name .= " - $title"; @@ -125,6 +180,9 @@ class ScribbleHubBridge extends FeedExpander case 'Series': $uri = self::URI . 'series/' . $this->getInput('sid') . '/a'; break; + case 'List': + $uri = $this->getInput('url'); + break; } return $uri; } From 815dc180ccd9298015a2e6c55aaf05a189259e6c Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:30:56 +0200 Subject: [PATCH 075/423] [PicukiBridge] Fix image URL (#4068) Image URL does not need to be faked anymore, as the content/type is now valid. --- bridges/PicukiBridge.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/bridges/PicukiBridge.php b/bridges/PicukiBridge.php index f1d45e2a..5f1096b8 100644 --- a/bridges/PicukiBridge.php +++ b/bridges/PicukiBridge.php @@ -89,9 +89,6 @@ class PicukiBridge extends BridgeAbstract $imageUrlParts[count($imageUrlParts) - 1] = urlencode($imageUrlParts[count($imageUrlParts) - 1]); $imageUrl = implode('/', $imageUrlParts); - // add fake file extension for it to be recognized as image/jpeg instead of application/octet-stream - $imageUrl = $imageUrl . '#.jpg'; - $this->items[] = [ 'uri' => $url, 'author' => $author, From a73b66f4d63766e42e6c772fe5d096e11de2a753 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 10 Apr 2024 18:32:48 +0200 Subject: [PATCH 076/423] fix(ScientificAmericanBridge) (#4070) --- bridges/ScientificAmericanBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/ScientificAmericanBridge.php b/bridges/ScientificAmericanBridge.php index d575bf94..da52e0ad 100644 --- a/bridges/ScientificAmericanBridge.php +++ b/bridges/ScientificAmericanBridge.php @@ -66,7 +66,7 @@ class ScientificAmericanBridge extends FeedExpander private function collectIssues() { $html = getSimpleHTMLDOMCached(self::ISSUES); - $content = $html->getElementById('content')->children(3); + $content = $html->getElementById('app')->children(3); $issues = $content->children(); $issues_count = min( (int)$this->getInput('parseIssues'), @@ -125,6 +125,7 @@ class ScientificAmericanBridge extends FeedExpander private function updateItem($item) { + return $item; $html = getSimpleHTMLDOMCached($item['uri']); $article = $html->find('#sa_body', 0)->find('article', 0); From 58c254ad3bd3f0c9a3e01dc8b4011996dffbaf75 Mon Sep 17 00:00:00 2001 From: Miika Launiainen <miika@miicat.eu> Date: Thu, 11 Apr 2024 18:18:37 +0300 Subject: [PATCH 077/423] [YorushikaBridge] Add language selection parameter (#4073) * Add language selection parameter * Fix typo * Fix lint errors --- bridges/YorushikaBridge.php | 50 ++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/bridges/YorushikaBridge.php b/bridges/YorushikaBridge.php index 12d02f1f..d528999a 100644 --- a/bridges/YorushikaBridge.php +++ b/bridges/YorushikaBridge.php @@ -7,6 +7,20 @@ class YorushikaBridge extends BridgeAbstract const DESCRIPTION = 'Return news from Yorushika\'s offical website'; const MAINTAINER = 'Miicat_47'; const PARAMETERS = [ + 'global' => [ + 'lang' => [ + 'name' => 'Language', + 'defaultValue' => 'jp', + 'type' => 'list', + 'values' => [ + '日本語' => 'jp', + 'English' => 'en', + '한국어' => 'ko', + '中文(繁體字)' => 'zh-tw', + '中文(簡体字)' => 'zh-cn', + ] + ], + ], 'All categories' => [ ], 'Only selected categories' => [ @@ -27,6 +41,28 @@ class YorushikaBridge extends BridgeAbstract public function collectData() { + $url = 'https://yorushika.com/news/5/'; + switch ($this->getInput('lang')) { + case 'jp': + $url = 'https://yorushika.com/news/5/'; + break; + case 'en': + $url = 'https://yorushika.com/news/5/?lang=en'; + break; + case 'ko': + $url = 'https://yorushika.com/news/5/?lang=ko'; + break; + case 'zh-tw': + $url = 'https://yorushika.com/news/5/?lang=zh-tw'; + break; + case 'zh-cn': + $url = 'https://yorushika.com/news/5/?lang=zh-cn'; + break; + default: + $url = 'https://yorushika.com/news/5/'; + break; + } + $categories = []; if ($this->queriedContext == 'All categories') { array_push($categories, 'all'); @@ -42,7 +78,7 @@ class YorushikaBridge extends BridgeAbstract } } - $html = getSimpleHTMLDOM('https://yorushika.com/news/5/')->find('.list--news', 0); + $html = getSimpleHTMLDOM($url)->find('.list--news', 0); $html = defaultLinkTo($html, $this->getURI()); foreach ($html->find('.inview') as $art) { @@ -62,10 +98,16 @@ class YorushikaBridge extends BridgeAbstract $url = $art->find('a.clearfix', 0)->href; // Get article date - $exp_date = '/\d+\.\d+\.\d+/'; $date = $art->find('.date', 0)->plaintext; - preg_match($exp_date, $date, $matches); - $date = date_create_from_format('Y.m.d', $matches[0]); + if (preg_match('/(\d)年(\d)月(\d)/', $date, $matches)) { + // Some dates will contain Chinese characters, remove those from the string + $formattedDate = sprintf('%d.%02d.%02d', $matches[1], $matches[2], $matches[3]); + } else { + // Assume the date is already in 'Y.m.d' format + preg_match('/\d+\.\d+\.\d+/', $date, $matches); + $formattedDate = $matches[0]; + } + $date = date_create_from_format('Y.m.d', $formattedDate); $date = date_format($date, 'd.m.Y'); // Get article info From 428c6c3c66b0f2730a8eb899d7a5768cdd279777 Mon Sep 17 00:00:00 2001 From: Korytov Pavel <thexcloud@gmail.com> Date: Fri, 12 Apr 2024 02:57:55 +0300 Subject: [PATCH 078/423] [ScientificAmericanBridge] Update bridge (#4074) * [ScientificAmericanBridge] Update bridge * [ScientificAmericanBridge] Fix lint --- bridges/ScientificAmericanBridge.php | 131 ++++++++++++--------------- 1 file changed, 60 insertions(+), 71 deletions(-) diff --git a/bridges/ScientificAmericanBridge.php b/bridges/ScientificAmericanBridge.php index da52e0ad..51cdc0d9 100644 --- a/bridges/ScientificAmericanBridge.php +++ b/bridges/ScientificAmericanBridge.php @@ -25,7 +25,7 @@ class ScientificAmericanBridge extends FeedExpander ]; const FEED = 'http://rss.sciam.com/ScientificAmerican-Global'; - const ISSUES = 'https://www.scientificamerican.com/archive/issues/2020s/'; + const ISSUES = 'https://www.scientificamerican.com/archive/issues/'; public function collectData() { @@ -50,7 +50,7 @@ class ScientificAmericanBridge extends FeedExpander if ($this->getInput('addContents') == 1) { usort($this->items, function ($item1, $item2) { - return $item1['timestamp'] - $item2['timestamp']; + return $item2['timestamp'] - $item1['timestamp']; }); } } @@ -66,8 +66,12 @@ class ScientificAmericanBridge extends FeedExpander private function collectIssues() { $html = getSimpleHTMLDOMCached(self::ISSUES); - $content = $html->getElementById('app')->children(3); - $issues = $content->children(); + $content = $html->getElementById('app'); + $issues_list = $content->find('div[class^="issue__list"]', 0); + if ($issues_list == null) { + return []; + } + $issues = $issues_list->find('div[class^="list__item"]'); $issues_count = min( (int)$this->getInput('parseIssues'), count($issues) @@ -87,36 +91,19 @@ class ScientificAmericanBridge extends FeedExpander $items = []; $html = getSimpleHTMLDOMCached($issue_link); - $features = $html->find('[class^=Detail_issue__article__previews__featured]', 0); - if ($features != null) { - $articles = $features->find('div', 0)->children(); + $blocks = $html->find('[class^="issueArchiveArticleListCompact"]'); + foreach ($blocks as $block) { + $articles = $block->find('article[class*="article"]'); foreach ($articles as $article) { - $h4 = $article->find('h4', 0); - $a = $h4->find('a', 0); + $a = $article->find('a[class^="articleLink"]', 0); $link = 'https://scientificamerican.com' . $a->getAttribute('href'); - $title = $a->plaintext; - $items[] = [ + $title = $a->find('h2[class^="articleTitle"]', 0); + array_push($items, [ 'uri' => $link, - 'title' => $title, + 'title' => $title->plaintext, 'uid' => $link, 'content' => '' - ]; - } - } - - $departments = $html->find('[class^=Detail_issue__article__previews__departments]', 0); - if ($departments != null) { - $headers = $departments->find('[class*=Listing_article__listing__title]'); - foreach ($headers as $header) { - $a = $header->find('a', 0); - $link = 'https://scientificamerican.com' . $a->getAttribute('href'); - $title = $a->plaintext; - $items[] = [ - 'uri' => $link, - 'title' => $title, - 'uid' => $link, - 'content' => '' - ]; + ]); } } @@ -125,65 +112,67 @@ class ScientificAmericanBridge extends FeedExpander private function updateItem($item) { - return $item; $html = getSimpleHTMLDOMCached($item['uri']); - $article = $html->find('#sa_body', 0)->find('article', 0); + $article = $html->find('#app', 0)->find('article', 0); - $time = $article->find('time[itemprop="datePublished"]', 0); - if ($time == null) { - $time = $html->find('span[itemprop="datePublished"]', 0); - } + $time = $article->find('p[class^="article_pub_date"]', 0); if ($time) { $datetime = DateTime::createFromFormat('F j, Y', $time->plaintext); + $datetime->setTime(0, 0, 0, 0); $item['timestamp'] = $datetime->format('U'); } - $main = $article->find('section.article-grid__main', 0); - if ($main == null) { - $main = $article->find('div.article-text', 0); + $authors = $article->find('a[class^="article_authors__link"]'); + if ($authors) { + $author = implode('; ', array_map(fn($a) => $a->plaintext, $authors)); + $item['author'] = $author; } - if ($main == null) { - return $item; + $res = ''; + $desc = $article->find('div[class^="article_dek"]', 0); + if ($desc) { + $res .= $desc->innertext; } - foreach ($main->find('img') as $img) { - $img->removeAttribute('width'); - $img->removeAttribute('height'); - $img->setAttribute('style', 'height: auto; width: auto; max-height: 768px'); + $lead_figure = $article->find('figure[class^="lead_image"]', 0); + if ($lead_figure) { + $res .= $lead_figure->outertext; } - $rights_link = $main->find('div.article-rightslink', 0); - if ($rights_link != null) { - $rights_link->parent->removeChild($rights_link); - } - $reprints_link = $main->find('div.article-reprintsLink', 0); - if ($reprints_link != null) { - $reprints_link->parent->removeChild($reprints_link); - } - $about_section = $main->find('section.article-author-container', 0); - if ($about_section != null) { - $about_section->parent->removeChild($about_section); - } - $read_next = $main->find('#read-next', 0); - if ($read_next != null) { - $read_next->parent->removeChild($read_next); + $content = $article->find('div[class^="article__content"]', 0); + if ($content) { + foreach ($content->children() as $block) { + if (str_contains($block->innertext, 'On supporting science journalism')) { + continue; + } + if ( + ($block->tag == 'p' && $block->getAttribute('data-block') == 'sciam/paragraph') + || ($block->tag == 'figure' && str_starts_with($block->class, 'article__image')) + ) { + $iframe = $block->find('iframe', 0); + if ($iframe) { + $res .= "<a href=\"{$iframe->src}\">{$iframe->src}</a>"; + } else { + $res .= $block->outertext; + } + } else if ($block->tag == 'h2') { + $res .= '<h3>' . $block->innertext . '</h3>'; + } else if ($block->tag == 'blockquote') { + $res .= $block->outertext; + } else if ($block->tag == 'hr' && $block->getAttribute('data-block') == 'sciam/raw_html') { + $res .= '<hr />'; + } + } } - foreach ($main->find('iframe') as $iframe) { - $a = $html->createElement('a'); - $a->href = $iframe->src; - $a->innertext = $iframe->src; - $iframe->parent->appendChild($a); - $iframe->parent->removeChild($iframe); + $footer = $article->find('footer[class*="footer"]', 0); + if ($footer) { + $bios = $footer->find('div[class^=bio]'); + $bio = implode('', array_map(fn($b) => $b->innertext, $bios)); + $res .= $bio; } - $authors = $main->find('span[itemprop="author"]', 0); - if ($authors != null) { - $item['author'] = $authors->plaintext; - } - - $item['content'] = $main->innertext; + $item['content'] = $res; return $item; } } From 89013faf7dc019db5ef42efb7021c7a517858bfd Mon Sep 17 00:00:00 2001 From: Arya K <arya@projectsegfau.lt> Date: Sat, 13 Apr 2024 19:29:25 +0530 Subject: [PATCH 079/423] Add Project Segfault Instance (#4076) --- docs/01_General/06_Public_Hosts.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index 1d59d918..1bc8fc20 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -24,6 +24,7 @@ | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rb.ash.fail | ![](https://img.shields.io/website/https/rb.ash.fail.svg) | [@ash](https://ash.fail/contact.html) | Hosted with Hostaris, Germany | ![](https://iplookup.flagfox.net/images/h16/UA.png) | https://rss.noleron.com | ![](https://img.shields.io/website/https/rss.noleron.com) | [@ihor](https://noleron.com/about) | Hosted with Hosting Ukraine, Ukraine +| ![](https://iplookup.flagfox.net/images/h16/IN.png) | https://rssbridge.projectsegfau.lt | ![](https://img.shields.io/website/https/rssbridge.projectsegfau.lt) | [@gi-yt](https://aryak.me) | Self-Hosted at Mumbai, India with Airtel (ISP) | ## Inactive instances From b4d397ff7064298f4ffb9afe74fa27b34d26d3c8 Mon Sep 17 00:00:00 2001 From: Miika Launiainen <miika@miicat.eu> Date: Sun, 14 Apr 2024 20:13:31 +0300 Subject: [PATCH 080/423] [YorushikaBridge] Fix getting date (#4077) * Remove unnecessary variable * Fix getting date --- bridges/YorushikaBridge.php | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/bridges/YorushikaBridge.php b/bridges/YorushikaBridge.php index d528999a..d75b97d7 100644 --- a/bridges/YorushikaBridge.php +++ b/bridges/YorushikaBridge.php @@ -41,7 +41,6 @@ class YorushikaBridge extends BridgeAbstract public function collectData() { - $url = 'https://yorushika.com/news/5/'; switch ($this->getInput('lang')) { case 'jp': $url = 'https://yorushika.com/news/5/'; @@ -99,14 +98,8 @@ class YorushikaBridge extends BridgeAbstract // Get article date $date = $art->find('.date', 0)->plaintext; - if (preg_match('/(\d)年(\d)月(\d)/', $date, $matches)) { - // Some dates will contain Chinese characters, remove those from the string - $formattedDate = sprintf('%d.%02d.%02d', $matches[1], $matches[2], $matches[3]); - } else { - // Assume the date is already in 'Y.m.d' format - preg_match('/\d+\.\d+\.\d+/', $date, $matches); - $formattedDate = $matches[0]; - } + preg_match('/(\d+)[\.年](\d+)[\.月](\d+)/u', $date, $matches); + $formattedDate = sprintf('%d.%02d.%02d', $matches[1], $matches[2], $matches[3]); $date = date_create_from_format('Y.m.d', $formattedDate); $date = date_format($date, 'd.m.Y'); From 957a820931dc45e9aadd996751cfb7321f4ba47e Mon Sep 17 00:00:00 2001 From: llamasblade <69692580+llamasblade@users.noreply.github.com> Date: Sun, 14 Apr 2024 17:14:52 +0000 Subject: [PATCH 081/423] [YandexZenBridge] Fix broken bridge for some channels (#4078) Fixes #4071. Major changes: - the bridge's URI changed from zen.yandex.com to dzen.ru, as the former redirects to the latter (perhaps the bridge's name should be changed as well); - the channel's URL is now required instead of the channel's username; - two kinds of URLs are supported, one for channels with usernames and one for channels with IDs in their URL; - the channel's real name, as shown in the webpage, is now used as the feed title. --- bridges/YandexZenBridge.php | 59 ++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/bridges/YandexZenBridge.php b/bridges/YandexZenBridge.php index 8a3db48b..57242328 100644 --- a/bridges/YandexZenBridge.php +++ b/bridges/YandexZenBridge.php @@ -3,17 +3,17 @@ class YandexZenBridge extends BridgeAbstract { const NAME = 'YandexZen Bridge'; - const URI = 'https://zen.yandex.com'; - const DESCRIPTION = 'Latest posts from the specified profile.'; + const URI = 'https://dzen.ru'; + const DESCRIPTION = 'Latest posts from the specified channel.'; const MAINTAINER = 'llamasblade'; const PARAMETERS = [ [ - 'username' => [ - 'name' => 'Username', + 'channelURL' => [ + 'name' => 'Channel URL', 'type' => 'text', 'required' => true, - 'title' => 'The account\'s username, found in its URL', - 'exampleValue' => 'dream_faity_diy', + 'title' => 'The channel\'s URL', + 'exampleValue' => 'https://dzen.ru/dream_faity_diy', ], 'limit' => [ 'name' => 'Limit', @@ -27,14 +27,41 @@ class YandexZenBridge extends BridgeAbstract ]; # credit: https://github.com/teromene see #1032 - const _API_URL = 'https://zen.yandex.ru/api/v3/launcher/more?channel_name='; + const _BASE_API_URL_WITH_CHANNEL_NAME = 'https://dzen.ru/api/v3/launcher/more?channel_name='; + const _BASE_API_URL_WITH_CHANNEL_ID = 'https://dzen.ru/api/v3/launcher/more?channel_id='; + + const _ACCOUNT_URL_WITH_CHANNEL_ID_REGEX = '#^https?://dzen\.ru/id/(?<channelID>[a-z0-9]{24})#'; + const _ACCOUNT_URL_WITH_CHANNEL_NAME_REGEX = '#^https?://dzen\.ru/(?<channelName>[\w\.]+)#'; + + private $channelRealName = null; # as shown in the webpage, not in the URL + public function collectData() { - $profile_json = json_decode(getContents($this->getAPIUrl())); + $channelURL = $this->getInput('channelURL'); + + if (preg_match(self::_ACCOUNT_URL_WITH_CHANNEL_ID_REGEX, $channelURL, $matches)) { + $channelID = $matches['channelID']; + $channelAPIURL = self::_BASE_API_URL_WITH_CHANNEL_ID . $channelID; + } elseif (preg_match(self::_ACCOUNT_URL_WITH_CHANNEL_NAME_REGEX, $channelURL, $matches)) { + $channelName = $matches['channelName']; + $channelAPIURL = self::_BASE_API_URL_WITH_CHANNEL_NAME . $channelName; + } else { + returnClientError(<<<EOT +Invalid channel URL provided. +The channel\'s URL must be in one of these two forms: +- https://dzen.ru/dream_faity_diy +- https://dzen.ru/id/5ad7777f1aa80ce576015250 +EOT); + } + + $APIResponse = json_decode(getContents($channelAPIURL)); + + $this->channelRealName = $APIResponse->header->title; + $limit = $this->getInput('limit'); - foreach (array_slice($profile_json->items, 0, $limit) as $post) { + foreach (array_slice($APIResponse->items, 0, $limit) as $post) { $item = []; $item['uri'] = $post->share_link; @@ -56,21 +83,19 @@ class YandexZenBridge extends BridgeAbstract } } - private function getAPIUrl() - { - return self::_API_URL . $this->getInput('username'); - } - public function getURI() { - return self::URI . '/' . $this->getInput('username'); + if (is_null($this->getInput('channelURL'))) { + return parent::getURI(); + } + return $this->getInput('channelURL'); } public function getName() { - if (is_null($this->getInput('username'))) { + if (is_null($this->channelRealName)) { return parent::getName(); } - return $this->getInput('username') . '\'s latest zen.yandex posts'; + return $this->channelRealName . '\'s latest zen.yandex posts'; } } From 97f5dafbc5e2ff9dadbe8a76cb7e07818da566e8 Mon Sep 17 00:00:00 2001 From: llamasblade <69692580+llamasblade@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:58:05 +0000 Subject: [PATCH 082/423] [HytaleBridge] Fix bridge not pulling all blog posts (#4079) --- bridges/HytaleBridge.php | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bridges/HytaleBridge.php b/bridges/HytaleBridge.php index 7ca11af6..01fc0f38 100644 --- a/bridges/HytaleBridge.php +++ b/bridges/HytaleBridge.php @@ -18,26 +18,27 @@ class HytaleBridge extends BridgeAbstract $blogPosts = json_decode(getContents(self::_API_URL_PUBLISHED)); $length = count($blogPosts); - for ($i = 1; $i < $length; $i += 3) { + for ($i = 0; $i < $length; $i += 3) { $slug = $blogPosts[$i]->slug; $blogPost = json_decode(getContents(self::_API_URL_BLOG_POST . $slug)); - if (property_exists($blogPost, 'previous')) { - $this->addBlogPost($blogPost->previous); + if (property_exists($blogPost, 'next')) { + $this->addBlogPost($blogPost->next); } $this->addBlogPost($blogPost); - if (property_exists($blogPost, 'next')) { - $this->addBlogPost($blogPost->next); + if (property_exists($blogPost, 'previous')) { + $this->addBlogPost($blogPost->previous); } } - if ($length % 3 == 1) { - $slug = $blogPosts[count($blogPosts) - 1]->slug; + if (($length >= 3) && ($length % 3 == 0)) { + $slug = $blogPosts[$length - 1]->slug; $blogPost = json_decode(getContents(self::_API_URL_BLOG_POST . $slug)); + $this->addBlogPost($blogPost); } } From 8c3e973b9f508f3ea928a720a616d4d95d71d22b Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Thu, 18 Apr 2024 01:43:53 +0200 Subject: [PATCH 083/423] [PepperBridgeAbstract] Fix the "no result" detection (#4082) The "no result" test did not work, it is fixed now. --- bridges/PepperBridgeAbstract.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 33d427bc..6e41cf20 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -55,8 +55,8 @@ class PepperBridgeAbstract extends BridgeAbstract ); // If there is no results, we don't parse the content because it display some random deals - $noresult = $html->find('section[class=subNav]', 0)->find('div[class*=page-center listLayout aGrid]', 0); - if ($noresult === null) { + $noresult = $html->find('div[id=content-list]', 0)->find('h2', 0); + if ($noresult !== null) { $this->items = []; } else { foreach ($list as $deal) { From 1f71d76ac1e93ef403098eec3c7a864fe8748ee6 Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:18:45 +0200 Subject: [PATCH 084/423] [HeiseBridge] Remove additional ad banners For example https://www.heise.de/meinung/Kommentar-Microsofts-Sicherheitspraxis-wird-zur-Gefahr-und-das-BSI-schweigt-9686629.html has two inline banners for a heise offering, not directly related to the article. Removing all "inline" figures, which seems to catch all inline unwanted elements, while avoiding removing useful figures/images. --- bridges/HeiseBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index ab40e6d9..a78b4609 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -160,7 +160,10 @@ class HeiseBridge extends FeedExpander $article = defaultLinkTo($article, $item['uri']); // remove unwanted stuff - foreach ($article->find('figure.branding, a-ad, div.ho-text, a-img, .a-toc__list, a-collapse, .opt-in__description, .opt-in__footnote') as $element) { + foreach ( + $article->find('figure.branding, figure.a-inline-image, a-ad, div.ho-text, a-img, + .a-toc__list, a-collapse, .opt-in__description, .opt-in__footnote') as $element + ) { $element->remove(); } // reload html, as remove() is buggy From 154b8b9cdb6ad7f125753c5e0964871c1730e8f8 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Fri, 19 Apr 2024 19:08:58 +0200 Subject: [PATCH 085/423] Create TarnkappeBridge.php (#4085) * Create TarnkappeBridge.php * Update TarnkappeBridge.php --- bridges/TarnkappeBridge.php | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 bridges/TarnkappeBridge.php diff --git a/bridges/TarnkappeBridge.php b/bridges/TarnkappeBridge.php new file mode 100644 index 00000000..c04c9546 --- /dev/null +++ b/bridges/TarnkappeBridge.php @@ -0,0 +1,79 @@ +<?php + +class TarnkappeBridge extends FeedExpander +{ + const MAINTAINER = 'Tone866'; + const NAME = 'tarnkappe Bridge'; + const URI = 'https://tarnkappe.info/'; + const CACHE_TIMEOUT = 1800; // 30min + const DESCRIPTION = 'Returns the full articles instead of only the intro'; + const PARAMETERS = [[ + 'category' => [ + 'name' => 'Category', + 'required' => false, + 'title' => <<<'TITLE' + If you only want to subscribe to a specific category + you can enter it here. + If not, leave it blank to subscribe to everything. + TITLE, + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify number of full articles to return', + 'defaultValue' => 10 + ] + ]]; + const LIMIT = 10; + + public function collectData() + { + if (empty($this->getInput('category'))) { + $category = 'https://tarnkappe.info/feed'; + } else { + $category = 'https://tarnkappe.info/artikel/' . $this->getInput('category') . '/feed'; + } + + $this->collectExpandableDatas( + $category, + $this->getInput('limit') ?: static::LIMIT + ); + } + + protected function parseItem(array $item) + { + if (strpos($item['uri'], 'https://tarnkappe.info/') !== 0) { + return $item; + } + + $article = getSimpleHTMLDOMCached($item['uri']); + + if ($article) { + $article = defaultLinkTo($article, $item['uri']); + $item = $this->addArticleToItem($item, $article); + } + + return $item; + } + + private function addArticleToItem($item, $article) + { + $item['content'] = $article->find('a.image-header', 0); + + $article = $article->find('main#article article div.card-content div.content.entry-content', 0); + + // remove unwanted stuff + foreach ( + $article->find('em, section, div.menu') as $element + ) { + $element->remove(); + } + // reload html, as remove() is buggy + $article = str_get_html($article->outertext); + + $item['content'] .= $article; + + return $item; + } +} From d31f20758cb9c3a0841c4cf1c2c283b4cfc29077 Mon Sep 17 00:00:00 2001 From: Thomas <mightymt@users.noreply.github.com> Date: Fri, 26 Apr 2024 18:47:06 +0200 Subject: [PATCH 086/423] [YouTubeCommunityTabBridge] Improve building of content & title (#4089) * [YouTubeCommunityTabBridge] Improve building of content & title Fixes truncated link hrefs in content and adds some general improvements regarding the building of item content and item title * [YouTubeCommunityTabBridge] Fix PHP deprecation warnings Fixes the following deprecation warnings: substr(): Passing null to parameter #1 ($string) of type string is deprecated --- bridges/YouTubeCommunityTabBridge.php | 37 ++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/bridges/YouTubeCommunityTabBridge.php b/bridges/YouTubeCommunityTabBridge.php index 20822828..74200b17 100644 --- a/bridges/YouTubeCommunityTabBridge.php +++ b/bridges/YouTubeCommunityTabBridge.php @@ -204,7 +204,15 @@ class YouTubeCommunityTabBridge extends BridgeAbstract $text = ''; foreach ($runs as $part) { - $text .= $this->formatUrls($part->text); + if (isset($part->navigationEndpoint->browseEndpoint->canonicalBaseUrl)) { + $text .= $this->formatUrls($part->text, $part->navigationEndpoint->browseEndpoint->canonicalBaseUrl); + } elseif (isset($part->navigationEndpoint->urlEndpoint->url)) { + $text .= $this->formatUrls($part->text, $part->navigationEndpoint->urlEndpoint->url); + } elseif (isset($part->navigationEndpoint->commandMetadata->webCommandMetadata->url)) { + $text .= $this->formatUrls($part->text, $part->navigationEndpoint->commandMetadata->webCommandMetadata->url); + } else { + $text .= $this->formatUrls($part->text, null); + } } return nl2br($text); @@ -275,6 +283,7 @@ EOD; { $length = 100; + $text = strip_tags($text); if (strlen($text) > $length) { $text = explode('<br>', wordwrap($text, $length, '<br>')); return $text[0] . '...'; @@ -283,12 +292,26 @@ EOD; return $text; } - private function formatUrls($content) + private function formatUrls($content, $url) { - return preg_replace( - '/(http[s]{0,1}\:\/\/[a-zA-Z0-9.\/\?\&=\-_]{4,})/ims', - '<a target="_blank" href="$1" target="_blank">$1</a> ', - $content - ); + if (substr(strval($url), 0, 1) == '/') { + // fix relative URL + $url = 'https://www.youtube.com' . $url; + } elseif (substr(strval($url), 0, 33) == 'https://www.youtube.com/redirect?') { + // extract actual URL from YouTube redirect + parse_str(substr($url, 33), $params); + if (strpos(($params['q'] ?? ''), rtrim($content, '.')) === 0) { + $url = $params['q']; + } + } + + // ensure all URLs are made clickable + $url = $url ?? $content; + + if (filter_var($url, FILTER_VALIDATE_URL)) { + return '<a href="' . $url . '" target="_blank">' . $content . '</a>'; + } + + return $content; } } From f3ca567159386f32ef26c424b14d9b6e2acf126a Mon Sep 17 00:00:00 2001 From: Korytov Pavel <thexcloud@gmail.com> Date: Sat, 27 Apr 2024 11:35:59 +0300 Subject: [PATCH 087/423] [TldrTechBridge] Fix and improve bridge (#4090) --- bridges/TldrTechBridge.php | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/bridges/TldrTechBridge.php b/bridges/TldrTechBridge.php index 984117b2..d2955347 100644 --- a/bridges/TldrTechBridge.php +++ b/bridges/TldrTechBridge.php @@ -22,11 +22,15 @@ class TldrTechBridge extends BridgeAbstract 'type' => 'list', 'values' => [ 'Tech' => 'tech', - 'Crypto' => 'crypto', + 'Web Dev' => 'webdev', 'AI' => 'ai', - 'Web Dev' => 'engineering', + 'Information Security' => 'infosec', + 'Product Management' => 'product', + 'DevOps' => 'devops', + 'Crypto' => 'crypto', + 'Design' => 'design', + 'Marketing' => 'marketing', 'Founders' => 'founders', - 'Cybersecurity' => 'cybersecurity' ], 'defaultValue' => 'tech' ] @@ -48,12 +52,17 @@ class TldrTechBridge extends BridgeAbstract // Convert /<topic>/2023-01-01 to unix timestamp $date_items = explode('/', $child->href); $date = strtotime(end($date_items)); - $this->items[] = [ - 'uri' => self::URI . $child->href, - 'title' => $child->plaintext, - 'timestamp' => $date, - 'content' => $this->extractContent(self::URI . $child->href), - ]; + $item_url = self::URI . ltrim($child->href, '/'); + try { + $this->items[] = [ + 'uri' => self::URI . $child->href, + 'title' => $child->plaintext, + 'timestamp' => $date, + 'content' => $this->extractContent($item_url), + ]; + } catch (HttpException $e) { + continue; + } $added++; if ($added >= $limit) { break; @@ -66,7 +75,7 @@ class TldrTechBridge extends BridgeAbstract $html = getSimpleHTMLDOM($url); $content = $html->find('div.content-center.mt-5', 0); if (!$content) { - return ''; + throw new HttpException('Could not find content', 500); } $subscribe_form = $content->find('div.mt-5 > div > form', 0); if ($subscribe_form) { From d15960f955be91c8fa236d96186fcb0524572859 Mon Sep 17 00:00:00 2001 From: Thomas <mightymt@users.noreply.github.com> Date: Thu, 2 May 2024 19:45:04 +0200 Subject: [PATCH 088/423] [YouTubeCommunityTabBridge] Multi-image attachment support (#4091) Adds support for multi-image attachments. Also changes individual if-statments in "getAttachments" to if/elseif as each post can apparently only have one attachment anyway. --- bridges/YouTubeCommunityTabBridge.php | 31 +++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/bridges/YouTubeCommunityTabBridge.php b/bridges/YouTubeCommunityTabBridge.php index 74200b17..0c145c02 100644 --- a/bridges/YouTubeCommunityTabBridge.php +++ b/bridges/YouTubeCommunityTabBridge.php @@ -228,8 +228,8 @@ class YouTubeCommunityTabBridge extends BridgeAbstract if (isset($details->backstageAttachment)) { $attachments = $details->backstageAttachment; - // Video if (isset($attachments->videoRenderer) && isset($attachments->videoRenderer->videoId)) { + // Video if (empty($this->itemTitle)) { $this->itemTitle = $this->feedName . ' posted a video'; } @@ -238,10 +238,8 @@ class YouTubeCommunityTabBridge extends BridgeAbstract <iframe width="100%" height="410" src="https://www.youtube.com/embed/{$attachments->videoRenderer->videoId}" frameborder="0" allow="encrypted-media;" allowfullscreen></iframe> EOD; - } - - // Image - if (isset($attachments->backstageImageRenderer)) { + } elseif (isset($attachments->backstageImageRenderer)) { + // Image if (empty($this->itemTitle)) { $this->itemTitle = $this->feedName . ' posted an image'; } @@ -251,10 +249,8 @@ EOD; $content = <<<EOD <p><img src="{$lastThumb->url}"></p> EOD; - } - - // Poll - if (isset($attachments->pollRenderer)) { + } elseif (isset($attachments->pollRenderer)) { + // Poll if (empty($this->itemTitle)) { $this->itemTitle = $this->feedName . ' posted a poll'; } @@ -270,6 +266,23 @@ EOD; $content = <<<EOD <hr><p>Poll ({$attachments->pollRenderer->totalVotes->simpleText})<br><ul>{$pollChoices}</ul><p> EOD; + } elseif (isset($attachments->postMultiImageRenderer->images)) { + // Multiple images + $images = $attachments->postMultiImageRenderer->images; + + if (is_array($images)) { + if (empty($this->itemTitle)) { + $this->itemTitle = $this->feedName . ' posted ' . count($images) . ' images'; + } + + foreach ($images as $image) { + $lastThumb = end($image->backstageImageRenderer->image->thumbnails); + + $content .= <<<EOD +<p><img src="{$lastThumb->url}"></p> +EOD; + } + } } } From f48020982530ff08636a5233021014c16fc67c36 Mon Sep 17 00:00:00 2001 From: Eugene Molotov <eugene.molotov@yandex.ru> Date: Mon, 6 May 2024 02:30:23 +0500 Subject: [PATCH 089/423] [YoutubeBridge] Fix empty result in search feed (#4098) --- bridges/YoutubeBridge.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 6a29e387..af14c856 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -193,14 +193,7 @@ class YoutubeBridge extends BridgeAbstract $html = $this->fetch($url_listing); $jsonData = $this->extractJsonFromHtml($html); $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; - $jsonData = $jsonData->sectionListRenderer->contents; - foreach ($jsonData as $data) { - // Search result includes some ads, have to filter them - if (isset($data->itemSectionRenderer->contents[0]->videoRenderer)) { - $jsonData = $data->itemSectionRenderer->contents; - break; - } - } + $jsonData = $jsonData->sectionListRenderer->contents[0]->itemSectionRenderer->contents; $this->fetchItemsFromFromJsonData($jsonData); $this->feeduri = $url_listing; $this->feedName = 'Search: ' . $search; From d11b7f77540760bf115b724dd06033ca6886485e Mon Sep 17 00:00:00 2001 From: Patrick <jummo4@yahoo.de> Date: Sun, 5 May 2024 23:30:38 +0200 Subject: [PATCH 090/423] Change URI for St. Johannes Blick (#4099) Co-authored-by: Patrick <jummo@mailbox.org> --- bridges/JohannesBlickBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/JohannesBlickBridge.php b/bridges/JohannesBlickBridge.php index 72583a53..80ca9a71 100644 --- a/bridges/JohannesBlickBridge.php +++ b/bridges/JohannesBlickBridge.php @@ -3,7 +3,7 @@ class JohannesBlickBridge extends BridgeAbstract { const NAME = 'Johannes Blick'; - const URI = 'https://www.st-johannes-baptist.de/index.php/unsere-medien/johannesblick-archiv'; + const URI = 'https://www.st-johannes-baptist.de/index.php/medien-und-downloads/archiv-johannesblick'; const DESCRIPTION = 'RSS feed for Johannes Blick'; const MAINTAINER = 'jummo4@yahoo.de'; From 1c3024fca7b8330b75e93fde673af09b1d797fa5 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca <facu@tuesca.com> Date: Wed, 8 May 2024 00:25:45 +0200 Subject: [PATCH 091/423] [MangaReaderBridge] Change feed title to manga name (#4092) --- bridges/MangaReaderBridge.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/bridges/MangaReaderBridge.php b/bridges/MangaReaderBridge.php index 1fa0c62d..1b8e765b 100644 --- a/bridges/MangaReaderBridge.php +++ b/bridges/MangaReaderBridge.php @@ -26,11 +26,26 @@ class MangaReaderBridge extends BridgeAbstract ] ]; + protected $feedName = ''; + + + public function getName() + { + if (empty($this->feedName)) { + return parent::getName(); + } else { + return $this->feedName; + } + } + public function collectData() { $url = $this->getInput('url'); $lang = $this->getInput('lang'); $dom = getSimpleHTMLDOM($url); + $aniDetail = $dom->getElementById('ani_detail'); + $this->feedName = html_entity_decode($aniDetail->find('h2', 0)->plaintext); + $chapters = $dom->getElementById($lang . '-chapters'); foreach ($chapters->getElementsByTagName('li') as $chapter) { From 776ee233bdc22833815ece7d87140a4c64aa540c Mon Sep 17 00:00:00 2001 From: Alex Balgavy <8124851+thezeroalpha@users.noreply.github.com> Date: Sun, 12 May 2024 20:30:23 +0200 Subject: [PATCH 092/423] [NOSBridge] fix bridge (#4102) CSS selectors were no longer valid. --- bridges/NOSBridge.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/bridges/NOSBridge.php b/bridges/NOSBridge.php index 33cad40b..60a560aa 100644 --- a/bridges/NOSBridge.php +++ b/bridges/NOSBridge.php @@ -14,7 +14,7 @@ class NOSBridge extends BridgeAbstract 'name' => 'Onderwerp', 'title' => 'Kies onderwerp', 'values' => [ - 'Laatste nieuws' => 'nieuws', + 'Laatste nieuws' => 'nieuws/laatste', 'Binnenland' => 'nieuws/binnenland', 'Buitenland' => 'nieuws/buitenland', 'Regionaal nieuws' => 'nieuws/regio', @@ -38,17 +38,16 @@ class NOSBridge extends BridgeAbstract { $url = sprintf('https://www.nos.nl/%s', $this->getInput('topic')); $dom = getSimpleHTMLDOM($url); - $dom = $dom->find('ul.list-items', 0); + $dom = $dom->find('main#content > div > section > ul', 0); if (!$dom) { throw new \Exception(sprintf('Unable to find css selector on `%s`', $url)); } $dom = defaultLinkTo($dom, $this->getURI()); - foreach ($dom->find('li.list-items__item') as $article) { - $a = $article->find('a', 0); + foreach ($dom->find('li') as $article) { $this->items[] = [ - 'title' => $article->find('h3.list-items__title', 0)->plaintext, - 'uri' => $article->find('a.list-items__link', 0)->href, - 'content' => $article->find('p.list-items__description', 0)->plaintext, + 'title' => $article->find('h2', 0)->plaintext, + 'uri' => $article->find('a', 0)->href, + 'content' => $article->find('p', 0)->plaintext, 'timestamp' => strtotime($article->find('time', 0)->datetime), ]; } From 494990086335bfa498f0ba7497765c09f3ff7570 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Sun, 12 May 2024 15:45:14 -0400 Subject: [PATCH 093/423] [ScribbleHubBridge] Handle 429 errors and use consistent GUID (#4104) --- bridges/ScribbleHubBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/ScribbleHubBridge.php b/bridges/ScribbleHubBridge.php index 60add802..b4f7beaa 100644 --- a/bridges/ScribbleHubBridge.php +++ b/bridges/ScribbleHubBridge.php @@ -107,12 +107,13 @@ class ScribbleHubBridge extends FeedExpander } $item['comments'] = $item['uri'] . '#comments'; + $item['uid'] = $item['uri']; try { $dom = getSimpleHTMLDOMCached($item['uri']); } catch (HttpException $e) { // 403 Forbidden, This means we got anti-bot response - if ($e->getCode() === 403) { + if ($e->getCode() === 403 || $e->getCode() === 429) { return $item; } throw $e; @@ -134,7 +135,6 @@ class ScribbleHubBridge extends FeedExpander //Generate UID $item_pid = $dom->find('#mypostid', 0)->value; - $item['uid'] = $item_sid . "/$item_pid"; return $item; } From 6e2aeda61d4aefb1106be2cef63cc775ee4f300d Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Sun, 12 May 2024 15:46:07 -0400 Subject: [PATCH 094/423] [GameBananaBridge] Include update contents in feed (#4103) * [GameBananaBridge] Include update contents in feed * [GameBananaBridge] Fix dynamic title property --- bridges/GameBananaBridge.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bridges/GameBananaBridge.php b/bridges/GameBananaBridge.php index 591ac0e9..9a0a0686 100644 --- a/bridges/GameBananaBridge.php +++ b/bridges/GameBananaBridge.php @@ -28,6 +28,8 @@ class GameBananaBridge extends BridgeAbstract return 'https://images.gamebanana.com/static/img/favicon/favicon.ico'; } + private $title; + public function collectData() { $url = 'https://api.gamebanana.com/Core/List/New?itemtype=Mod&page=1&gameid=' . $this->getInput('gid'); @@ -38,7 +40,7 @@ class GameBananaBridge extends BridgeAbstract $json_list = json_decode($api_response, true); // Get first page mod list $url = 'https://api.gamebanana.com/Core/Item/Data?itemtype[]=Game&fields[]=name&itemid[]=' . $this->getInput('gid'); - $fields = 'name,Owner().name,text,screenshots,Files().aFiles(),date,Url().sProfileUrl(),udate'; + $fields = 'name,Owner().name,text,screenshots,Files().aFiles(),date,Url().sProfileUrl(),udate,Updates().aLatestUpdates()'; foreach ($json_list as $element) { // Build api request to minimize API calls $mid = $element[1]; $url .= '&itemtype[]=Mod&fields[]=' . $fields . '&itemid[]=' . $mid; @@ -72,6 +74,10 @@ class GameBananaBridge extends BridgeAbstract foreach ($img_list as $img_element) { $item['content'] .= '<img src="https://images.gamebanana.com/img/ss/mods/' . $img_element['_sFile'] . '"/>'; } + if ($this->getInput('updates') && sizeof($element[8]) > 0) { + $item['content'] .= '<br><strong>Update: ' . $element[8][0]['_sTitle']; + $item['content'] .= '</strong><br>' . $element[8][0]['_sText'] . '<hr>'; + } $item['content'] .= '<br>' . $element[2]; $item['uid'] = $item['uri'] . $item['title'] . $item['timestamp']; From b785a4b64ee7fb295b845a59bd9d573a3e775e3e Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Fri, 17 May 2024 15:29:17 -0400 Subject: [PATCH 095/423] ArsTechnicaBridge: restore categories lost by FeedExpander (#4030) --- bridges/ArsTechnicaBridge.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index 2c631871..fcb1bd4f 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -39,6 +39,10 @@ class ArsTechnicaBridge extends FeedExpander $item_html = defaultLinkTo($item_html, self::URI); $item['content'] = $item_html->find('.article-content', 0); + $parsely = $item_html->find('[name="parsely-page"]', 0); + $parsely_json = json_decode(html_entity_decode($parsely->content), true); + $item['categories'] = $parsely_json['tags']; + $pages = $item_html->find('nav.page-numbers > .numbers > a', -2); if (null !== $pages) { for ($i = 2; $i <= $pages->innertext; $i++) { From a7ed3d56f9b8ea3194f78925e3d339f933aa726e Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sat, 18 May 2024 15:46:53 +0200 Subject: [PATCH 096/423] [ZeitBridge] Prettify author field By removing HTML tags (plaintext) and trimming it. --- bridges/ZeitBridge.php | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index b9806e5a..07b8e70c 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -108,12 +108,9 @@ class ZeitBridge extends FeedExpander } // authors - $authors = $article->find('*[itemtype*="schema.org/Person"]'); - if (!$authors) { - $authors = $article->find('.metadata__source'); - } + $authors = $article->find('*[itemtype*="schema.org/Person"]') ?? $article->find('.metadata__source'); if ($authors) { - $item['author'] = implode(', ', $authors); + $item['author'] = implode(', ', array_map(function ($e) { return trim($e->plaintext); }, $authors)); } // header image From 4d12aa2a9ee03b32b270a4a0930d7e9cef0515ca Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sat, 18 May 2024 16:11:26 +0200 Subject: [PATCH 097/423] [ZeitBridge] Remove annoyances, add content Remove navigational elements, podcast images. Add many more header images, article content in <ul> (and for ggod measure in <ol>) and quotes with their content and not only their author. Extreme example: https://www.zeit.de/campus/2024-05/protest-palaestina-universitaet-europa-uebersicht --- bridges/ZeitBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index 07b8e70c..d79f7e7c 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -87,7 +87,7 @@ class ZeitBridge extends FeedExpander // remove known bad elements foreach ( $article->find( - 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge, .article-heading__container--podcast, div[data-paywall], .js-embed-consent' + 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge, .article-heading__container--podcast, .podcast-player__image, div[data-paywall], .js-embed-consent, script, nav, .article-flexible-toc__subheading-link, .faq-link' ) as $bad ) { $bad->remove(); @@ -114,7 +114,7 @@ class ZeitBridge extends FeedExpander } // header image - $headerimg = $article->find('*[data-ct-row="headerimage"]', 0) ?? $article->find('header', 0); + $headerimg = $article->find('*[data-ct-row="headerimage"]', 0) ?? $article->find('.article-header', 0) ?? $article->find('header', 0); if ($headerimg) { $item['content'] .= implode('', $headerimg->find('img[src], figcaption')); } @@ -124,7 +124,7 @@ class ZeitBridge extends FeedExpander if ($pages) { foreach ($pages as $page) { - $elements = $page->find('p, h2, figcaption, img[src]'); + $elements = $page->find('p, ul, ol, h2, figure.article__media img[src], figure.article__media figcaption, figure.quote'); $item['content'] .= implode('', $elements); } } From 7bde7a56f95a50f0394a1efb45cbbd80af643f9f Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sat, 18 May 2024 16:18:23 +0200 Subject: [PATCH 098/423] [ZeitBridge] Fix linting --- bridges/ZeitBridge.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index d79f7e7c..ae8a1a66 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -87,7 +87,9 @@ class ZeitBridge extends FeedExpander // remove known bad elements foreach ( $article->find( - 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge, .article-heading__container--podcast, .podcast-player__image, div[data-paywall], .js-embed-consent, script, nav, .article-flexible-toc__subheading-link, .faq-link' + 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge, + .article-heading__container--podcast, .podcast-player__image, div[data-paywall], + .js-embed-consent, script, nav, .article-flexible-toc__subheading-link, .faq-link' ) as $bad ) { $bad->remove(); @@ -110,7 +112,9 @@ class ZeitBridge extends FeedExpander // authors $authors = $article->find('*[itemtype*="schema.org/Person"]') ?? $article->find('.metadata__source'); if ($authors) { - $item['author'] = implode(', ', array_map(function ($e) { return trim($e->plaintext); }, $authors)); + $item['author'] = implode(', ', array_map(function ($e) { + return trim($e->plaintext); + }, $authors)); } // header image From 75f35391fa3399f230d425a33a6836ccf07340ff Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sat, 18 May 2024 16:51:00 +0200 Subject: [PATCH 099/423] [HeiseBridge] Add missing <ol> elements (#4110) The following article has <ol> elements that were missing. Adding them to have the full content. https://heise.de/-9714438 --- bridges/HeiseBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index a78b4609..9aa7209d 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -214,7 +214,7 @@ class HeiseBridge extends FeedExpander $content = $article->find('.article-content', 0); if ($content) { $contentElements = $content->find( - 'p, h3, ul, table, pre, noscript img, a-bilderstrecke h2, a-bilderstrecke figure, a-bilderstrecke figcaption' + 'p, h3, ul, ol, table, pre, noscript img, a-bilderstrecke h2, a-bilderstrecke figure, a-bilderstrecke figcaption' ); $item['content'] .= implode('', $contentElements); } From dc199ebf5c134960c59bceb233afff0379062a77 Mon Sep 17 00:00:00 2001 From: Albert Kiskorov <me@albi.io> Date: Sun, 19 May 2024 19:37:59 +0700 Subject: [PATCH 100/423] Fix: Ensure `$time` is set from `innertext` when `datetime` attribute is not found (#4111) This commit addresses a bug where the $time variable is not set from the innertext of the $time_element when the datetime attribute is not found. The previous implementation only checked if $time was null or an empty string, which did not cover all cases where the datetime attribute might be missing. By using the empty() function, we ensure that $time is correctly set from the innertext when the datetime attribute is not present. --- bridges/CssSelectorComplexBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/CssSelectorComplexBridge.php b/bridges/CssSelectorComplexBridge.php index 632e6b6a..a2e001b2 100644 --- a/bridges/CssSelectorComplexBridge.php +++ b/bridges/CssSelectorComplexBridge.php @@ -442,7 +442,7 @@ class CssSelectorComplexBridge extends BridgeAbstract if (!is_null($time_selector) && $time_selector != '') { $time_element = $entry_html->find($time_selector, 0); $time = $time_element->getAttribute('datetime'); - if (is_null($time)) { + if (empty($time)) { $time = $time_element->innertext; } From 5a68ee0c87bc2392886efd411511e094ae4fe5ac Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Sun, 26 May 2024 20:51:14 +0530 Subject: [PATCH 101/423] [HinduTamilBridge] New (#4115) --- bridges/HinduTamilBridge.php | 91 ++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 bridges/HinduTamilBridge.php diff --git a/bridges/HinduTamilBridge.php b/bridges/HinduTamilBridge.php new file mode 100644 index 00000000..cdbdfe35 --- /dev/null +++ b/bridges/HinduTamilBridge.php @@ -0,0 +1,91 @@ +<?php + +class HinduTamilBridge extends FeedExpander +{ + const NAME = 'HinduTamil'; + const URI = 'https://www.hindutamil.in'; + const DESCRIPTION = 'Retrieve full articles from hindutamil.in feeds'; + const MAINTAINER = 'tillcash'; + const PARAMETERS = [ + [ + 'topic' => [ + 'name' => 'topic', + 'type' => 'list', + 'defaultValue' => 'crime', + 'values' => [ + 'Astrology' => 'astrology', + 'Blogs' => 'blogs', + 'Business' => 'business', + 'Cartoon' => 'cartoon', + 'Cinema' => 'cinema', + 'Crime' => 'crime', + 'Discussion' => 'discussion', + 'Education' => 'education', + 'Environment' => 'environment', + 'India' => 'india', + 'Lifestyle' => 'life-style', + 'Literature' => 'literature', + 'Opinion' => 'opinion', + 'Reporters' => 'reporters-page', + 'Socialmedia' => 'social-media', + 'Spirituals' => 'spirituals', + 'Sports' => 'sports', + 'Supplements' => 'supplements', + 'Tamilnadu' => 'tamilnadu', + 'Technology' => 'technology', + 'Tourism' => 'tourism', + 'World' => 'world', + ], + ], + 'limit' => [ + 'name' => 'limit (max 100)', + 'type' => 'number', + 'defaultValue' => 10, + ], + ], + ]; + + const FEED_BASE_URL = 'https://feeds.feedburner.com/Hindu_Tamil_'; + + public function getName() + { + $topic = $this->getKey('topic'); + return self::NAME . ($topic ? ' - ' . $topic : ''); + } + + public function collectData() + { + $limit = min(100, $this->getInput('limit')); + $url = self::FEED_BASE_URL . $this->getInput('topic'); + $this->collectExpandableDatas($url, $limit); + } + + protected function parseItem($item) + { + $dom = getSimpleHTMLDOMCached($item['uri']); + + $date = $dom->find('p span.date', 1); + if ($date) { + $item['timestamp'] = $date->innertext; + } + + $content = $dom->find('#pgContentPrint', 0); + if (!$content) { + return $item; + } + + $image = $dom->find('#LoadArticle figure', 0); + $item['content'] = $image . $this->cleanContent($content); + + return $item; + } + + private function cleanContent($content) + { + foreach ($content->find('div[align="center"], script') as $remove) { + $remove->outertext = ''; + } + + return $content; + } +} From bd90109c70fd3d4b302ed38e7b768e6df7d323d9 Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Thu, 30 May 2024 00:46:10 +0530 Subject: [PATCH 102/423] [HarvardHealthBlogBridge] New (#4116) --- bridges/HarvardHealthBlogBridge.php | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 bridges/HarvardHealthBlogBridge.php diff --git a/bridges/HarvardHealthBlogBridge.php b/bridges/HarvardHealthBlogBridge.php new file mode 100644 index 00000000..75e7e2cf --- /dev/null +++ b/bridges/HarvardHealthBlogBridge.php @@ -0,0 +1,56 @@ +<?php + +class HarvardHealthBlogBridge extends BridgeAbstract +{ + const NAME = 'Harvard Health Blog'; + const URI = 'https://www.health.harvard.edu/blog'; + const DESCRIPTION = 'Retrieve articles from health.harvard.edu'; + const MAINTAINER = 'tillcash'; + const MAX_ARTICLES = 10; + + public function collectData() + { + $dom = getSimpleHTMLDOM(self::URI); + $count = 0; + + foreach ($dom->find('div[class="mb-16 md:flex"]') as $element) { + if ($count >= self::MAX_ARTICLES) { + break; + } + + $data = $element->find('a[class="hover:text-red transition-colors duration-200"]', 0); + if (!$data) { + continue; + } + + $url = $data->href; + + $this->items[] = [ + 'content' => $this->constructContent($url), + 'timestamp' => $element->find('time', 0)->datetime, + 'title' => $data->plaintext, + 'uid' => $url, + 'uri' => $url, + ]; + + $count++; + } + } + + private function constructContent($url) + { + $dom = getSimpleHTMLDOMCached($url); + + $article = $dom->find('div[class*="content-repository-content"]', 0); + if (!$article) { + return 'Content Not Found'; + } + + // Remove ads + foreach ($article->find('.inline-ad') as $remove) { + $remove->outertext = ''; + } + + return $article->innertext; + } +} From cfd406861eeb6f90b61df56981cc0337ab504a06 Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Thu, 30 May 2024 19:38:08 +0530 Subject: [PATCH 103/423] [HarvardHealthBlogBridge] Update (#4117) Make article image optional as all images are representative --- bridges/HarvardHealthBlogBridge.php | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/bridges/HarvardHealthBlogBridge.php b/bridges/HarvardHealthBlogBridge.php index 75e7e2cf..bb6a5ede 100644 --- a/bridges/HarvardHealthBlogBridge.php +++ b/bridges/HarvardHealthBlogBridge.php @@ -7,6 +7,15 @@ class HarvardHealthBlogBridge extends BridgeAbstract const DESCRIPTION = 'Retrieve articles from health.harvard.edu'; const MAINTAINER = 'tillcash'; const MAX_ARTICLES = 10; + const PARAMETERS = [ + [ + 'image' => [ + 'name' => 'Article Image', + 'type' => 'checkbox', + 'defaultValue' => 'checked', + ], + ], + ]; public function collectData() { @@ -46,9 +55,15 @@ class HarvardHealthBlogBridge extends BridgeAbstract return 'Content Not Found'; } - // Remove ads - foreach ($article->find('.inline-ad') as $remove) { - $remove->outertext = ''; + // remove article image + if (!$this->getInput('image')) { + $image = $article->find('p', 0); + $image->remove(); + } + + // remove ads + foreach ($article->find('.inline-ad') as $ad) { + $ad->outertext = ''; } return $article->innertext; From 36706a3dec464f1ab59b21356bc838ab77c657b4 Mon Sep 17 00:00:00 2001 From: Tim-Florian Feulner <50834839+R3dError@users.noreply.github.com> Date: Mon, 3 Jun 2024 00:55:39 +0200 Subject: [PATCH 104/423] Fix NACSouthGermanyMediaLibraryBridge due to website changes (#4121) --- bridges/NACSouthGermanyMediaLibraryBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/NACSouthGermanyMediaLibraryBridge.php b/bridges/NACSouthGermanyMediaLibraryBridge.php index 030ded8f..70129b37 100644 --- a/bridges/NACSouthGermanyMediaLibraryBridge.php +++ b/bridges/NACSouthGermanyMediaLibraryBridge.php @@ -85,12 +85,12 @@ class NACSouthGermanyMediaLibraryBridge extends BridgeAbstract foreach ($page->find('div.flex-columns.entry') as $parent) { # Find title - $title = $parent->find('h2', 0)->plaintext; + $title = trim($parent->find('h2')[0]->innertext); # Find content - $contentBlock = $parent->find('ul', 0); + $contentBlock = $parent->find('div')[2]; $content = ''; - foreach ($contentBlock->find('li') as $li) { + foreach ($contentBlock->find('li,p') as $li) { $content .= '<p>' . $li->plaintext . '</p>'; } From 87fa6ea71e8ae8c6e0cddd47e28fd65664a6cb05 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Mon, 10 Jun 2024 19:40:07 +0200 Subject: [PATCH 105/423] [HeiseBridge.php] Prevent Youtube videos from being filtered out (#4125) --- bridges/HeiseBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index 9aa7209d..e26a4607 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -214,7 +214,7 @@ class HeiseBridge extends FeedExpander $content = $article->find('.article-content', 0); if ($content) { $contentElements = $content->find( - 'p, h3, ul, ol, table, pre, noscript img, a-bilderstrecke h2, a-bilderstrecke figure, a-bilderstrecke figcaption' + 'p, h3, ul, ol, table, pre, noscript img, a-bilderstrecke h2, a-bilderstrecke figure, a-bilderstrecke figcaption, noscript iframe' ); $item['content'] .= implode('', $contentElements); } From d3d33c72bda0cb6f222b415da075b32f2a60a61d Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Tue, 11 Jun 2024 19:10:49 +0530 Subject: [PATCH 106/423] [HinduTamilBridge] fix timestamp (#4127) --- bridges/HinduTamilBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/HinduTamilBridge.php b/bridges/HinduTamilBridge.php index cdbdfe35..cab04171 100644 --- a/bridges/HinduTamilBridge.php +++ b/bridges/HinduTamilBridge.php @@ -66,7 +66,7 @@ class HinduTamilBridge extends FeedExpander $date = $dom->find('p span.date', 1); if ($date) { - $item['timestamp'] = $date->innertext; + $item['timestamp'] = $date->innertext . ' IST'; } $content = $dom->find('#pgContentPrint', 0); From e1b74aeb1bd120ac0c337fae9f55943cf7d00a0d Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Wed, 12 Jun 2024 23:02:17 -0400 Subject: [PATCH 107/423] [GameBananaBridge] Add categories and more detailed updates (#4129) * [GameBananaBridge] Add mod categorie(s) * [GameBananaBridge] Include full update changelog details --- bridges/GameBananaBridge.php | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/bridges/GameBananaBridge.php b/bridges/GameBananaBridge.php index 9a0a0686..88b19ef0 100644 --- a/bridges/GameBananaBridge.php +++ b/bridges/GameBananaBridge.php @@ -40,7 +40,7 @@ class GameBananaBridge extends BridgeAbstract $json_list = json_decode($api_response, true); // Get first page mod list $url = 'https://api.gamebanana.com/Core/Item/Data?itemtype[]=Game&fields[]=name&itemid[]=' . $this->getInput('gid'); - $fields = 'name,Owner().name,text,screenshots,Files().aFiles(),date,Url().sProfileUrl(),udate,Updates().aLatestUpdates()'; + $fields = 'name,Owner().name,text,screenshots,Files().aFiles(),date,Url().sProfileUrl(),udate,Updates().aLatestUpdates(),Category().name,RootCategory().name'; foreach ($json_list as $element) { // Build api request to minimize API calls $mid = $element[1]; $url .= '&itemtype[]=Mod&fields[]=' . $fields . '&itemid[]=' . $mid; @@ -52,11 +52,18 @@ class GameBananaBridge extends BridgeAbstract array_shift($json_list); // Take title from API request and remove from json foreach ($json_list as $element) { + // Trashed mod IDs are still picked up and return null; skip + if ($element[0] == null) { + continue; + } + $item = []; $item['uri'] = $element[6]; $item['comments'] = $item['uri'] . '#PostsListModule'; $item['title'] = $element[0]; $item['author'] = $element[1]; + $item['categories'][] = $element[9]; + $item['categories'][] = $element[10]; $item['timestamp'] = $element[5]; if ($this->getInput('updates')) { @@ -74,9 +81,21 @@ class GameBananaBridge extends BridgeAbstract foreach ($img_list as $img_element) { $item['content'] .= '<img src="https://images.gamebanana.com/img/ss/mods/' . $img_element['_sFile'] . '"/>'; } + + // Get updates from element[8], if applicable if ($this->getInput('updates') && sizeof($element[8]) > 0) { - $item['content'] .= '<br><strong>Update: ' . $element[8][0]['_sTitle']; - $item['content'] .= '</strong><br>' . $element[8][0]['_sText'] . '<hr>'; + $update = $element[8][0]; + $item['content'] .= '<br><strong>Update:</strong> ' . $update['_sTitle']; + if ($update['_sText'] != '') { + $item['content'] .= '<br>' . $update['_sText']; + } + foreach ($update['_aChangeLog'] as $change) { + if ($change['cat'] == '') { + $change['cat'] = 'Change'; + } + $item['content'] .= '<br><em>' . $change['cat'] . '</em>: ' . $change['text']; + } + $item['content'] .= '<br><hr>'; } $item['content'] .= '<br>' . $element[2]; From bb1e308057b7e8740d67d0fb0fa9c6298e9b9730 Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Thu, 13 Jun 2024 05:03:20 +0200 Subject: [PATCH 108/423] [IdealoBridge] Fix price comparison and some PHP Notice (#4130) * [IdealoBridge] Fix price comparison and some PHP Notice - The prices were compared as String and the comparison was wrong in some case : now the price are converted to float before the comparison, so the logic works really. - Don't show a new or used product price if it does not exist : this prevents a PHP Notice to be thrown * [IdealoBridge] Fix price conversion in case the price is null The conversion as float of the text price won't work if the price is null : we retunr null in this case now. --- bridges/IdealoBridge.php | 43 ++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php index 4eb66dcb..f426a45c 100644 --- a/bridges/IdealoBridge.php +++ b/bridges/IdealoBridge.php @@ -81,6 +81,25 @@ class IdealoBridge extends BridgeAbstract return $title . ' - ' . $this::NAME; } + /** + * Returns the Price as float + * @return float rhe price converted in float + */ + private function convertPriceToFloat($price) + { + // Every price is stored / displayed as "xxx,xx €", but PHP can't convert it as float + + if ($price !== null) { + // Convert comma as dot + $price = str_replace(',', '.', $price); + // Remove the '€' char + $price = str_replace('€', '', $price); + // Convert to float + return floatval($price); + } else { + return $price; + } + } /** * Returns the Price Trend emoji @@ -88,8 +107,10 @@ class IdealoBridge extends BridgeAbstract */ private function getPriceTrend($NewPrice, $OldPrice) { - // In case there is no old PRice, then show no trend - if ($OldPrice === null) { + $NewPrice = $this->convertPriceToFloat($NewPrice); + $OldPrice = $this->convertPriceToFloat($OldPrice); + // In case there is no old Price, then show no trend + if ($OldPrice === null || $OldPrice == 0) { $trend = ''; } else if ($NewPrice > $OldPrice) { $trend = '↗'; @@ -125,7 +146,7 @@ class IdealoBridge extends BridgeAbstract $OldPriceNew = $this->loadCacheValue($KeyNEW); $OldPriceUsed = $this->loadCacheValue($KeyUSED); - // First button is new. Found at oopStage-conditionButton-wrapper-text class (.) + // First button contains the new price. Found at oopStage-conditionButton-wrapper-text class (.) $FirstButton = $html->find('.oopStage-conditionButton-wrapper-text', 0); if ($FirstButton) { $PriceNew = $FirstButton->find('strong', 0)->plaintext; @@ -133,7 +154,7 @@ class IdealoBridge extends BridgeAbstract $this->saveCacheValue($KeyNEW, $PriceNew); } - // Second Button is used + // Second Button contains the used product price $SecondButton = $html->find('.oopStage-conditionButton-wrapper-text', 1); if ($SecondButton) { $PriceUsed = $SecondButton->find('strong', 0)->plaintext; @@ -149,7 +170,7 @@ class IdealoBridge extends BridgeAbstract $content = ''; // Generate Content - if (isset($PriceNew) && $PriceNew > 1) { + if (isset($PriceNew) && $this->convertPriceToFloat($PriceNew) > 0) { $content .= sprintf('<p><b>Price New:</b><br>%s %s</p>', $PriceNew, $this->getPriceTrend($PriceNew, $OldPriceNew)); $content .= "<p><b>Price New before:</b><br>$OldPriceNew</p>"; } @@ -158,7 +179,7 @@ class IdealoBridge extends BridgeAbstract $content .= sprintf('<p><b>Max Price New:</b><br>%s,00 €</p>', $this->getInput('MaxPriceNew')); } - if (isset($PriceUsed) && $PriceUsed > 1) { + if (isset($PriceUsed) && $this->convertPriceToFloat($PriceUsed) > 0) { $content .= sprintf('<p><b>Price Used:</b><br>%s %s</p>', $PriceUsed, $this->getPriceTrend($PriceUsed, $OldPriceUsed)); $content .= "<p><b>Price Used before:</b><br>$OldPriceUsed</p>"; } @@ -176,7 +197,7 @@ class IdealoBridge extends BridgeAbstract // Currently under Max new price if ($this->getInput('MaxPriceNew') != '') { - if (isset($PriceNew) && $PriceNew < $this->getInput('MaxPriceNew')) { + if (isset($PriceNew) && $this->convertPriceToFloat($PriceNew) < $this->getInput('MaxPriceNew')) { $title = sprintf($Pricealarm, 'New', $PriceNew, $Productname, $now); $item = [ 'title' => $title, @@ -190,7 +211,7 @@ class IdealoBridge extends BridgeAbstract // Currently under Max used price if ($this->getInput('MaxPriceUsed') != '') { - if (isset($PriceUsed) && $PriceUsed < $this->getInput('MaxPriceUsed')) { + if (isset($PriceUsed) && $this->convertPriceToFloat($PriceUsed) < $this->getInput('MaxPriceUsed')) { $title = sprintf($Pricealarm, 'Used', $PriceUsed, $Productname, $now); $item = [ 'title' => $title, @@ -202,7 +223,7 @@ class IdealoBridge extends BridgeAbstract } } - // General Priceupdate + // General Priceupdate Without any Max Price for new and Used product if ($this->getInput('MaxPriceUsed') == '' && $this->getInput('MaxPriceNew') == '') { // check if a relevant pricechange happened if ( @@ -211,11 +232,11 @@ class IdealoBridge extends BridgeAbstract ) { $title = 'Priceupdate! '; - if (!$this->getInput('ExcludeNew')) { + if (!$this->getInput('ExcludeNew') && isset($PriceNew)) { $title .= 'NEW' . $this->getPriceTrend($PriceNew, $OldPriceNew) . ' '; } - if (!$this->getInput('ExcludeUsed')) { + if (!$this->getInput('ExcludeUsed') && isset($PriceUsed)) { $title .= 'USED' . $this->getPriceTrend($PriceUsed, $OldPriceUsed) . ' '; } $title .= $Productname; From 649dfa72929c8aabdbbf4c693b35c484ffa135e9 Mon Sep 17 00:00:00 2001 From: Ftonans <77411099+Ftonans@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:11:02 +0000 Subject: [PATCH 109/423] Update instance list (#4131) vern's instance seems to be working, I changed the url to https since they have automatic redirect. I removed trailing slashes from the urls so they look the same. I removed [rss.m3wz.su](https://rss.m3wz.su] since I didn't see the website online and the owner last posted on Fediverse two months ago. I'm not sure maybe it should be in "Inactive" category, I can try to contact m3wz for information about his instance. I removed rss.foxhaven.cyou because of [this](https://shitpost.poridge.club/notes/9lumb2gll8) (TL;DR the owner lost access to the domain) bus-hit is offline but the main website is working. I guess the rss-bridge just crashed and the owner will restart it. --- docs/01_General/06_Public_Hosts.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index 1bc8fc20..fa8d5fdd 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -3,12 +3,12 @@ | Country | Address | Status | Contact | Comment | |:-------:|---------|--------|----------|---------| | ![](https://iplookup.flagfox.net/images/h16/GB.png) | https://rss-bridge.org/bridge01 | ![](https://img.shields.io/website/https/rss-bridge.org/bridge01.svg) | [@dvikan](https://github.com/dvikan) | London, Digital Ocean| -| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rssbridge.flossboxin.org.in/ | ![](https://img.shields.io/badge/website-up-brightgreen) | [@vdbhb59](https://github.com/vdbhb59) | Hosted with OVH SAS (Maintained in India) | -| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rss-bridge.cheredeprince.net/ | ![](https://img.shields.io/website/https/rss-bridge.cheredeprince.net) | [@La_Bécasse](https://cheredeprince.net/contact) | Self-Hosted at home in France | +| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rssbridge.flossboxin.org.in | ![](https://img.shields.io/badge/website-up-brightgreen) | [@vdbhb59](https://github.com/vdbhb59) | Hosted with OVH SAS (Maintained in India) | +| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rss-bridge.cheredeprince.net | ![](https://img.shields.io/website/https/rss-bridge.cheredeprince.net) | [@La_Bécasse](https://cheredeprince.net/contact) | Self-Hosted at home in France | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rss-bridge.sans-nuage.fr | ![](https://img.shields.io/website/https/rss-bridge.sans-nuage.fr) | [@Alsace Réseau Neutre](https://arn-fai.net/contact) | Hosted in Alsace, France | | ![](https://iplookup.flagfox.net/images/h16/GB.png) | https://rss-bridge.lewd.tech | ![](https://img.shields.io/website/https/rss-bridge.lewd.tech.svg) | [@Erisa](https://github.com/Erisa) | Hosted in London, protected by Cloudflare Rate Limiting | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://bridge.easter.fr | ![](https://img.shields.io/website/https/bridge.easter.fr.svg) | [@chatainsim](https://github.com/chatainsim) | Hosted in Isère, France | -| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://wtf.roflcopter.fr/rss-bridge/ | ![](https://img.shields.io/website/https/wtf.roflcopter.fr/rss-bridge.svg) | [roflcopter.fr](https://wtf.roflcopter.fr/) | Hosted in France | +| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://wtf.roflcopter.fr/rss-bridge | ![](https://img.shields.io/website/https/wtf.roflcopter.fr/rss-bridge.svg) | [roflcopter.fr](https://wtf.roflcopter.fr/) | Hosted in France | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss.nixnet.services | ![](https://img.shields.io/website/https/rss.nixnet.services.svg) | [@amolith](https://nixnet.services/contact) | Hosted in Wunstorf, Germany | | ![](https://iplookup.flagfox.net/images/h16/AT.png) | https://rss-bridge.ggc-project.de | ![](https://img.shields.io/website/https/rss-bridge.ggc-project.de) | [@ggc-project.de](https://social.dev-wiki.de/@ggc_project) | Hosted in Steyr, Austria | | ![](https://iplookup.flagfox.net/images/h16/CA.png) | https://rssbridge.bus-hit.me | ![](https://img.shields.io/website/https/rssbridge.bus-hit.me.svg)| [@austinhuang0131](https://austinhuang.me/) | Hosted with Oracle in Québec, Canada | @@ -16,15 +16,15 @@ | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rssbridge.boldair.dev | ![](https://img.shields.io/website?down_color=red&down_message=down&up_color=lime&up_message=up&url=https%3A%2F%2Frssbridge.boldair.dev) | [@Boldairdev](https://github.com/Boldairdev) | Latest Github release, Hosted on PHP 8.0 in Roubaix, France | | ![](https://iplookup.flagfox.net/images/h16/IN.png) | https://rss-bridge.bb8.fun | ![](https://img.shields.io/website/https/rss-bridge.bb8.fun.svg) | [@captn3m0](https://github.com/captn3m0) | Hosted in Bengaluru, India | | ![](https://iplookup.flagfox.net/images/h16/RU.png) | https://ololbu.ru/rss-bridge | ![](https://img.shields.io/website/https/ololbu.ru) | [@Ololbu](https://github.com/Ololbu) | Hosted in Moscow, Russia | -| ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://tools.bheil.net/rss-bridge/ | ![](https://img.shields.io/website/https/tools.bheil.net.svg) | [@bheil](https://www.bheil.net) | Hosted in Germany | +| ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://tools.bheil.net/rss-bridge | ![](https://img.shields.io/website/https/tools.bheil.net.svg) | [@bheil](https://www.bheil.net) | Hosted in Germany | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://bridge.suumitsu.eu | ![](https://img.shields.io/website/https/bridge.suumitsu.eu.svg) | [@mitsukarenai](https://github.com/mitsukarenai) | Hosted in Paris, France | | ![](https://iplookup.flagfox.net/images/h16/NL.png) | https://feed.eugenemolotov.ru | ![](https://img.shields.io/website/https/feed.eugenemolotov.ru.svg) | [@em92](https://github.com/em92) | Hosted in Amsterdam, Netherlands | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss-bridge.mediani.de | ![](https://img.shields.io/website/https/rss-bridge.mediani.de.svg) | [@sokai](https://github.com/sokai) | Hosted with Netcup, Germany | -| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | -| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rb.ash.fail | ![](https://img.shields.io/website/https/rb.ash.fail.svg) | [@ash](https://ash.fail/contact.html) | Hosted with Hostaris, Germany | ![](https://iplookup.flagfox.net/images/h16/UA.png) | https://rss.noleron.com | ![](https://img.shields.io/website/https/rss.noleron.com) | [@ihor](https://noleron.com/about) | Hosted with Hosting Ukraine, Ukraine | ![](https://iplookup.flagfox.net/images/h16/IN.png) | https://rssbridge.projectsegfau.lt | ![](https://img.shields.io/website/https/rssbridge.projectsegfau.lt) | [@gi-yt](https://aryak.me) | Self-Hosted at Mumbai, India with Airtel (ISP) | +| ![](https://iplookup.flagfox.net/images/h16/US.png) | https://rb.vern.cc | ![](https://img.shields.io/website/https/rb.vern.cc.svg) | [@vern.cc](https://vern.cc/en/admin) | Hosted with Hetzner, US | +| ![](https://iplookup.flagfox.net/images/h16/RO.png) | https://rss.bloat.cat | ![](https://img.shields.io/website/https/rss.bloat.cat) | [@vlnst](https://bloat.cat/contact) | Hosted with Kyun, Romania | ## Inactive instances @@ -32,4 +32,3 @@ | Country | Address | Status | Contact | Comment | |:-------:|---------|--------|----------|---------| | ![](https://iplookup.flagfox.net/images/h16/FI.png) | https://rss-bridge.snopyta.org | ![](https://img.shields.io/website/https/rss-bridge.snopyta.org.svg) | [@Perflyst](https://github.com/Perflyst) | Hosted in Helsinki, Finland | -| ![](https://iplookup.flagfox.net/images/h16/US.png) | http://rb.vern.cc/ | ![](https://img.shields.io/website/https/rb.vern.cc.svg) | [@vern.cc](https://vern.cc/en/admin) | Hosted with Hetzner, US | From 0eac7a078479eac48be6c841a79f77ff04f429a8 Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sun, 16 Jun 2024 13:16:42 +0200 Subject: [PATCH 110/423] [HeiseBridge] Remove lost+found icon Remove the icon visible in l+f articles, e.g. https://www.heise.de/news/l-f-DISGOMOJI-die-Linux-Malware-die-auf-Emojis-steht-9765024.html Using a css selector in the form img[alt*="l+f"] was tried, but is not supported by the used library. --- bridges/HeiseBridge.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index e26a4607..82c1f1aa 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -166,6 +166,11 @@ class HeiseBridge extends FeedExpander ) { $element->remove(); } + foreach ($article->find('img') as $element) { + if (str_contains($element->alt, 'l+f')) { + $element->remove(); + } + } // reload html, as remove() is buggy $article = str_get_html($article->outertext); From 206bebc7bdf90b00b2457a5693fe07079b3df40a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 18 Jun 2024 20:22:46 +0200 Subject: [PATCH 111/423] ci: disallow the sizeof function in linter (#4134) --- bridges/DiarioDoAlentejoBridge.php | 2 +- bridges/GameBananaBridge.php | 2 +- bridges/ItakuBridge.php | 28 ++++++++++++++-------------- bridges/MagellantvBridge.php | 2 +- bridges/PatreonBridge.php | 2 +- bridges/PresidenciaPTBridge.php | 2 +- bridges/VieDeMerdeBridge.php | 2 +- phpcs.xml | 8 ++++++++ 8 files changed, 28 insertions(+), 20 deletions(-) diff --git a/bridges/DiarioDoAlentejoBridge.php b/bridges/DiarioDoAlentejoBridge.php index 9b82b49f..0bd0f1d4 100644 --- a/bridges/DiarioDoAlentejoBridge.php +++ b/bridges/DiarioDoAlentejoBridge.php @@ -47,7 +47,7 @@ class DiarioDoAlentejoBridge extends BridgeAbstract }, self::PT_MONTH_NAMES), array_map(function ($num) { return sprintf('-%02d-', $num); - }, range(1, sizeof(self::PT_MONTH_NAMES))), + }, range(1, count(self::PT_MONTH_NAMES))), $element->find('span.date', 0)->innertext ); diff --git a/bridges/GameBananaBridge.php b/bridges/GameBananaBridge.php index 88b19ef0..0f04f56b 100644 --- a/bridges/GameBananaBridge.php +++ b/bridges/GameBananaBridge.php @@ -83,7 +83,7 @@ class GameBananaBridge extends BridgeAbstract } // Get updates from element[8], if applicable - if ($this->getInput('updates') && sizeof($element[8]) > 0) { + if ($this->getInput('updates') && count($element[8]) > 0) { $update = $element[8][0]; $item['content'] .= '<br><strong>Update:</strong> ' . $update['_sTitle']; if ($update['_sText'] != '') { diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index 0577752c..4f414574 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -347,17 +347,17 @@ class ItakuBridge extends BridgeAbstract $url = self::URI . "/api/galleries/images/?by_following=false&date_range={$opt['range']}&ordering={$opt['order']}&is_video={$opt['video_only']}"; $url .= "&text={$opt['text']}&visibility=PUBLIC&visibility=PROFILE_ONLY&page=1&page_size=30&format=json"; - if (sizeof($opt['optional_tags']) > 0) { + if (count($opt['optional_tags']) > 0) { foreach ($opt['optional_tags'] as $tag) { $url .= "&optional_tags=$tag"; } } - if (sizeof($opt['negative_tags']) > 0) { + if (count($opt['negative_tags']) > 0) { foreach ($opt['negative_tags'] as $tag) { $url .= "&negative_tags=$tag"; } } - if (sizeof($opt['required_tags']) > 0) { + if (count($opt['required_tags']) > 0) { foreach ($opt['required_tags'] as $tag) { $url .= "&required_tags=$tag"; } @@ -381,17 +381,17 @@ class ItakuBridge extends BridgeAbstract $url = self::URI . "/api/posts/?by_following=false&date_range={$opt['range']}&ordering={$opt['order']}"; $url .= '&visibility=PUBLIC&visibility=PROFILE_ONLY&page=1&page_size=30&format=json'; - if (sizeof($opt['optional_tags']) > 0) { + if (count($opt['optional_tags']) > 0) { foreach ($opt['optional_tags'] as $tag) { $url .= "&optional_tags=$tag"; } } - if (sizeof($opt['negative_tags']) > 0) { + if (count($opt['negative_tags']) > 0) { foreach ($opt['negative_tags'] as $tag) { $url .= "&negative_tags=$tag"; } } - if (sizeof($opt['required_tags']) > 0) { + if (count($opt['required_tags']) > 0) { foreach ($opt['required_tags'] as $tag) { $url .= "&required_tags=$tag"; } @@ -446,7 +446,7 @@ class ItakuBridge extends BridgeAbstract private function getPost($id, array $metadata = null) { - if (isset($metadata) && sizeof($metadata['gallery_images']) < $metadata['num_images']) { + if (isset($metadata) && count($metadata['gallery_images']) < $metadata['num_images']) { $metadata = null; //force re-fetch of metadata } $uri = self::URI . '/posts/' . $id; @@ -457,7 +457,7 @@ class ItakuBridge extends BridgeAbstract $content_str = nl2br($data['content']); $content = "<p>{$content_str}</p><br/>"; //TODO: Add link and itaku user mention detection and convert into links. - if (array_key_exists('tags', $data) && sizeof($data['tags']) > 0) { + if (array_key_exists('tags', $data) && count($data['tags']) > 0) { $tag_types = [ 'ARTIST' => '', 'COPYRIGHT' => '', @@ -479,7 +479,7 @@ class ItakuBridge extends BridgeAbstract } } - if (sizeof($data['folders']) > 0) { + if (count($data['folders']) > 0) { $content .= '📁 In Folder(s): '; foreach ($data['folders'] as $folder) { $url = self::URI . '/profile/' . $data['owner_username'] . '/posts/' . $folder['id']; @@ -488,7 +488,7 @@ class ItakuBridge extends BridgeAbstract } $content .= '<hr/>'; - if (sizeof($data['gallery_images']) > 0) { + if (count($data['gallery_images']) > 0) { foreach ($data['gallery_images'] as $media) { $title = $media['title']; $url = self::URI . '/images/' . $media['id']; @@ -529,7 +529,7 @@ class ItakuBridge extends BridgeAbstract $content_str = nl2br($data['description']); $content = "<p>{$content_str}</p><br>"; //TODO: Add link and itaku user mention detection and convert into links. - if (array_key_exists('tags', $data) && sizeof($data['tags']) > 0) { + if (array_key_exists('tags', $data) && count($data['tags']) > 0) { // $content .= "🏷 Tag(s): "; $tag_types = [ 'ARTIST' => '', @@ -552,7 +552,7 @@ class ItakuBridge extends BridgeAbstract } } - if (array_key_exists('reference_gallery_sections', $data) && sizeof($data['reference_gallery_sections']) > 0) { + if (array_key_exists('reference_gallery_sections', $data) && count($data['reference_gallery_sections']) > 0) { $content .= '📁 Example folder(s): '; foreach ($data['folders'] as $folder) { $url = self::URI . '/profile/' . $data['owner_username'] . '/gallery/' . $folder['id']; @@ -601,7 +601,7 @@ class ItakuBridge extends BridgeAbstract $content_str = nl2br($data['description']); $content = "<p>{$content_str}</p><br/>"; //TODO: Add link and itaku user mention detection and convert into links. - if (array_key_exists('tags', $data) && sizeof($data['tags']) > 0) { + if (array_key_exists('tags', $data) && count($data['tags']) > 0) { // $content .= "🏷 Tag(s): "; $tag_types = [ 'ARTIST' => '', @@ -624,7 +624,7 @@ class ItakuBridge extends BridgeAbstract } } - if (array_key_exists('sections', $data) && sizeof($data['sections']) > 0) { + if (array_key_exists('sections', $data) && count($data['sections']) > 0) { $content .= '📁 In Folder(s): '; foreach ($data['sections'] as $folder) { $url = self::URI . '/profile/' . $data['owner_username'] . '/gallery/' . $folder['id']; diff --git a/bridges/MagellantvBridge.php b/bridges/MagellantvBridge.php index b1f0403e..0a225160 100644 --- a/bridges/MagellantvBridge.php +++ b/bridges/MagellantvBridge.php @@ -63,7 +63,7 @@ class MagellantvBridge extends BridgeAbstract // Check whether items exists $article_list = $dom->find('div.articlePreview_preview-card__mLMOm'); - if (sizeof($article_list) == 0) { + if (count($article_list) == 0) { throw new Exception(sprintf('Unable to find css selector on `%s`', $url)); } diff --git a/bridges/PatreonBridge.php b/bridges/PatreonBridge.php index a2162425..895a9306 100644 --- a/bridges/PatreonBridge.php +++ b/bridges/PatreonBridge.php @@ -228,7 +228,7 @@ class PatreonBridge extends BridgeAbstract //post attachments if ( isset($post->relationships->attachments->data) && - sizeof($post->relationships->attachments->data) > 0 + count($post->relationships->attachments->data) > 0 ) { $item['enclosures'] = []; $item['content'] .= '<hr><p><b>Attachments:</b><ul>'; diff --git a/bridges/PresidenciaPTBridge.php b/bridges/PresidenciaPTBridge.php index 052b2751..247e8fce 100644 --- a/bridges/PresidenciaPTBridge.php +++ b/bridges/PresidenciaPTBridge.php @@ -76,7 +76,7 @@ class PresidenciaPTBridge extends BridgeAbstract }, self::PT_MONTH_NAMES), array_map(function ($num) { return sprintf('-%02d-', $num); - }, range(1, sizeof(self::PT_MONTH_NAMES))), + }, range(1, count(self::PT_MONTH_NAMES))), $edt ); diff --git a/bridges/VieDeMerdeBridge.php b/bridges/VieDeMerdeBridge.php index 9e6166fb..be384157 100644 --- a/bridges/VieDeMerdeBridge.php +++ b/bridges/VieDeMerdeBridge.php @@ -26,7 +26,7 @@ class VieDeMerdeBridge extends BridgeAbstract $html = getSimpleHTMLDOM(self::URI, []); $quotes = $html->find('article.bg-white'); - if (sizeof($quotes) === 0) { + if (count($quotes) === 0) { return; } diff --git a/phpcs.xml b/phpcs.xml index bd1aca28..9e393a13 100644 --- a/phpcs.xml +++ b/phpcs.xml @@ -35,6 +35,14 @@ </properties> </rule> + <rule ref="Generic.PHP.ForbiddenFunctions"> + <properties> + <property name="forbiddenFunctions" type="array"> + <element key="sizeof" value="count"/> + </property> + </properties> + </rule> + <!-- Duplicate class names are not allowed --> <rule ref="Generic.Classes.DuplicateClassName"/> From 00074b9bfc4c1ed3174b83d2f33015a1dc240464 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 18 Jun 2024 20:55:05 +0200 Subject: [PATCH 112/423] fix: dont remove www from anchors in DOM, fix #4114 (#4135) --- static/rss-bridge.js | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/static/rss-bridge.js b/static/rss-bridge.js index b9b466d6..9cd004cb 100644 --- a/static/rss-bridge.js +++ b/static/rss-bridge.js @@ -1,21 +1,14 @@ function rssbridge_list_search() { - function remove_www_from_url(url) { - if (url.hostname.indexOf('www.') === 0) { - url.hostname = url.hostname.substr(4); - } - } - var search = document.getElementById('searchfield').value; - var searchAsUrl = document.createElement('a'); - searchAsUrl.href = search; - remove_www_from_url(searchAsUrl); + var bridgeCards = document.querySelectorAll('section.bridge-card'); for (var i = 0; i < bridgeCards.length; i++) { var bridgeName = bridgeCards[i].getAttribute('data-ref'); var bridgeShortName = bridgeCards[i].getAttribute('data-short-name'); var bridgeDescription = bridgeCards[i].querySelector('.description'); - var bridgeUrl = bridgeCards[i].getElementsByTagName('a')[0]; - remove_www_from_url(bridgeUrl); + var bridgeUrlElement = bridgeCards[i].getElementsByTagName('a')[0]; + var bridgeUrl = bridgeUrlElement.toString(); + bridgeCards[i].style.display = 'none'; if (!bridgeName || !bridgeUrl) { continue; @@ -30,10 +23,7 @@ function rssbridge_list_search() { if (bridgeDescription.textContent.match(searchRegex)) { bridgeCards[i].style.display = 'block'; } - if (bridgeUrl.toString().match(searchRegex)) { - bridgeCards[i].style.display = 'block'; - } - if (bridgeUrl.hostname === searchAsUrl.hostname) { + if (bridgeUrl.match(searchRegex)) { bridgeCards[i].style.display = 'block'; } } From d60f0b0e74278db9d30b50e40533d92d92c079ec Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 18 Jun 2024 21:12:29 +0200 Subject: [PATCH 113/423] feat(FilterBridge): custom feed name parameter (#4136) fix #4100 --- bridges/FilterBridge.php | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index 3448a8c7..a1066fb5 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -15,6 +15,12 @@ class FilterBridge extends FeedExpander 'exampleValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day', 'required' => true, ], + 'name' => [ + 'name' => 'Feed name (optional)', + 'type' => 'text', + 'exampleValue' => 'My feed', + 'required' => false, + ], 'filter' => [ 'name' => 'Filter (regular expression!!!)', 'required' => false, @@ -158,11 +164,18 @@ class FilterBridge extends FeedExpander public function getURI() { $url = $this->getInput('url'); - - if (empty($url)) { - $url = parent::getURI(); + if ($url) { + return $url; } + return parent::getURI(); + } - return $url; + public function getName() + { + $name = $this->getInput('name'); + if ($name) { + return $name; + } + return parent::getName(); } } From 2a84350cb2bf4d62111c0294e71d56302714ee63 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Fri, 21 Jun 2024 09:47:34 -0400 Subject: [PATCH 114/423] [HumbleBundleBridge] Create new bridge (#4139) * [HumbleBundleBridge] Create new bridge * [HumbleBundleBridge] Use less redundant bundle type handling --- bridges/HumbleBundleBridge.php | 68 ++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 bridges/HumbleBundleBridge.php diff --git a/bridges/HumbleBundleBridge.php b/bridges/HumbleBundleBridge.php new file mode 100644 index 00000000..42e025a5 --- /dev/null +++ b/bridges/HumbleBundleBridge.php @@ -0,0 +1,68 @@ +<?php + +class HumbleBundleBridge extends BridgeAbstract +{ + const NAME = 'Humble Bundle'; + const MAINTAINER = 'phantop'; + const URI = 'https://humblebundle.com/'; + const DESCRIPTION = 'Returns bundles from Humble Bundle.'; + const PARAMETERS = [[ + 'type' => [ + 'name' => 'Bundle type', + 'type' => 'list', + 'defaultValue' => 'bundles', + 'values' => [ + 'All' => 'bundles', + 'Books' => 'books', + 'Games' => 'games', + 'Software' => 'software', + ] + ] + ]]; + + public function collectData() + { + $page = getSimpleHTMLDOMCached($this->getURI()); + $json_text = $page->find('#landingPage-json-data', 0)->innertext; + $json = json_decode(html_entity_decode($json_text), true)['data']; + + $products = []; + $types = ['books', 'games', 'software']; + $types = $this->getInput('type') === 'bundles' ? $types : [$this->getInput('type')]; + foreach ($types as $type) { + $products = array_merge($products, $json[$type]['mosaic'][0]['products']); + } + + foreach ($products as $element) { + $item = []; + $item['author'] = $element['author']; + $item['timestamp'] = $element['start_date|datetime']; + $item['title'] = $element['tile_short_name']; + $item['uid'] = $element['machine_name']; + $item['uri'] = parent::getURI() . $element['product_url']; + + $item['content'] = $element['marketing_blurb']; + $item['content'] .= '<br>' . $element['detailed_marketing_blurb']; + + $item['categories'] = $element['hover_highlights']; + array_unshift($item['categories'], explode(':', $element['tile_name'])[0]); + array_unshift($item['categories'], $element['tile_stamp']); + + $item['enclosures'] = [$element['tile_logo'], $element['high_res_tile_image']]; + $this->items[] = $item; + } + } + + public function getName() + { + $name = parent::getName(); + $name .= $this->getInput('type') ? ' - ' . $this->getInput('type') : ''; + return $name; + } + + public function getURI() + { + $uri = parent::getURI() . $this->getInput('type'); + return $uri; + } +} From adad9d6405efb1b19987bf6933ea8309b5f3c28e Mon Sep 17 00:00:00 2001 From: Thomas <mightymt@users.noreply.github.com> Date: Mon, 24 Jun 2024 22:32:03 +0200 Subject: [PATCH 115/423] [YouTubeCommunityTabBridge] Improve JSON extraction (#4140) Small change that should make the extraction of JSON from HTML work more reliably --- bridges/YouTubeCommunityTabBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/YouTubeCommunityTabBridge.php b/bridges/YouTubeCommunityTabBridge.php index 0c145c02..284b81f9 100644 --- a/bridges/YouTubeCommunityTabBridge.php +++ b/bridges/YouTubeCommunityTabBridge.php @@ -32,7 +32,7 @@ class YouTubeCommunityTabBridge extends BridgeAbstract private $itemTitle = ''; private $urlRegex = '/youtube\.com\/(channel|user|c)\/([\w]+)\/community/'; - private $jsonRegex = '/var ytInitialData = (.*);<\/script>/'; + private $jsonRegex = '/var ytInitialData = ([^<]*);<\/script>/'; public function detectParameters($url) { @@ -70,7 +70,7 @@ class YouTubeCommunityTabBridge extends BridgeAbstract $html = getSimpleHTMLDOM($this->feedUrl); } - $json = $this->extractJson($html->find('body', 0)->innertext); + $json = $this->extractJson($html->find('html', 0)->innertext); $this->feedName = $json->header->c4TabbedHeaderRenderer->title; From d0c35146dd9b8b0625eacefb59402a16519b5c5f Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Sat, 29 Jun 2024 00:21:59 +0530 Subject: [PATCH 116/423] [HinduTamilBridge] Fix timestamp again (#4142) --- bridges/HinduTamilBridge.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bridges/HinduTamilBridge.php b/bridges/HinduTamilBridge.php index cab04171..1b556ed8 100644 --- a/bridges/HinduTamilBridge.php +++ b/bridges/HinduTamilBridge.php @@ -66,7 +66,7 @@ class HinduTamilBridge extends FeedExpander $date = $dom->find('p span.date', 1); if ($date) { - $item['timestamp'] = $date->innertext . ' IST'; + $item['timestamp'] = $this->convertToRFC3339($date->plaintext); } $content = $dom->find('#pgContentPrint', 0); @@ -88,4 +88,16 @@ class HinduTamilBridge extends FeedExpander return $content; } + + private function convertToRFC3339($DateString) + { + $timestamp = strtotime(trim($DateString)); + + if ($timestamp !== false) { + $rfc3339DateTime = date('Y-m-d\TH:i:s', $timestamp) . '+05:30'; + return $rfc3339DateTime; + } else { + return null; + } + } } From 8bf153705463582d9949e21aaf167cb742b5c197 Mon Sep 17 00:00:00 2001 From: Niehztog <Niehztog@users.noreply.github.com> Date: Thu, 4 Jul 2024 20:53:16 +0200 Subject: [PATCH 117/423] delete obsolete bridge (#4143) --- bridges/NiusBridge.php | 51 ------------------------------------------ 1 file changed, 51 deletions(-) delete mode 100644 bridges/NiusBridge.php diff --git a/bridges/NiusBridge.php b/bridges/NiusBridge.php deleted file mode 100644 index 0be6e89c..00000000 --- a/bridges/NiusBridge.php +++ /dev/null @@ -1,51 +0,0 @@ -<?php - -class NiusBridge extends XPathAbstract -{ - const NAME = 'Nius'; - const URI = 'https://www.nius.de/news'; - const DESCRIPTION = 'Die Stimme der Mehrheit'; - const MAINTAINER = 'Niehztog'; - - const CACHE_TIMEOUT = 3600; - - const FEED_SOURCE_URL = 'https://www.nius.de/news'; - const XPATH_EXPRESSION_ITEM = './/div[contains(@class, "compact-story") or contains(@class, "regular-story")]'; - const XPATH_EXPRESSION_ITEM_TITLE = './/h2[@class="title"]//node()'; - const XPATH_EXPRESSION_ITEM_CONTENT = self::XPATH_EXPRESSION_ITEM_TITLE; - const XPATH_EXPRESSION_ITEM_URI = './/a[1]/@href'; - - const XPATH_EXPR_AUTHOR_PART1 = 'normalize-space(.//span[@class="author"]/text()[1])'; - const XPATH_EXPR_AUTHOR_PART2 = 'normalize-space(.//span[@class="author"]/text()[2])'; - const XPATH_EXPRESSION_ITEM_AUTHOR = 'substring-after(concat(' . self::XPATH_EXPR_AUTHOR_PART1 . ', " ", ' . self::XPATH_EXPR_AUTHOR_PART2 . '), " ")'; - - const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img[@sizes and @alt="Article background picture"]/@src'; - const XPATH_EXPRESSION_ITEM_CATEGORIES = './/div[@class="subtitle"]/text()'; - const SETTING_FIX_ENCODING = false; - - public function getIcon() - { - return 'https://www.nius.de/favicon.ico'; - } - - protected function formatItemTitle($value) - { - return strip_tags($value); - } - - protected function formatItemContent($value) - { - return strip_tags($value); - } - - protected function cleanMediaUrl($mediaUrl) - { - $result = preg_match('~https:\/\/www\.nius\.de\/_next\/image\?url=(.*)\?~', $mediaUrl, $matches); - return $result ? $matches[1] . '#.jpg' : $mediaUrl; - } - - protected function generateItemId(FeedItem $item) - { - return substr($item->getURI(), strrpos($item->getURI(), '/') + 1); - } -} From 4539eb69aa5f2bea3dde61687c08164062cba767 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Thu, 4 Jul 2024 20:53:49 +0200 Subject: [PATCH 118/423] [GolemBridge] fix youtube links (#4144) --- bridges/GolemBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index b52d3c2f..7f59ee90 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -109,10 +109,10 @@ class GolemBridge extends FeedExpander //built youtube iframes foreach ($article->find('.embedcontent') as &$embedcontent) { $ytscript = $embedcontent->find('script', 0); - if (preg_match('/www.youtube.com.*?\"/', $ytscript->innertext, $link)) { - $link = 'https://' . str_replace('\\', '', $link[0]); + if (preg_match('/(www.youtube.com.*?)\"/', $ytscript->innertext, $link)) { + $link = 'https://' . str_replace('\\', '', $link[1]); $embedcontent->innertext .= <<<EOT - <iframe width="560" height="315" src=$link title="YouTube video player" frameborder="0" + <iframe width="560" height="315" src="$link" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>'; EOT; From da8cfdf1798e4fee09288b65bfc015775627875d Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Sat, 6 Jul 2024 02:09:47 +0530 Subject: [PATCH 119/423] [HinduTamilBridge] refactor (#4146) * [HinduTamilBridge] refactor * [HinduTamilBridge] fixed lint * [HinduTamilBridge] fixed lint 2 * Update HinduTamilBridge.php --- bridges/HinduTamilBridge.php | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/bridges/HinduTamilBridge.php b/bridges/HinduTamilBridge.php index 1b556ed8..d12f5131 100644 --- a/bridges/HinduTamilBridge.php +++ b/bridges/HinduTamilBridge.php @@ -63,18 +63,18 @@ class HinduTamilBridge extends FeedExpander protected function parseItem($item) { $dom = getSimpleHTMLDOMCached($item['uri']); - - $date = $dom->find('p span.date', 1); - if ($date) { - $item['timestamp'] = $this->convertToRFC3339($date->plaintext); - } - $content = $dom->find('#pgContentPrint', 0); - if (!$content) { + + if ($content === null) { return $item; } - $image = $dom->find('#LoadArticle figure', 0); + $date = $dom->find('p span.date', 1); + if ($date) { + $item['timestamp'] = $this->toRFC3339($date->plaintext); + } + + $image = $dom->find('#LoadArticle figure', 0) ?? ''; $item['content'] = $image . $this->cleanContent($content); return $item; @@ -89,15 +89,14 @@ class HinduTamilBridge extends FeedExpander return $content; } - private function convertToRFC3339($DateString) + private function toRFC3339($dateString) { - $timestamp = strtotime(trim($DateString)); + $timestamp = strtotime(trim($dateString)); - if ($timestamp !== false) { - $rfc3339DateTime = date('Y-m-d\TH:i:s', $timestamp) . '+05:30'; - return $rfc3339DateTime; - } else { + if ($timestamp === false) { return null; } + + return date('Y-m-d\TH:i:s', $timestamp) . '+05:30'; } } From f7ddbcd7335e3879c13035e98b1a40c32ff5a710 Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Sun, 28 Jul 2024 21:58:08 +0200 Subject: [PATCH 120/423] [GBAtemp] Fix title extraction (#4151) Fix title extraction for news and reviews --- bridges/GBAtempBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/GBAtempBridge.php b/bridges/GBAtempBridge.php index 4aa04799..361e3f1d 100644 --- a/bridges/GBAtempBridge.php +++ b/bridges/GBAtempBridge.php @@ -31,7 +31,7 @@ class GBAtempBridge extends BridgeAbstract $img = $this->findItemImage($newsItem, 'a.news_image'); $time = $this->findItemDate($newsItem); $author = $newsItem->find('a.username', 0)->plaintext; - $title = $this->decodeHtmlEntities($newsItem->find('h3.news_title', 0)->plaintext); + $title = $this->decodeHtmlEntities($newsItem->find('h2.news_title', 0)->plaintext); $content = $this->fetchPostContent($url, self::URI); $this->items[] = $this->buildItem($url, $title, $author, $time, $img, $content); unset($newsItem); // Some items are heavy, freeing the item proactively helps saving memory @@ -41,7 +41,7 @@ class GBAtempBridge extends BridgeAbstract foreach ($html->find('li.portal_review') as $reviewItem) { $url = urljoin(self::URI, $reviewItem->find('a.review_boxart', 0)->href); $img = $this->findItemImage($reviewItem, 'a.review_boxart'); - $title = $this->decodeHtmlEntities($reviewItem->find('h2.review_title', 0)->plaintext); + $title = $this->decodeHtmlEntities($reviewItem->find('div.review_title', 0)->find('h2', 0)->plaintext); $content = getSimpleHTMLDOMCached($url); $author = $content->find('span.author--name', 0)->plaintext; $time = $this->findItemDate($content); From 00d524287141179ca591065ad7b8b23db4ffcaab Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Mon, 29 Jul 2024 01:30:36 +0530 Subject: [PATCH 121/423] [GithubTrendingBridge] Add support for spoken languages (#4149) * [GithubTrendingBridge] Add support for spoken languages * Update GithubTrendingBridge.php --- bridges/GithubTrendingBridge.php | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/bridges/GithubTrendingBridge.php b/bridges/GithubTrendingBridge.php index 0f8e5e96..2ce47270 100644 --- a/bridges/GithubTrendingBridge.php +++ b/bridges/GithubTrendingBridge.php @@ -586,16 +586,18 @@ class GithubTrendingBridge extends BridgeAbstract 'Monthly' => 'monthly', ], 'defaultValue' => 'today' + ], + 'spokenLanguage' => [ + 'name' => 'Spoken Language Code', + 'type' => 'text', + 'exampleValue' => 'en', ] ] - ]; public function collectData() { - $params = ['since' => urlencode($this->getInput('date_range'))]; - $url = self::URI . '/' . $this->getInput('language') . '?' . http_build_query($params); - + $url = $this->constructUrl(); $html = getSimpleHTMLDOM($url); $this->items = []; @@ -630,4 +632,32 @@ class GithubTrendingBridge extends BridgeAbstract return parent::getName(); } + + private function constructUrl() + { + $url = self::URI; + $language = $this->getInput('language'); + $dateRange = $this->getInput('date_range'); + $spokenLanguage = $this->getInput('spokenLanguage'); + + if (!empty($language)) { + $url .= '/' . $language; + } + + $queryParams = []; + + if (!empty($dateRange)) { + $queryParams['since'] = $dateRange; + } + + if (!empty($spokenLanguage)) { + $queryParams['spoken_language_code'] = trim($spokenLanguage); + } + + if (!empty($queryParams)) { + $url .= '?' . http_build_query($queryParams); + } + + return $url; + } } From 376e711f0350698107a88d268c5f825ada593b41 Mon Sep 17 00:00:00 2001 From: "Dmitry R." <me@mazy.wtf> Date: Sun, 28 Jul 2024 22:02:47 +0200 Subject: [PATCH 122/423] [NovayaGazetaEuropeBridge]: fix warnings (#4154) --- bridges/NovayaGazetaEuropeBridge.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bridges/NovayaGazetaEuropeBridge.php b/bridges/NovayaGazetaEuropeBridge.php index ec288f2b..89d31a94 100644 --- a/bridges/NovayaGazetaEuropeBridge.php +++ b/bridges/NovayaGazetaEuropeBridge.php @@ -41,6 +41,9 @@ class NovayaGazetaEuropeBridge extends BridgeAbstract $data = json_decode($json); foreach ($data->records as $record) { + if (!isset($record->blocks)) { + continue; + } foreach ($record->blocks as $block) { if (!property_exists($block, 'date')) { continue; From 049af3cef77721fe2d999c7ecefe899116779f82 Mon Sep 17 00:00:00 2001 From: Tostiman <18124323+t0stiman@users.noreply.github.com> Date: Sun, 28 Jul 2024 22:03:35 +0200 Subject: [PATCH 123/423] [HardwareInfoBridge] delete bridge for discontinued website (#4124) --- bridges/HardwareInfoBridge.php | 66 ---------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 bridges/HardwareInfoBridge.php diff --git a/bridges/HardwareInfoBridge.php b/bridges/HardwareInfoBridge.php deleted file mode 100644 index dc32c33a..00000000 --- a/bridges/HardwareInfoBridge.php +++ /dev/null @@ -1,66 +0,0 @@ -<?php - -class HardwareInfoBridge extends FeedExpander -{ - const NAME = 'Hardware Info Bridge'; - const URI = 'https://nl.hardware.info/'; - const DESCRIPTION = 'Tech news from hardware.info (Dutch)'; - const MAINTAINER = 't0stiman'; - const DONATION_URI = 'https://ko-fi.com/tostiman'; - - public function collectData() - { - $this->collectExpandableDatas('https://nl.hardware.info/updates/all.rss', 10); - } - - protected function parseItem(array $item) - { - $itemUrl = $item['uri']; - $articlePage = getSimpleHTMLDOMCached($itemUrl); - - $article = $articlePage->find('div.article__content', 0); - - //everything under the social bar is not part of the article, remove it - $reachedEndOfArticle = false; - - foreach ($article->find('*') as $child) { - if ( - !$reachedEndOfArticle && isset($child->attr['class']) - && $child->attr['class'] == 'article__content__social-bar' - ) { - $reachedEndOfArticle = true; - } - - if ($reachedEndOfArticle) { - $child->outertext = ''; - } - } - - //get rid of some more elements we don't need - $to_remove_selectors = [ - 'script', - 'div.incontent', - 'div.article__content__social-bar', - 'div#revealNewsTip', - 'div.article__previous_next' - ]; - - foreach ($to_remove_selectors as $selector) { - foreach ($article->find($selector) as $found) { - $found->outertext = ''; - } - } - - // convert iframes to links. meant for embedded YouTube videos. - foreach ($article->find('iframe') as $found) { - $iframeUrl = $found->getAttribute('src'); - - if ($iframeUrl) { - $found->outertext = '<a href="' . $iframeUrl . '">' . $iframeUrl . '</a>'; - } - } - - $item['content'] = $article; - return $item; - } -} From 2fcba49433886ef2e1237f1bd256aadcea494309 Mon Sep 17 00:00:00 2001 From: enwuenwu <108224417+enwuenwu@users.noreply.github.com> Date: Sun, 28 Jul 2024 20:11:48 +0000 Subject: [PATCH 124/423] [Mailman2Bridge] fix message separation and improve "From_ lines" disambiguation (#4156) * [Mailman2Bridge.php] enable PCRE_MULTILINE pattern modifier Enable PCRE_MULTILINE pattern modifier on mbox content parsing. Without it parsing monthly archives results in only a single message each. * [Mailman2Bridge.php] extend mbox "From_ lines" pattern Extend PCRE pattern matching individual "From_ lines" used to split single messages in mbox content. In addition to the matching line having to start with 'From ' it now also has to end with time and date (hh:mm:ss yyyy). This makes the pattern slightly more robust against accidental matches when a line within the actual message body starts with 'From ' which Mailman 2 (Pipermail) may not be configured to disambiguate. * [Mailman2Bridge.php] remove trailing slash from URI constant --------- Co-authored-by: enwu <108224417+8279279374@users.noreply.github.com> --- bridges/Mailman2Bridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/Mailman2Bridge.php b/bridges/Mailman2Bridge.php index ad0d8110..6b620c03 100644 --- a/bridges/Mailman2Bridge.php +++ b/bridges/Mailman2Bridge.php @@ -3,7 +3,7 @@ class Mailman2Bridge extends BridgeAbstract { const NAME = 'Mailman2Bridge'; - const URI = 'https://list.org/'; + const URI = 'https://list.org'; const MAINTAINER = 'imagoiq'; const CACHE_TIMEOUT = 60 * 30; // 30m const DESCRIPTION = 'Fetch latest messages from Mailman 2 archive (Pipermail)'; @@ -68,7 +68,7 @@ class Mailman2Bridge extends BridgeAbstract throw new \Exception('Failed to gzdecode'); } } - $mboxParts = preg_split('/^From /', $data); + $mboxParts = preg_split('/^From\s.+\d{2}:\d{2}:\d{2}\s\d{4}$/m', $data); // Drop the first element which is always an empty string array_shift($mboxParts); $mboxMails = array_reverse($mboxParts); From a1b3e596fc24845d5c74a80c061bf3eee94618c0 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Sun, 28 Jul 2024 22:21:14 +0200 Subject: [PATCH 125/423] [AnisearchBridge.php] fixed youtube link (#4159) $trailer->{'data-xsrc'} wasn't read correctly in EOT context --- bridges/AnisearchBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/AnisearchBridge.php b/bridges/AnisearchBridge.php index d5aad1c9..869b3049 100644 --- a/bridges/AnisearchBridge.php +++ b/bridges/AnisearchBridge.php @@ -68,8 +68,9 @@ class AnisearchBridge extends BridgeAbstract if (isset($trailerlink)) { $trailersite = getSimpleHTMLDOM($baseurl . $trailerlink->href); $trailer = $trailersite->find('div#player > iframe', 0); + $trailer = $trailer->{'data-xsrc'}; $ytlink = <<<EOT - <br /><iframe width="560" height="315" src="' . $trailer->{'data-xsrc'} . '" title="YouTube video player" + <br /><iframe width="560" height="315" src="$trailer" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>'; EOT; From bba225dfe85e8db237df114991657708ec4210c0 Mon Sep 17 00:00:00 2001 From: Eugene Molotov <eugene.molotov@yandex.ru> Date: Mon, 29 Jul 2024 01:33:48 +0500 Subject: [PATCH 126/423] [RutubeBridge] New option to fetch video from search results (#4162) --- bridges/RutubeBridge.php | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/bridges/RutubeBridge.php b/bridges/RutubeBridge.php index 452dbde4..6ec830aa 100644 --- a/bridges/RutubeBridge.php +++ b/bridges/RutubeBridge.php @@ -24,6 +24,13 @@ class RutubeBridge extends BridgeAbstract 'required' => true ], ], + 'По результатам поиска' => [ + 's' => [ + 'name' => 'Запрос', + 'exampleValue' => 'SUREN', + 'required' => true, + ] + ] ]; protected $title; @@ -34,6 +41,8 @@ class RutubeBridge extends BridgeAbstract return self::URI . '/channel/' . strval($this->getInput('c')) . '/videos/'; } elseif ($this->getInput('p')) { return self::URI . '/plst/' . strval($this->getInput('p')) . '/'; + } elseif ($this->getInput('s')) { + return self::URI . '/search/?suggest=1&query=' . strval($this->getInput('s')); } else { return parent::getURI(); } @@ -60,7 +69,7 @@ class RutubeBridge extends BridgeAbstract return json_decode(str_replace('\x', '\\\x', $matches[1])); } - public function collectData() + private function getVideosFromReduxState() { $link = $this->getURI(); @@ -73,6 +82,26 @@ class RutubeBridge extends BridgeAbstract } elseif ($this->getInput('p')) { $videos = $reduxState->playlist->data->results; $this->title = $reduxState->playlist->title; + } elseif ($this->getInput('s')) { + $this->title = 'Поиск ' . $this->getInput('s'); + } + + return $videos; + } + + private function getVideosFromSearchAPI() + { + $contents = getContents(self::URI . '/api/search/video/?suggest=1&client=wdp&query=' . $this->getInput('s')); + $json = json_decode($contents); + return $json->results; + } + + public function collectData() + { + if ($this->getInput('c') || $this->getInput('p')) { + $videos = $this->getVideosFromReduxState(); + } else { + $videos = $this->getVideosFromSearchAPI(); } foreach ($videos as $video) { From d28a0fd94b0f863cf630e327ed5ad646a05909b9 Mon Sep 17 00:00:00 2001 From: Eugene Molotov <eugene.molotov@yandex.ru> Date: Mon, 29 Jul 2024 01:34:12 +0500 Subject: [PATCH 127/423] [Vk2Bridge] Handling albums (#4163) --- bridges/Vk2Bridge.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bridges/Vk2Bridge.php b/bridges/Vk2Bridge.php index 0bc0879f..6fecba84 100644 --- a/bridges/Vk2Bridge.php +++ b/bridges/Vk2Bridge.php @@ -168,6 +168,12 @@ class Vk2Bridge extends BridgeAbstract $ret .= "* {$text}: {$votes} ({$rate}%)<br />"; } $ret .= '</p>'; + } elseif ($attachment['type'] == 'album') { + $album = $attachment['album']; + $url = "https://vk.com/album{$album['owner_id']}_{$album['id']}"; + $title = 'Альбом: ' . $album['title']; + $photo = $this->getImageURLWithLargestWidth($album['thumb']['sizes']); + $ret .= "<p><a href='{$url}'><img src='{$photo}' alt='{$title}'><br>{$title}</a></p>"; } elseif (!in_array($attachment['type'], ['video', 'audio', 'doc'])) { $ret .= "<p>Unknown attachment type: {$attachment['type']}</p>"; } From f773878459c3c12ee01b09ef2a805c57760656ea Mon Sep 17 00:00:00 2001 From: Pavel Korytov <thexcloud@gmail.com> Date: Sun, 28 Jul 2024 23:41:08 +0300 Subject: [PATCH 128/423] [EconomistWorldInBriefBridge] Add cookie to options (#4165) * [EconomistWorldInBriefBridge] Add cookie * [EconomistWorldInBriefBridge] Add docs * [EconomistWorldInBriefBridge] Best-effort to work without cookie --- bridges/EconomistWorldInBriefBridge.php | 24 ++++++++++++++++++++++-- docs/10_Bridge_Specific/Economist.md | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 docs/10_Bridge_Specific/Economist.md diff --git a/bridges/EconomistWorldInBriefBridge.php b/bridges/EconomistWorldInBriefBridge.php index 47782a51..4e65b15f 100644 --- a/bridges/EconomistWorldInBriefBridge.php +++ b/bridges/EconomistWorldInBriefBridge.php @@ -9,6 +9,12 @@ class EconomistWorldInBriefBridge extends BridgeAbstract const CACHE_TIMEOUT = 3600; // 1 hour const DESCRIPTION = 'Returns stories from the World in Brief section'; + const CONFIGURATION = [ + 'cookie' => [ + 'required' => false, + ] + ]; + const PARAMETERS = [ '' => [ 'splitGobbets' => [ @@ -41,7 +47,19 @@ class EconomistWorldInBriefBridge extends BridgeAbstract public function collectData() { - $html = getSimpleHTMLDOM(self::URI); + $headers = []; + if ($this->getOption('cookie')) { + $headers = [ + 'Authority: www.economist.com', + 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-language: en-US,en;q=0.9', + 'Cache-control: max-age=0', + 'Cookie: ' . $this->getOption('cookie'), + 'Upgrade-insecure-requests: 1', + 'User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36' + ]; + } + $html = getSimpleHTMLDOM(self::URI, $headers); $gobbets = $html->find('._gobbets', 0); if ($this->getInput('splitGobbets') == 1) { $this->splitGobbets($gobbets); @@ -50,7 +68,9 @@ class EconomistWorldInBriefBridge extends BridgeAbstract }; if ($this->getInput('agenda') == 1) { $articles = $html->find('._articles', 0); - $this->collectArticles($articles); + if ($articles != null) { + $this->collectArticles($articles); + } } if ($this->getInput('quote') == 1) { $quote = $html->find('._quote-container', 0); diff --git a/docs/10_Bridge_Specific/Economist.md b/docs/10_Bridge_Specific/Economist.md new file mode 100644 index 00000000..1a792eb8 --- /dev/null +++ b/docs/10_Bridge_Specific/Economist.md @@ -0,0 +1,18 @@ +# EconomistWorldInBriefBridge + +In May 2024, The Economist finally fixed its paywall, and it started requiring authorization. Which means you can't use this bridge unless you have an active subscription. + +If you do, the way to use the bridge is to snitch a cookie: +1. Log in to The Economist +2. Open DevTools (Chrome DevTools or Firefox Developer Tools) +2. Go to https://www.economist.com/the-world-in-brief +3. In DevTools, go to the "Network" tab, there select the first request (`the-world-in-brief`) and copy the value of the `Cookie:` header from "Request Headers". + +The cookie lives three months. + +Once you've done this, add the cookie to your `config.ini.php`: + +``` +[EconomistWorldInBriefBridge] +cookie = "<value>" +``` From 8dd56bca051c69462fd1464d23336c8b07c2aee4 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger <chris.schabesberger@mailbox.org> Date: Sun, 28 Jul 2024 22:42:18 +0200 Subject: [PATCH 129/423] fix bulletpoints for nordbayern (#4166) --- bridges/NordbayernBridge.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bridges/NordbayernBridge.php b/bridges/NordbayernBridge.php index aa32f4ba..a5f34817 100644 --- a/bridges/NordbayernBridge.php +++ b/bridges/NordbayernBridge.php @@ -92,6 +92,8 @@ class NordbayernBridge extends BridgeAbstract $content .= self::getUseFullContent($element); } elseif ($element->tag === 'picture') { $content .= self::getValidImage($element); + } elseif ($element->tag === 'ul') { + $content .= $element; } } return $content; From 955fb6f315ca86693b1b3185b47ef313168c6bd2 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Mon, 29 Jul 2024 00:18:28 +0200 Subject: [PATCH 130/423] fix(reddit): increase default cache ttl (#4168) --- bridges/RedditBridge.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 7ece0e15..434ae74a 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -10,6 +10,7 @@ class RedditBridge extends BridgeAbstract const MAINTAINER = 'dawidsowa'; const NAME = 'Reddit Bridge'; const URI = 'https://old.reddit.com'; + const CACHE_TIMEOUT = 60 * 60 * 2; // 2h const DESCRIPTION = 'Return hot submissions from Reddit'; const PARAMETERS = [ @@ -107,9 +108,8 @@ class RedditBridge extends BridgeAbstract // 403 Forbidden // This can possibly mean that reddit has permanently blocked this server's ip address $this->cache->set($forbiddenKey, true, 60 * 61); - } - if ($e->getCode() === 429) { - $this->cache->set($rateLimitKey, true, 60 * 16); + } elseif ($e->getCode() === 429) { + $this->cache->set($rateLimitKey, true, 60 * 61); } throw $e; } @@ -143,10 +143,14 @@ class RedditBridge extends BridgeAbstract $flareInput = $this->getInput('f'); foreach ($subreddits as $subreddit) { - $version = 'v0.0.1'; + $version = 'v0.0.2'; $useragent = "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"; $url = self::createUrl($search, $flareInput, $subreddit, $user, $section, $this->queriedContext); - $json = getContents($url, ['User-Agent: ' . $useragent]); + + $response = getContents($url, ['User-Agent: ' . $useragent], [], true); + + $json = $response['content']; + $parsedJson = Json::decode($json, false); foreach ($parsedJson->data->children as $post) { From 6d81d6d3068a56e121c2c448f84f93874172c25c Mon Sep 17 00:00:00 2001 From: Zack Puhl <31320277+NotsoanoNimus@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:53:14 -0400 Subject: [PATCH 131/423] [RumbleBridge] Facelift, Validation, & Livestreams (#4160) * [RumbleBridge] Facelift+media types (livestreams) * [RumbleBridge] Remove 'required' from list input. * [RumbleBridge] lint --- bridges/RumbleBridge.php | 49 +++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/bridges/RumbleBridge.php b/bridges/RumbleBridge.php index f6bfca7d..a8841e00 100644 --- a/bridges/RumbleBridge.php +++ b/bridges/RumbleBridge.php @@ -2,10 +2,10 @@ class RumbleBridge extends BridgeAbstract { - const NAME = 'rumble.com bridge'; - const URI = 'https://rumble.com'; - const DESCRIPTION = 'Fetches the latest channel/user videos'; - const MAINTAINER = 'dvikan'; + const NAME = 'Rumble.com Bridge'; + const URI = 'https://rumble.com/'; + const DESCRIPTION = 'Fetches the latest channel/user videos and livestreams.'; + const MAINTAINER = 'dvikan, NotsoanoNimus'; const CACHE_TIMEOUT = 60 * 60; // 1h const PARAMETERS = [ [ @@ -13,15 +13,19 @@ class RumbleBridge extends BridgeAbstract 'name' => 'Account', 'type' => 'text', 'required' => true, + 'title' => 'Name of the target account to create into a feed.', 'defaultValue' => 'bjornandreasbullhansen', ], 'type' => [ + 'name' => 'Account Type', 'type' => 'list', - 'name' => 'Type', + 'title' => 'The type of profile to create a feed from.', 'values' => [ - 'Channel' => 'channel', - 'User' => 'user', - ] + 'Channel (All)' => 'channel', + 'Channel Videos' => 'channel-videos', + 'Channel Livestreams' => 'channel-livestream', + 'User (All)' => 'user', + ], ], ] ]; @@ -30,12 +34,28 @@ class RumbleBridge extends BridgeAbstract { $account = $this->getInput('account'); $type = $this->getInput('type'); + $url = self::getURI(); - if ($type === 'channel') { - $url = "https://rumble.com/c/$account"; + if (!preg_match('#^[\w\-_.@]+$#', $account) || strlen($account) > 64) { + throw new \Exception('Invalid target account.'); } - if ($type === 'user') { - $url = "https://rumble.com/user/$account"; + + switch ($type) { + case 'user': + $url .= "user/$account"; + break; + case 'channel': + $url .= "c/$account"; + break; + case 'channel-videos': + $url .= "c/$account/videos"; + break; + case 'channel-livestream': + $url .= "c/$account/livestreams"; + break; + default: + // Shouldn't ever happen. + throw new \Exception('Invalid media type.'); } $dom = getSimpleHTMLDOM($url); @@ -57,6 +77,9 @@ class RumbleBridge extends BridgeAbstract public function getName() { - return 'Rumble.com ' . $this->getInput('account'); + if ($this->getInput('account')) { + return 'Rumble.com - ' . $this->getInput('account'); + } + return self::NAME; } } From 22b39e3fcd75705db99f53b74eb03aa1e16d0fab Mon Sep 17 00:00:00 2001 From: Zack Puhl <31320277+NotsoanoNimus@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:53:39 -0400 Subject: [PATCH 132/423] [EBayBridge] Repair & Augment the eBay Feed (#4157) * [EBayBridge]: discount details; fix DOM parsing * [EBayBridge] Ending slash. No "www.ebay.commyhijack.net", for example. * [EBayBridge] Trim discountLine details when set. * [EBayBridge] Refactor and update content * shameless self-addition to CONTRIBUTORS.md * [EBayBridge] Toggle original search links w/ checkbox * [EBayBridge] oops: fix introduced XSS vuln * [EBayBridge] Fix linting error: use array_column * [EBayBridge] fix compat with <php8 --- CONTRIBUTORS.md | 1 + bridges/EBayBridge.php | 118 +++++++++++++++++++++++++++++++---------- 2 files changed, 92 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f1080743..922d9453 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -144,6 +144,7 @@ * [Niehztog](https://github.com/Niehztog) * [NikNikYkt](https://github.com/NikNikYkt) * [Nono-m0le](https://github.com/Nono-m0le) +* [NotsoanoNimus](https://github.com/NotsoanoNimus) * [obsiwitch](https://github.com/obsiwitch) * [Ololbu](https://github.com/Ololbu) * [ORelio](https://github.com/ORelio) diff --git a/bridges/EBayBridge.php b/bridges/EBayBridge.php index 87958164..11b90bf8 100644 --- a/bridges/EBayBridge.php +++ b/bridges/EBayBridge.php @@ -5,15 +5,21 @@ class EBayBridge extends BridgeAbstract const NAME = 'eBay'; const DESCRIPTION = 'Returns the search results from the eBay auctioning platforms'; const URI = 'https://www.eBay.com'; - const MAINTAINER = 'wrobelda'; + const MAINTAINER = 'NotsoanoNimus, wrobelda'; const PARAMETERS = [[ 'url' => [ 'name' => 'Search URL', 'title' => 'Copy the URL from your browser\'s address bar after searching for your items and paste it here', - 'pattern' => '^(https:\/\/)?(www\.)?(befr\.|benl\.)?ebay\.(com|com\.au|at|be|ca|ch|cn|es|fr|de|com\.hk|ie|it|com\.my|nl|ph|pl|com\.sg|co\.uk).*$', + 'pattern' => '^(https:\/\/)?(www\.)?(befr\.|benl\.)?ebay\.(com|com\.au|at|be|ca|ch|cn|es|fr|de|com\.hk|ie|it|com\.my|nl|ph|pl|com\.sg|co\.uk)\/.*$', 'exampleValue' => 'https://www.ebay.com/sch/i.html?_nkw=atom+rss', 'required' => true, - ] + ], + 'includesSearchLink' => [ + 'name' => 'Include Original Search Link', + 'title' => 'Whether or not each feed item should include the original search query link to eBay which was used to find the given listing.', + 'type' => 'checkbox', + 'defaultValue' => false, + ], ]]; public function getURI() @@ -23,6 +29,10 @@ class EBayBridge extends BridgeAbstract $uri = trim(preg_replace('/([?&])_sop=[^&]+(&|$)/', '$1', $this->getInput('url')), '?&/'); $uri .= (parse_url($uri, PHP_URL_QUERY) ? '&' : '?') . '_sop=10'; + // Ensure the List View is used instead of the Gallery View. + $uri = trim(preg_replace('/[?&]_dmd=[^&]+(&|$)/i', '$1', $uri), '?&/'); + $uri .= '&_dmd=1'; + return $uri; } else { return parent::getURI(); @@ -46,7 +56,7 @@ class EBayBridge extends BridgeAbstract }); if ($searchQuery) { - return $searchQuery[0]; + return 'eBay - ' . $searchQuery[0]; } return parent::getName(); @@ -61,44 +71,88 @@ class EBayBridge extends BridgeAbstract $inexactMatches->remove(); } + // Remove "NEW LISTING" labels: we sort by the newest, so this is redundant. + foreach ($html->find('.LIGHT_HIGHLIGHT') as $new_listing_label) { + $new_listing_label->remove(); + } + $results = $html->find('ul.srp-results > li.s-item'); foreach ($results as $listing) { $item = []; - // Remove "NEW LISTING" label, we sort by the newest, so this is redundant - foreach ($listing->find('.LIGHT_HIGHLIGHT') as $new_listing_label) { - $new_listing_label->remove(); + // Define a closure to shorten the ugliness of querying the current listing. + $find = function ($query, $altText = '') use ($listing) { + return $listing->find($query, 0)->plaintext ?? $altText; + }; + + $item['title'] = $find('.s-item__title'); + if (!$item['title']) { + // Skip entries where the title cannot be found (for w/e reason). + continue; } - $listingTitle = $listing->find('.s-item__title', 0); - if ($listingTitle) { - $item['title'] = $listingTitle->plaintext; - } - - $subtitle = implode('', $listing->find('.s-item__subtitle')); - - $listingUrl = $listing->find('.s-item__link', 0); - if ($listingUrl) { - $item['uri'] = $listingUrl->href; + // It appears there may be more than a single 'subtitle' subclass in the listing. Collate them. + $subtitles = $listing->find('.s-item__subtitle'); + if (is_array($subtitles)) { + $subtitle = trim(implode(' ', array_column($subtitles, 'plaintext'))); } else { - $item['uri'] = null; + $subtitle = trim($subtitles->plaintext ?? ''); } + // Get the listing's link and uid. + $itemUri = $listing->find('.s-item__link', 0); + if ($itemUri) { + $item['uri'] = $itemUri->href; + } if (preg_match('/.*\/itm\/(\d+).*/i', $item['uri'], $matches)) { $item['uid'] = $matches[1]; } - $priceDom = $listing->find('.s-item__details > .s-item__detail > .s-item__price', 0); - $price = $priceDom->plaintext ?? 'N/A'; + // Price should be fetched on its own so we can provide the alt text without complication. + $price = $find('.s-item__price', '[NO PRICE]'); - $shippingFree = $listing->find('.s-item__details > .s-item__detail > .s-item__freeXDays', 0)->plaintext ?? ''; - $localDelivery = $listing->find('.s-item__details > .s-item__detail > .s-item__localDelivery', 0)->plaintext ?? ''; - $logisticsCost = $listing->find('.s-item__details > .s-item__detail > .s-item__logisticsCost', 0)->plaintext ?? ''; + // Map a list of dynamic variable names to their subclasses within the listing. + // This is just a bit of sugar to make this cleaner and more maintainable. + $propertyMappings = [ + 'additionalPrice' => '.s-item__additional-price', + 'discount' => '.s-item__discount', + 'shippingFree' => '.s-item__freeXDays', + 'localDelivery' => '.s-item__localDelivery', + 'logisticsCost' => '.s-item__logisticsCost', + 'location' => '.s-item__location', + 'obo' => '.s-item__formatBestOfferEnabled', + 'sellerInfo' => '.s-item__seller-info-text', + 'bids' => '.s-item__bidCount', + 'timeLeft' => '.s-item__time-left', + 'timeEnd' => '.s-item__time-end', + ]; - $location = $listing->find('.s-item__details > .s-item__detail > .s-item__location', 0)->plaintext ?? ''; + foreach ($propertyMappings as $k => $v) { + $$k = $find($v); + } - $sellerInfo = $listing->find('.s-item__seller-info-text', 0)->plaintext ?? ''; + // When an additional price detail or discount is defined, create the 'discountLine'. + if ($additionalPrice || $discount) { + $discountLine = '<br /><em>(' + . trim($additionalPrice ?? '') + . '; ' . trim($discount ?? '') + . ')</em>'; + } + // Prepend the time-left info with a comma if the right details were found. + $timeInfo = trim($timeLeft . ' ' . $timeEnd); + if ($timeInfo) { + $timeInfo = ', ' . $timeInfo; + } + + // Set the listing type. + if ($bids) { + $listingTypeDetails = "Auction: {$bids}{$timeInfo}"; + } else { + $listingTypeDetails = 'Buy It Now'; + } + + // Acquire the listing's primary image and atach it. $image = $listing->find('.s-item__image-wrapper > img', 0); if ($image) { // Not quite sure why append fragment here @@ -106,11 +160,21 @@ class EBayBridge extends BridgeAbstract $item['enclosures'] = [$imageUrl]; } + // Include the original search link, if specified. + if ($this->getInput('includesSearchLink')) { + $searchLink = '<p><small><a target="_blank" href="' . e($this->getURI()) . '">View Search</a></small></p>'; + } + + // Build the final item's content to display and add the item onto the list. $item['content'] = <<<CONTENT <p>$sellerInfo $location</p> -<p><span style="font-weight:bold">$price</span> $shippingFree $localDelivery $logisticsCost<span></span></p> -<p>$subtitle</p> +<p><strong>$price</strong> $obo ($listingTypeDetails) + $discountLine + <br /><small>$shippingFree $localDelivery $logisticsCost</small></p> +<p>{$subtitle}</p> +$searchLink CONTENT; + $this->items[] = $item; } } From cb91afbd71d545013b6c59f6841fbc4f64311a58 Mon Sep 17 00:00:00 2001 From: MarKoeh <75181140+Mar-Koeh@users.noreply.github.com> Date: Tue, 30 Jul 2024 22:08:18 +0200 Subject: [PATCH 133/423] [ARDMediathekBridge] fixing API URL, start using show title (#4170) (#4172) The bridge stopped working after the API server stopped accepting a trailing slash after the ID in the URL. This is being fixed. Also, the show title in the JSON was ignored. This is being fixed as well --- bridges/ARDMediathekBridge.php | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bridges/ARDMediathekBridge.php b/bridges/ARDMediathekBridge.php index 6de8dad7..da11dd64 100644 --- a/bridges/ARDMediathekBridge.php +++ b/bridges/ARDMediathekBridge.php @@ -40,6 +40,11 @@ class ARDMediathekBridge extends BridgeAbstract * @const IMAGEWIDTHPLACEHOLDER */ const IMAGEWIDTHPLACEHOLDER = '{width}'; + /** + * Title of the current show + * @var string + */ + private $title; const PARAMETERS = [ [ @@ -72,7 +77,7 @@ class ARDMediathekBridge extends BridgeAbstract } } - $url = self::APIENDPOINT . $showID . '/?pageSize=' . self::PAGESIZE; + $url = self::APIENDPOINT . $showID . '?pageSize=' . self::PAGESIZE; $rawJSON = getContents($url); $processedJSON = json_decode($rawJSON); @@ -93,6 +98,17 @@ class ARDMediathekBridge extends BridgeAbstract $this->items[] = $item; } + $this->title = $processedJSON->title; + date_default_timezone_set($oldTz); } + + /** {@inheritdoc} */ + public function getName() + { + if (!empty($this->title)) { + return $this->title; + } + return parent::getName(); + } } From aa3989873cbcd20919725e847e4d299ee5724740 Mon Sep 17 00:00:00 2001 From: Pavel Korytov <thexcloud@gmail.com> Date: Tue, 30 Jul 2024 23:10:57 +0300 Subject: [PATCH 134/423] [EconomistBridge] Add cookie (#4173) * [EconomistBridge] Add cookie * [EconomistBridge] Fix lint --- bridges/EconomistBridge.php | 29 +++++++++++++++++++++++++++- docs/10_Bridge_Specific/Economist.md | 5 ++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/bridges/EconomistBridge.php b/bridges/EconomistBridge.php index 70117cb0..eaa50ba1 100644 --- a/bridges/EconomistBridge.php +++ b/bridges/EconomistBridge.php @@ -8,6 +8,12 @@ class EconomistBridge extends FeedExpander const CACHE_TIMEOUT = 3600; //1hour const DESCRIPTION = 'Returns the latest articles for the selected category'; + const CONFIGURATION = [ + 'cookie' => [ + 'required' => false, + ] + ]; + const PARAMETERS = [ 'global' => [ 'limit' => [ @@ -99,8 +105,20 @@ class EconomistBridge extends FeedExpander protected function parseItem(array $item) { + $headers = []; + if ($this->getOption('cookie')) { + $headers = [ + 'Authority: www.economist.com', + 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-language: en-US,en;q=0.9', + 'Cache-control: max-age=0', + 'Cookie: ' . $this->getOption('cookie'), + 'Upgrade-insecure-requests: 1', + 'User-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36' + ]; + } try { - $dom = getSimpleHTMLDOM($item['uri']); + $dom = getSimpleHTMLDOM($item['uri'], $headers); } catch (Exception $e) { $item['content'] = $e->getMessage(); return $item; @@ -209,6 +227,15 @@ class EconomistBridge extends FeedExpander foreach ($elem->find('a.ds-link-with-arrow-icon') as $a) { $a->parent->removeChild($a); } + // Sections like "Leaders on day X" + foreach ($elem->find('div[data-tracking-id=content-well-chapter-list]') as $div) { + $div->parent->removeChild($div); + } + // "Explore more" section + foreach ($elem->find('h3[id=article-tags]') as $h3) { + $div = $h3->parent; + $div->parent->removeChild($div); + } // The Economist puts infographics into iframes, which doesn't // work in any of my readers. So this replaces iframes with diff --git a/docs/10_Bridge_Specific/Economist.md b/docs/10_Bridge_Specific/Economist.md index 1a792eb8..d11ad43c 100644 --- a/docs/10_Bridge_Specific/Economist.md +++ b/docs/10_Bridge_Specific/Economist.md @@ -1,4 +1,4 @@ -# EconomistWorldInBriefBridge +# EconomistWorldInBriefBridge and EconomistBridge In May 2024, The Economist finally fixed its paywall, and it started requiring authorization. Which means you can't use this bridge unless you have an active subscription. @@ -15,4 +15,7 @@ Once you've done this, add the cookie to your `config.ini.php`: ``` [EconomistWorldInBriefBridge] cookie = "<value>" + +[EconomistBridge] +cookie = "<value>" ``` From 891c8979a3ea7678b61d62fbe7f69250d32d1c78 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 31 Jul 2024 17:30:06 +0200 Subject: [PATCH 135/423] refactor: return proper response object (#4169) --- actions/ConnectivityAction.php | 4 ++-- bridges/BadDragonBridge.php | 3 +-- bridges/BandcampBridge.php | 11 ++++++----- bridges/CubariBridge.php | 4 +++- bridges/DemosBerlinBridge.php | 3 ++- bridges/DerpibooruBridge.php | 10 +++------- bridges/EZTVBridge.php | 4 +++- bridges/ElloBridge.php | 8 +++----- bridges/FDroidBridge.php | 2 +- bridges/FunkBridge.php | 2 +- bridges/GlowficBridge.php | 6 +++--- ...tionalInstituteForStrategicStudiesBridge.php | 2 +- bridges/ItakuBridge.php | 4 ++-- bridges/KilledbyGoogleBridge.php | 3 +-- bridges/LegoIdeasBridge.php | 3 +-- bridges/OpenCVEBridge.php | 13 +++++-------- bridges/PepperBridgeAbstract.php | 7 ++----- bridges/PixivBridge.php | 17 ++++++++--------- bridges/RainbowSixSiegeBridge.php | 2 +- bridges/RedditBridge.php | 2 +- bridges/ReutersBridge.php | 4 +++- bridges/RoadAndTrackBridge.php | 5 ----- bridges/SpotifyBridge.php | 4 ++-- bridges/SummitsOnTheAirBridge.php | 8 ++++++-- bridges/TwitterV2Bridge.php | 2 +- bridges/UnogsBridge.php | 2 +- bridges/VkBridge.php | 6 +++--- lib/contents.php | 12 +++--------- 28 files changed, 69 insertions(+), 84 deletions(-) diff --git a/actions/ConnectivityAction.php b/actions/ConnectivityAction.php index 09d9c6c6..eb9edeb1 100644 --- a/actions/ConnectivityAction.php +++ b/actions/ConnectivityAction.php @@ -54,8 +54,8 @@ class ConnectivityAction implements ActionInterface ]; try { $response = getContents($bridge::URI, [], $curl_opts, true); - $result['http_code'] = $response['code']; - if (in_array($response['code'], [200])) { + $result['http_code'] = $response->getCode(); + if (in_array($result['http_code'], [200])) { $result['successful'] = true; } } catch (\Exception $e) { diff --git a/bridges/BadDragonBridge.php b/bridges/BadDragonBridge.php index d38e3408..2249d6f7 100644 --- a/bridges/BadDragonBridge.php +++ b/bridges/BadDragonBridge.php @@ -284,8 +284,7 @@ class BadDragonBridge extends BridgeAbstract case 'Clearance': $toyData = json_decode(getContents($this->inputToURL(true))); - $productList = json_decode(getContents(self::URI - . 'api/inventory-toy/product-list')); + $productList = json_decode(getContents(self::URI . 'api/inventory-toy/product-list')); foreach ($toyData->toys as $toy) { $item = []; diff --git a/bridges/BandcampBridge.php b/bridges/BandcampBridge.php index a9bd2ea1..80bb7fd0 100644 --- a/bridges/BandcampBridge.php +++ b/bridges/BandcampBridge.php @@ -111,12 +111,12 @@ class BandcampBridge extends BridgeAbstract $url = self::URI . 'api/hub/1/dig_deeper'; $data = $this->buildRequestJson(); $header = [ - 'Content-Type: application/json', - 'Content-Length: ' . strlen($data) + 'Content-Type: application/json', + 'Content-Length: ' . strlen($data), ]; $opts = [ - CURLOPT_CUSTOMREQUEST => 'POST', - CURLOPT_POSTFIELDS => $data + CURLOPT_CUSTOMREQUEST => 'POST', + CURLOPT_POSTFIELDS => $data, ]; $content = getContents($url, $header, $opts); @@ -314,7 +314,8 @@ class BandcampBridge extends BridgeAbstract { $url = self::URI . 'api/' . $endpoint . '?' . http_build_query($query_data); // todo: 429 Too Many Requests happens a lot - $data = json_decode(getContents($url)); + $response = getContents($url); + $data = json_decode($response); return $data; } diff --git a/bridges/CubariBridge.php b/bridges/CubariBridge.php index a7b6d69d..72fadf6e 100644 --- a/bridges/CubariBridge.php +++ b/bridges/CubariBridge.php @@ -47,8 +47,10 @@ class CubariBridge extends BridgeAbstract */ public function collectData() { + // TODO: fix trivial SSRF $json = getContents($this->getInput('gist')); - $jsonFile = json_decode($json, true); + + $jsonFile = Json::decode($json); $this->mangaTitle = $jsonFile['title']; diff --git a/bridges/DemosBerlinBridge.php b/bridges/DemosBerlinBridge.php index 05fd2335..cc44a7cf 100644 --- a/bridges/DemosBerlinBridge.php +++ b/bridges/DemosBerlinBridge.php @@ -24,7 +24,8 @@ class DemosBerlinBridge extends BridgeAbstract public function collectData() { - $json = getContents('https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/index/all.json'); + $url = 'https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/index/all.json'; + $json = getContents($url); $jsonFile = json_decode($json, true); $daysInterval = DateInterval::createFromDateString($this->getInput('days') . ' day'); diff --git a/bridges/DerpibooruBridge.php b/bridges/DerpibooruBridge.php index e06e0eff..2d650d57 100644 --- a/bridges/DerpibooruBridge.php +++ b/bridges/DerpibooruBridge.php @@ -78,13 +78,9 @@ class DerpibooruBridge extends BridgeAbstract public function collectData() { - $queryJson = json_decode(getContents( - self::URI - . 'api/v1/json/search/images?filter_id=' - . urlencode($this->getInput('f')) - . '&q=' - . urlencode($this->getInput('q')) - )); + $url = self::URI . 'api/v1/json/search/images?filter_id=' . urlencode($this->getInput('f')) . '&q=' . urlencode($this->getInput('q')); + + $queryJson = json_decode(getContents($url)); foreach ($queryJson->images as $post) { $item = []; diff --git a/bridges/EZTVBridge.php b/bridges/EZTVBridge.php index 25a88124..556bd39e 100644 --- a/bridges/EZTVBridge.php +++ b/bridges/EZTVBridge.php @@ -50,7 +50,9 @@ class EZTVBridge extends BridgeAbstract $eztv_uri = $this->getEztvUri(); $ids = explode(',', trim($this->getInput('ids'))); foreach ($ids as $id) { - $data = json_decode(getContents(sprintf('%s/api/get-torrents?imdb_id=%s', $eztv_uri, $id))); + $url = sprintf('%s/api/get-torrents?imdb_id=%s', $eztv_uri, $id); + $json = getContents($url); + $data = json_decode($json); if (!isset($data->torrents)) { // No results continue; diff --git a/bridges/ElloBridge.php b/bridges/ElloBridge.php index 42c88a06..a9e69cfe 100644 --- a/bridges/ElloBridge.php +++ b/bridges/ElloBridge.php @@ -34,11 +34,9 @@ class ElloBridge extends BridgeAbstract ]; if (!empty($this->getInput('u'))) { - $postData = getContents(self::URI . 'api/v2/users/~' . urlencode($this->getInput('u')) . '/posts', $header) or - returnServerError('Unable to query Ello API.'); + $postData = getContents(self::URI . 'api/v2/users/~' . urlencode($this->getInput('u')) . '/posts', $header); } else { - $postData = getContents(self::URI . 'api/v2/posts?terms=' . urlencode($this->getInput('s')), $header) or - returnServerError('Unable to query Ello API.'); + $postData = getContents(self::URI . 'api/v2/posts?terms=' . urlencode($this->getInput('s')), $header); } $postData = json_decode($postData); @@ -117,7 +115,7 @@ class ElloBridge extends BridgeAbstract $apiKey = $this->cache->get($cacheKey); if (!$apiKey) { - $keyInfo = getContents(self::URI . 'api/webapp-token') or returnServerError('Unable to get token.'); + $keyInfo = getContents(self::URI . 'api/webapp-token'); $apiKey = json_decode($keyInfo)->token->access_token; $ttl = 60 * 60 * 20; $this->cache->set($cacheKey, $apiKey, $ttl); diff --git a/bridges/FDroidBridge.php b/bridges/FDroidBridge.php index 8d3b7808..fdf0262f 100644 --- a/bridges/FDroidBridge.php +++ b/bridges/FDroidBridge.php @@ -31,7 +31,7 @@ class FDroidBridge extends BridgeAbstract CURLOPT_NOBODY => true, ]; $reponse = getContents($url, [], $curlOptions, true); - $lastModified = $reponse['headers']['last-modified'][0] ?? null; + $lastModified = $reponse->getHeader('last-modified'); $timestamp = strtotime($lastModified ?? 'today'); return $timestamp; } diff --git a/bridges/FunkBridge.php b/bridges/FunkBridge.php index df499035..e4935ffb 100644 --- a/bridges/FunkBridge.php +++ b/bridges/FunkBridge.php @@ -32,7 +32,7 @@ class FunkBridge extends BridgeAbstract $url .= '?size=' . $this->getInput('max'); } - $jsonString = getContents($url) or returnServerError('No contents received!'); + $jsonString = getContents($url); $json = json_decode($jsonString, true); foreach ($json['list'] as $element) { diff --git a/bridges/GlowficBridge.php b/bridges/GlowficBridge.php index b51ead8d..0e4b8d93 100644 --- a/bridges/GlowficBridge.php +++ b/bridges/GlowficBridge.php @@ -41,8 +41,7 @@ class GlowficBridge extends BridgeAbstract $first_page = 1; } for ($page_offset = $first_page; $page_offset <= $metadata['Last-Page']; $page_offset++) { - $jsonContents = getContents($url . '/replies?page=' . $page_offset) or - returnClientError('Could not retrieve replies for page ' . $page_offset . '.'); + $jsonContents = getContents($url . '/replies?page=' . $page_offset); $replies = json_decode($jsonContents); foreach ($replies as $reply) { $item = []; @@ -75,8 +74,9 @@ class GlowficBridge extends BridgeAbstract private function getPost() { $url = $this->getAPIURI(); - $jsonPost = getContents($url) or returnClientError('Could not retrieve post metadata.'); + $jsonPost = getContents($url); $post = json_decode($jsonPost); + return $post; } diff --git a/bridges/InternationalInstituteForStrategicStudiesBridge.php b/bridges/InternationalInstituteForStrategicStudiesBridge.php index b5b589ab..9b82dbd5 100644 --- a/bridges/InternationalInstituteForStrategicStudiesBridge.php +++ b/bridges/InternationalInstituteForStrategicStudiesBridge.php @@ -30,7 +30,7 @@ class InternationalInstituteForStrategicStudiesBridge extends BridgeAbstract ]; $headers = [ 'Accept: application/json, text/plain, */*', - 'Content-Type: application/json;charset=UTF-8' + 'Content-Type: application/json;charset=UTF-8', ]; $json = getContents($url, $headers, $opts); $data = json_decode($json); diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index 4f414574..506805f7 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -669,11 +669,11 @@ class ItakuBridge extends BridgeAbstract if ($cache) { $data = $this->loadCacheValue($url); if (is_null($data)) { - $data = getContents($url, $httpHeaders, $curlOptions) or returnServerError("Could not load $url"); + $data = getContents($url, $httpHeaders, $curlOptions); $this->saveCacheValue($url, $data); } } else { - $data = getContents($url, $httpHeaders, $curlOptions) or returnServerError("Could not load $url"); + $data = getContents($url, $httpHeaders, $curlOptions); } return json_decode($data, true); } else { //get simpleHTMLDOM object diff --git a/bridges/KilledbyGoogleBridge.php b/bridges/KilledbyGoogleBridge.php index 54c5b59f..7b8f7f6e 100644 --- a/bridges/KilledbyGoogleBridge.php +++ b/bridges/KilledbyGoogleBridge.php @@ -12,8 +12,7 @@ class KilledbyGoogleBridge extends BridgeAbstract public function collectData() { - $json = getContents(self::URI . '/graveyard.json') - or returnServerError('Could not request: ' . self::URI . '/graveyard.json'); + $json = getContents(self::URI . '/graveyard.json'); $this->handleJson($json); $this->orderItems(); diff --git a/bridges/LegoIdeasBridge.php b/bridges/LegoIdeasBridge.php index c4361f1f..e983e56d 100644 --- a/bridges/LegoIdeasBridge.php +++ b/bridges/LegoIdeasBridge.php @@ -52,8 +52,7 @@ Once a project reaches 10,000 supporters, it gets reviewed by the lego experts.' CURLOPT_POST => 1, CURLOPT_POSTFIELDS => $this->getHttpPostData() ]; - $responseData = getContents($this->getHttpPostURI(), $header, $opts) or - returnServerError('Unable to query Lego Ideas API.'); + $responseData = getContents($this->getHttpPostURI(), $header, $opts); foreach (json_decode($responseData)->results as $project) { preg_match('/datetime=\"(\S+)\"/', $project->entity->published_at, $date_matches); diff --git a/bridges/OpenCVEBridge.php b/bridges/OpenCVEBridge.php index 594bb9ec..b5fc852b 100644 --- a/bridges/OpenCVEBridge.php +++ b/bridges/OpenCVEBridge.php @@ -147,10 +147,9 @@ class OpenCVEBridge extends BridgeAbstract for ($i = 1; $i <= $this->getInput('pages'); $i++) { $queryPaginated = array_merge($query, ['page' => $i]); $url = $instance . '/api/cve?' . http_build_query($queryPaginated); - $response = getContents( - $url, - [$authHeader] - ); + + $response = getContents($url, [$authHeader]); + $titlePrefix = ''; if (count($queries) > 1) { $titlePrefix = '[' . $queryName . '] '; @@ -205,10 +204,8 @@ class OpenCVEBridge extends BridgeAbstract private function fetchContents($cveItem, $titlePrefix, $instance, $authHeader) { $url = $instance . '/api/cve/' . $cveItem->id; - $response = getContents( - $url, - [$authHeader] - ); + + $response = getContents($url, [$authHeader]); $datum = json_decode($response); $title = $this->getTitleFromDatum($datum, $titlePrefix); diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 6e41cf20..4e9ab0b5 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -191,15 +191,12 @@ HEREDOC; } } - /** - * Extract the cookies obtained from the URL - * @return array the array containing the cookies set by the URL - */ private function getCookiesHeaderValue($url) { $response = getContents($url, [], [], true); - $setCookieHeaders = $response['headers']['set-cookie'] ?? []; + $setCookieHeaders = $response->getHeader('set-cookie', true); $cookies = array_map(fn($c): string => explode(';', $c)[0], $setCookieHeaders); + return implode('; ', $cookies); } diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php index 604b5d4b..820b3a7c 100644 --- a/bridges/PixivBridge.php +++ b/bridges/PixivBridge.php @@ -332,21 +332,20 @@ class PixivBridge extends BridgeAbstract } if ($cache) { - $data = $this->loadCacheValue($url); - if (!$data) { - $data = getContents($url, $httpHeaders, $curlOptions, true); - $this->saveCacheValue($url, $data); + $response = $this->loadCacheValue($url); + if (!$response || is_array($response)) { + $response = getContents($url, $httpHeaders, $curlOptions, true); + $this->saveCacheValue($url, $response); } } else { - $data = getContents($url, $httpHeaders, $curlOptions, true); + $response = getContents($url, $httpHeaders, $curlOptions, true); } - $this->checkCookie($data['headers']); + $this->checkCookie($response->getHeaders()); if ($getJSON) { - return json_decode($data['content'], true); - } else { - return $data['content']; + return json_decode($response->getBody(), true); } + return $response->getBody(); } } diff --git a/bridges/RainbowSixSiegeBridge.php b/bridges/RainbowSixSiegeBridge.php index 77495a3c..d725e3e9 100644 --- a/bridges/RainbowSixSiegeBridge.php +++ b/bridges/RainbowSixSiegeBridge.php @@ -22,7 +22,7 @@ class RainbowSixSiegeBridge extends BridgeAbstract $dlUrl = $dlUrl . '&limit=6&mediaFilter=all&skip=0&startIndex=0&tags=BR-rainbow-six%20GA-siege'; $dlUrl = $dlUrl . '&locale=en-us&fallbackLocale=en-us&environment=master'; $jsonString = getContents($dlUrl, [ - 'Authorization: ' . self::NIMBUS_API_KEY + 'Authorization: ' . self::NIMBUS_API_KEY, ]); $json = json_decode($jsonString, true); diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 434ae74a..ef74fdcd 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -149,7 +149,7 @@ class RedditBridge extends BridgeAbstract $response = getContents($url, ['User-Agent: ' . $useragent], [], true); - $json = $response['content']; + $json = $response->getBody(); $parsedJson = Json::decode($json, false); diff --git a/bridges/ReutersBridge.php b/bridges/ReutersBridge.php index fdf4e2a9..07b3061c 100644 --- a/bridges/ReutersBridge.php +++ b/bridges/ReutersBridge.php @@ -417,9 +417,11 @@ class ReutersBridge extends BridgeAbstract $get_embed_url = 'https://publish.twitter.com/oembed?url=' . urlencode($tweet_url) . '&partner=&hide_thread=false'; + $oembed_json = json_decode(getContents($get_embed_url), true); $embed .= $oembed_json['html']; - } catch (Exception $e) { // In case not found any tweet. + } catch (\Exception $e) { + // In case not found any tweet. $embed .= ''; } break; diff --git a/bridges/RoadAndTrackBridge.php b/bridges/RoadAndTrackBridge.php index c236036c..eb2dcc53 100644 --- a/bridges/RoadAndTrackBridge.php +++ b/bridges/RoadAndTrackBridge.php @@ -68,9 +68,4 @@ class RoadAndTrackBridge extends BridgeAbstract $item['content'] = $content; return $item; } - - private function getArticleContent($article) - { - return getContents($article->contentUrl); - } } diff --git a/bridges/SpotifyBridge.php b/bridges/SpotifyBridge.php index 25948011..e03d43a1 100644 --- a/bridges/SpotifyBridge.php +++ b/bridges/SpotifyBridge.php @@ -286,9 +286,9 @@ class SpotifyBridge extends BridgeAbstract } else { $basicAuth = base64_encode(sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret'))); $json = getContents('https://accounts.spotify.com/api/token', [ - "Authorization: Basic $basicAuth" + "Authorization: Basic $basicAuth", ], [ - CURLOPT_POSTFIELDS => 'grant_type=client_credentials' + CURLOPT_POSTFIELDS => 'grant_type=client_credentials', ]); $data = Json::decode($json); $this->token = $data['access_token']; diff --git a/bridges/SummitsOnTheAirBridge.php b/bridges/SummitsOnTheAirBridge.php index 53bba7ab..17431214 100644 --- a/bridges/SummitsOnTheAirBridge.php +++ b/bridges/SummitsOnTheAirBridge.php @@ -20,8 +20,12 @@ class SummitsOnTheAirBridge extends BridgeAbstract public function collectData() { - $header = ['Content-type:application/json']; - $opts = [CURLOPT_HTTPGET => 1]; + $header = [ + 'Content-type:application/json', + ]; + $opts = [ + CURLOPT_HTTPGET => 1, + ]; $json = getContents($this->getURI() . $this->getInput('c'), $header, $opts); $spots = json_decode($json, true); diff --git a/bridges/TwitterV2Bridge.php b/bridges/TwitterV2Bridge.php index 83bfae29..07af8301 100644 --- a/bridges/TwitterV2Bridge.php +++ b/bridges/TwitterV2Bridge.php @@ -598,7 +598,7 @@ EXTERNAL; private function makeApiCall($api, $authHeaders, $params) { $uri = self::API_URI . $api . '?' . http_build_query($params); - $result = getContents($uri, $authHeaders, [], false); + $result = getContents($uri, $authHeaders); $data = json_decode($result); return $data; } diff --git a/bridges/UnogsBridge.php b/bridges/UnogsBridge.php index 486bac3d..7aff10c6 100644 --- a/bridges/UnogsBridge.php +++ b/bridges/UnogsBridge.php @@ -92,7 +92,7 @@ class UnogsBridge extends BridgeAbstract { $header = [ 'Referer: https://unogs.com/', - 'referrer: http://unogs.com' + 'referrer: http://unogs.com', ]; $raw = getContents($url, $header); diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 980b4154..22957f26 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -511,11 +511,11 @@ class VkBridge extends BridgeAbstract while ($redirects < 2) { $response = getContents($uri, $httpHeaders, [CURLOPT_FOLLOWLOCATION => false], true); - if (in_array($response['code'], [200, 304])) { - return $response['content']; + if (in_array($response->getCode(), [200, 304])) { + return $response->getBody(); } - $headers = $response['headers']; + $headers = $response->getHeaders(); $uri = urljoin(self::URI, $headers['location'][0]); if (str_contains($uri, '/429.html')) { diff --git a/lib/contents.php b/lib/contents.php index ba6dd531..893a3512 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -5,8 +5,8 @@ * * @param array $httpHeaders E.g. ['Content-type: text/plain'] * @param array $curlOptions Associative array e.g. [CURLOPT_MAXREDIRS => 3] - * @param bool $returnFull Whether to return an array: ['code' => int, 'headers' => array, 'content' => string] - * @return string|array + * @param bool $returnFull Whether to return Response object + * @return string|Response */ function getContents( string $url, @@ -113,13 +113,7 @@ function getContents( throw $e; } if ($returnFull === true) { - // todo: return the actual response object - return [ - 'code' => $response->getCode(), - 'headers' => $response->getHeaders(), - // For legacy reasons, use 'content' instead of 'body' - 'content' => $response->getBody(), - ]; + return $response; } return $response->getBody(); } From 1a8d0fb8ab9fe32350c1395bd5a9d1b3166980fa Mon Sep 17 00:00:00 2001 From: Zack Puhl <31320277+NotsoanoNimus@users.noreply.github.com> Date: Wed, 31 Jul 2024 11:51:05 -0400 Subject: [PATCH 136/423] [EBayBridge] fix undefined vars errors (#4175) --- bridges/EBayBridge.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bridges/EBayBridge.php b/bridges/EBayBridge.php index 11b90bf8..463f73d6 100644 --- a/bridges/EBayBridge.php +++ b/bridges/EBayBridge.php @@ -137,6 +137,8 @@ class EBayBridge extends BridgeAbstract . trim($additionalPrice ?? '') . '; ' . trim($discount ?? '') . ')</em>'; + } else { + $discountLine = ''; } // Prepend the time-left info with a comma if the right details were found. @@ -163,6 +165,8 @@ class EBayBridge extends BridgeAbstract // Include the original search link, if specified. if ($this->getInput('includesSearchLink')) { $searchLink = '<p><small><a target="_blank" href="' . e($this->getURI()) . '">View Search</a></small></p>'; + } else { + $searchLink = ''; } // Build the final item's content to display and add the item onto the list. From 9982bfce1f0ae2f9aa5a755787912089da145da8 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 31 Jul 2024 17:51:44 +0200 Subject: [PATCH 137/423] fix: convert php errors to exceptions when in debug mode (#4176) --- index.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/index.php b/index.php index 7144ae27..f23d8441 100644 --- a/index.php +++ b/index.php @@ -25,8 +25,10 @@ set_error_handler(function ($code, $message, $file, $line) { // Deprecation messages and other masked errors are typically ignored here return false; } - // In the future, uncomment this: - //throw new \ErrorException($message, 0, $code, $file, $line); + if (Debug::isEnabled()) { + // This might be annoying, but it's for the greater good + throw new \ErrorException($message, 0, $code, $file, $line); + } $text = sprintf( '%s at %s line %s', sanitize_root($message), From e55e9b8facacee38671cf7d9c3dc2ca3d3be28b5 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 31 Jul 2024 17:53:10 +0200 Subject: [PATCH 138/423] feat: enable all bridges by default (#4177) --- config.default.ini.php | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/config.default.ini.php b/config.default.ini.php index 8f7de832..c6b0779d 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -8,23 +8,24 @@ ; Only these bridges are available for feed production ; How to enable all bridges: enabled_bridges[] = * -enabled_bridges[] = CssSelectorBridge -enabled_bridges[] = FeedMerge -enabled_bridges[] = FeedReducerBridge -enabled_bridges[] = Filter -enabled_bridges[] = GettrBridge -enabled_bridges[] = MastodonBridge -enabled_bridges[] = Reddit -enabled_bridges[] = RumbleBridge -enabled_bridges[] = SoundcloudBridge -enabled_bridges[] = Telegram -enabled_bridges[] = ThePirateBay -enabled_bridges[] = TikTokBridge -enabled_bridges[] = Twitch -enabled_bridges[] = Vk -enabled_bridges[] = XPathBridge -enabled_bridges[] = Youtube -enabled_bridges[] = YouTubeCommunityTabBridge +;enabled_bridges[] = CssSelectorBridge +;enabled_bridges[] = FeedMerge +;enabled_bridges[] = FeedReducerBridge +;enabled_bridges[] = Filter +;enabled_bridges[] = GettrBridge +;enabled_bridges[] = MastodonBridge +;enabled_bridges[] = Reddit +;enabled_bridges[] = RumbleBridge +;enabled_bridges[] = SoundcloudBridge +;enabled_bridges[] = Telegram +;enabled_bridges[] = ThePirateBay +;enabled_bridges[] = TikTokBridge +;enabled_bridges[] = Twitch +;enabled_bridges[] = Vk +;enabled_bridges[] = XPathBridge +;enabled_bridges[] = Youtube +;enabled_bridges[] = YouTubeCommunityTabBridge +enabled_bridges[] = * ; Defines the timezone used by RSS-Bridge ; Find a list of supported timezones at From b8a9f34527c15f99cd8566b72d9a9a5c2b9f95a5 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 31 Jul 2024 19:04:07 +0200 Subject: [PATCH 139/423] fix(FeedParser): scrape out content from rss content:encoded (#4178) * fix(FeedParser): parse content module from rss2 * refactor --- README.md | 9 +++++++++ actions/DisplayAction.php | 19 ++++++++++--------- lib/FeedItem.php | 9 +++++++-- lib/FeedParser.php | 3 ++- tests/FeedParserTest.php | 19 +++++++++++-------- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6124a4ea..982cd63b 100644 --- a/README.md +++ b/README.md @@ -418,7 +418,16 @@ See `formats/PlaintextFormat.php` for an example. These commands require that you have installed the dev dependencies in `composer.json`. +Run all tests: + ./vendor/bin/phpunit + +Run a single test class: + + ./vendor/bin/phpunit --filter UrlTest + +Run linter: + ./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./ https://github.com/squizlabs/PHP_CodeSniffer/wiki diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 93813004..54782edc 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -112,15 +112,6 @@ class DisplayAction implements ActionInterface $input = array_diff_key($requestArray, array_fill_keys($remove, '')); $bridge->setInput($input); $bridge->collectData(); - $items = $bridge->getItems(); - if (isset($items[0]) && is_array($items[0])) { - $feedItems = []; - foreach ($items as $item) { - $feedItems[] = FeedItem::fromArray($item); - } - $items = $feedItems; - } - $feed = $bridge->getFeed(); } catch (\Exception $e) { // Probably an exception inside a bridge if ($e instanceof HttpException) { @@ -154,6 +145,16 @@ class DisplayAction implements ActionInterface } } + $items = $bridge->getItems(); + if (isset($items[0]) && is_array($items[0])) { + $feedItems = []; + foreach ($items as $item) { + $feedItems[] = FeedItem::fromArray($item); + } + $items = $feedItems; + } + $feed = $bridge->getFeed(); + $formatFactory = new FormatFactory(); $format = $formatFactory->create($format); diff --git a/lib/FeedItem.php b/lib/FeedItem.php index fc4549a7..dfb954a0 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -186,21 +186,26 @@ class FeedItem } /** - * @param string|object $content The item content as text or simple_html_dom object. + * @param string|array|\simple_html_dom|\simple_html_dom_node $content The item content */ public function setContent($content) { $this->content = null; + if ( $content instanceof simple_html_dom || $content instanceof simple_html_dom_node ) { $content = (string) $content; + } elseif (is_array($content)) { + // Assuming this is the rss2.0 content module + $content = $content['encoded'] ?? ''; } + if (is_string($content)) { $this->content = $content; } else { - Debug::log(sprintf('Feed content must be a string but got %s', gettype($content))); + Debug::log(sprintf('Unable to convert feed content to string: %s', gettype($content))); } } diff --git a/lib/FeedParser.php b/lib/FeedParser.php index b774cc14..3a2f15d4 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -167,8 +167,9 @@ final class FeedParser if (isset($namespaces['media'])) { $media = $feedItem->children($namespaces['media']); } + foreach ($namespaces as $namespaceName => $namespaceUrl) { - if (in_array($namespaceName, ['', 'content', 'media'])) { + if (in_array($namespaceName, ['', 'media'])) { continue; } $item[$namespaceName] = $this->parseModule($feedItem, $namespaceName, $namespaceUrl); diff --git a/tests/FeedParserTest.php b/tests/FeedParserTest.php index 05d5ef69..45dc1234 100644 --- a/tests/FeedParserTest.php +++ b/tests/FeedParserTest.php @@ -8,6 +8,13 @@ use PHPUnit\Framework\TestCase; class FeedParserTest extends TestCase { + private \FeedParser $sut; + + public function setUp(): void + { + $this->sut = new \FeedParser(); + } + public function testRss1() { $xml = <<<XML @@ -37,8 +44,7 @@ class FeedParserTest extends TestCase </rdf:RDF> XML; - $sut = new \FeedParser(); - $feed = $sut->parseFeed($xml); + $feed = $this->sut->parseFeed($xml); $this->assertSame('hello feed', $feed['title']); $this->assertSame('http://meerkat.oreillynet.com', $feed['uri']); @@ -74,8 +80,7 @@ class FeedParserTest extends TestCase </rss> XML; - $sut = new \FeedParser(); - $feed = $sut->parseFeed($xml); + $feed = $this->sut->parseFeed($xml); $this->assertSame('hello feed', $feed['title']); $this->assertSame('https://example.com/', $feed['uri']); @@ -111,8 +116,7 @@ class FeedParserTest extends TestCase </feed> XML; - $sut = new \FeedParser(); - $feed = $sut->parseFeed($xml); + $feed = $this->sut->parseFeed($xml); $this->assertSame('hello feed', $feed['title']); $this->assertSame('https://example.com/1', $feed['uri']); @@ -151,8 +155,7 @@ class FeedParserTest extends TestCase </rss> XML; - $sut = new \FeedParser(); - $feed = $sut->parseFeed($xml); + $feed = $this->sut->parseFeed($xml); $expected = [ 'title' => '', 'uri' => '', From 8a1f2604aa9358c6f70e078e704e5c674683eaab Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 31 Jul 2024 19:25:51 +0200 Subject: [PATCH 140/423] fix: bug in prior refactor (#4179) * fix: bug in prior refactor * fix deprecation notice --- actions/DisplayAction.php | 19 +++++++++---------- bridges/GelbooruBridge.php | 18 ++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 54782edc..93813004 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -112,6 +112,15 @@ class DisplayAction implements ActionInterface $input = array_diff_key($requestArray, array_fill_keys($remove, '')); $bridge->setInput($input); $bridge->collectData(); + $items = $bridge->getItems(); + if (isset($items[0]) && is_array($items[0])) { + $feedItems = []; + foreach ($items as $item) { + $feedItems[] = FeedItem::fromArray($item); + } + $items = $feedItems; + } + $feed = $bridge->getFeed(); } catch (\Exception $e) { // Probably an exception inside a bridge if ($e instanceof HttpException) { @@ -145,16 +154,6 @@ class DisplayAction implements ActionInterface } } - $items = $bridge->getItems(); - if (isset($items[0]) && is_array($items[0])) { - $feedItems = []; - foreach ($items as $item) { - $feedItems[] = FeedItem::fromArray($item); - } - $items = $feedItems; - } - $feed = $bridge->getFeed(); - $formatFactory = new FormatFactory(); $format = $formatFactory->create($format); diff --git a/bridges/GelbooruBridge.php b/bridges/GelbooruBridge.php index 5fc6b33c..96d16bf9 100644 --- a/bridges/GelbooruBridge.php +++ b/bridges/GelbooruBridge.php @@ -33,7 +33,7 @@ class GelbooruBridge extends BridgeAbstract return $this->getURI() . 'index.php?&page=dapi&s=post&q=index&json=1&pid=' . $this->getInput('p') . '&limit=' . $this->getInput('l') - . '&tags=' . urlencode($this->getInput('t')); + . '&tags=' . urlencode($this->getInput('t') ?? ''); } /* @@ -76,18 +76,16 @@ class GelbooruBridge extends BridgeAbstract public function collectData() { - $content = getContents($this->getFullURI()); - // $content is empty string + $url = $this->getFullURI(); + $content = getContents($url); - // Most other Gelbooru-based boorus put their content in the root of - // the JSON. This check is here for Bridges that inherit from this one - $posts = json_decode($content); - if (isset($posts->post)) { - $posts = $posts->post; + if ($content === '') { + return; } - if (is_null($posts)) { - returnServerError('No posts found.'); + $posts = Json::decode($content, false); + if (isset($posts->post)) { + $posts = $posts->post; } foreach ($posts as $post) { From 615c5335878be434a9051278126f7286e1549e14 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 31 Jul 2024 20:34:33 +0200 Subject: [PATCH 141/423] fix(FeedParser): dont emit content module (#4180) --- lib/FeedItem.php | 3 --- lib/FeedParser.php | 7 ++++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/FeedItem.php b/lib/FeedItem.php index dfb954a0..09aab129 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -197,9 +197,6 @@ class FeedItem || $content instanceof simple_html_dom_node ) { $content = (string) $content; - } elseif (is_array($content)) { - // Assuming this is the rss2.0 content module - $content = $content['encoded'] ?? ''; } if (is_string($content)) { diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 3a2f15d4..0ad90965 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -168,8 +168,13 @@ final class FeedParser $media = $feedItem->children($namespaces['media']); } + if (isset($namespaces['content'])) { + $content = $feedItem->children($namespaces['content']); + $item['content'] = (string) $content; + } + foreach ($namespaces as $namespaceName => $namespaceUrl) { - if (in_array($namespaceName, ['', 'media'])) { + if (in_array($namespaceName, ['', 'content', 'media'])) { continue; } $item[$namespaceName] = $this->parseModule($feedItem, $namespaceName, $namespaceUrl); From b505667168225a74c30e7b58ab6a54bc00c3b89e Mon Sep 17 00:00:00 2001 From: Pavel Korytov <thexcloud@gmail.com> Date: Wed, 31 Jul 2024 22:57:20 +0300 Subject: [PATCH 142/423] [SubstackBridge] Add Substack bridge (#4174) * [SubstackBridge] Add Substack * [SubstackBridge] Add docs * [SubstackBridge] Fix lint * [SubstackBridge] Update description * [SubstackBridge] Update description (x2) --- bridges/SubstackBridge.php | 50 +++++++++++++++++++++++++++++ docs/10_Bridge_Specific/Substack.md | 18 +++++++++++ lib/FeedExpander.php | 4 +-- 3 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 bridges/SubstackBridge.php create mode 100644 docs/10_Bridge_Specific/Substack.md diff --git a/bridges/SubstackBridge.php b/bridges/SubstackBridge.php new file mode 100644 index 00000000..13eea02e --- /dev/null +++ b/bridges/SubstackBridge.php @@ -0,0 +1,50 @@ +<?php + +class SubstackBridge extends FeedExpander +{ + const MAINTAINER = 'sqrtminusone'; + const NAME = 'Substack Bridge'; + const URI = 'https://substack.com/'; + const CACHE_TIMEOUT = 3600; //1hour + const DESCRIPTION = 'Access Substack. Add full content for paywalled posts if you have a session cookie with an active subscription.'; + + const CONFIGURATION = [ + 'sid' => [ + 'required' => false, + ] + ]; + + const PARAMETERS = [ + '' => [ + 'url' => [ + 'name' => 'Substack RSS URL', + 'required' => true, + 'type' => 'text', + 'defaultValue' => 'https://newsletter.pragmaticengineer.com/feed', + 'title' => 'Usually https://<blog-url>/feed' + ] + ] + ]; + + public function collectData() + { + $headers = []; + if ($this->getOption('sid')) { + $url_parsed = parse_url($this->getInput('url')); + $authority = $url_parsed['host']; + $cookies = [ + 'ab_experiment_sampled=%22false%22', + 'substack.sid=' . $this->getOption('sid'), + 'substack.lli=1', + 'intro_popup_last_hidden_at=' . (new DateTime())->format('Y-m-d\TH:i:s.v\Z') + ]; + $headers = [ + 'Authority: ' . $authority, + 'Cache-Control: max-age=0', + 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36', + 'Cookie: ' . implode('; ', $cookies) + ]; + } + $this->collectExpandableDatas($this->getInput('url'), -1, $headers); + } +} diff --git a/docs/10_Bridge_Specific/Substack.md b/docs/10_Bridge_Specific/Substack.md new file mode 100644 index 00000000..7595bbef --- /dev/null +++ b/docs/10_Bridge_Specific/Substack.md @@ -0,0 +1,18 @@ +# SubstackBridge + +[Substack](https://substack.com) provides RSS feeds at `/feed` path, e.g., https://newsletter.pragmaticengineer.com/feed/. However, these feeds have two problems, addressed by this bridge: +- They use RSS 2.0 with the draft [content extension](https://web.resource.org/rss/1.0/modules/content/), which isn't supported by some readers; +- They don't have the full content for paywalled posts. + +Retrieving the full content is only possible _with an active subscription to the blog_. If you have one, Substack will return the full feed if it's fetched with the right set of cookies. Figuring out whether it's the intended behaviour is left as an exercise for the reader. + +To obtain the session cookie, authorize at https://substack.com/, open DevTools, go to Application -> Cookies -> https://substack.com, copy the value of `substack.sid` and paste it to the RSS bridge config: + +``` +[SubstackBridge] +sid = "<your-sid>" +``` + +Authorization sometimes requires CAPTCHA, hence this operation is manual. The cookie lives for three months. + +After you've done this, the bridge should return full feeds for your subscriptions. diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index fe809bc2..ef001af1 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -7,7 +7,7 @@ abstract class FeedExpander extends BridgeAbstract { private array $feed; - public function collectExpandableDatas(string $url, $maxItems = -1) + public function collectExpandableDatas(string $url, $maxItems = -1, $headers = []) { if (!$url) { throw new \Exception('There is no $url for this RSS expander'); @@ -17,7 +17,7 @@ abstract class FeedExpander extends BridgeAbstract $maxItems = 999; } $accept = [MrssFormat::MIME_TYPE, AtomFormat::MIME_TYPE, '*/*']; - $httpHeaders = ['Accept: ' . implode(', ', $accept)]; + $httpHeaders = array_merge(['Accept: ' . implode(', ', $accept)], $headers); $xmlString = getContents($url, $httpHeaders); if ($xmlString === '') { throw new \Exception(sprintf('Unable to parse xml from `%s` because we got the empty string', $url), 10); From 8ae716e75cefba6fc2676e70b3dfb975593d83dd Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 31 Jul 2024 21:57:33 +0200 Subject: [PATCH 143/423] fix: improve github issue template (#4181) --- actions/DisplayAction.php | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 93813004..d39d1812 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -213,22 +213,33 @@ class DisplayAction implements ActionInterface return $report['count']; } - private static function createGithubIssueUrl($bridge, $e, string $message): string + private static function createGithubIssueUrl(BridgeAbstract $bridge, \Exception $e, string $message): string { - return sprintf('https://github.com/RSS-Bridge/rss-bridge/issues/new?%s', http_build_query([ - 'title' => sprintf('%s failed with error %s', $bridge->getName(), $e->getCode()), + $maintainer = $bridge->getMaintainer(); + if (str_contains($maintainer, ',')) { + $maintainers = explode(',', $maintainer); + } else { + $maintainers = [$maintainer]; + } + $maintainers = array_map('trim', $maintainers); + + $query = [ + 'title' => $bridge->getName() . ' failed with: ' . $e->getMessage(), 'body' => sprintf( - "```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```", + "```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```\nMaintainer: @%s", $message, implode("\n", trace_to_call_points(trace_from_exception($e))), $_SERVER['QUERY_STRING'] ?? '', Configuration::getVersion(), PHP_OS_FAMILY, - phpversion() ?: 'Unknown' + phpversion() ?: 'Unknown', + implode(', @', $maintainers), ), 'labels' => 'Bridge-Broken', - 'assignee' => $bridge->getMaintainer(), - ])); + 'assignee' => $maintainer[0], + ]; + + return 'https://github.com/RSS-Bridge/rss-bridge/issues/new?' . http_build_query($query); } private static function createGithubSearchUrl($bridge): string From d050fe9a9b6476efde29891eb59e87505e3200bf Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:36:26 +0200 Subject: [PATCH 144/423] [AnisearchBridge] fixed typo (#4182) don't know why it was there --- bridges/AnisearchBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/AnisearchBridge.php b/bridges/AnisearchBridge.php index 869b3049..c805cfcb 100644 --- a/bridges/AnisearchBridge.php +++ b/bridges/AnisearchBridge.php @@ -72,7 +72,7 @@ class AnisearchBridge extends BridgeAbstract $ytlink = <<<EOT <br /><iframe width="560" height="315" src="$trailer" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" - referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>'; + referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe> EOT; } } From 0051e0fcddfe008b34e990dce6e7e3c61bae3eea Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 1 Aug 2024 23:36:14 +0200 Subject: [PATCH 145/423] docs: improve docker docs (#4183) * docs: improve docker docs * fix: cleanup and remove duplicate docker instructions --- README.md | 54 ++++++++++++------- docker-compose.yml | 9 ++++ docs/01_General/03_Requirements.md | 10 ---- docs/03_For_Hosts/01_Installation.md | 4 +- docs/03_For_Hosts/02_Updating.md | 4 -- docs/03_For_Hosts/03_Docker_Installation.md | 49 ----------------- docs/03_For_Hosts/index.md | 13 +++-- .../07_Development_Environment_Setup.md | 36 +------------ 8 files changed, 53 insertions(+), 126 deletions(-) create mode 100644 docker-compose.yml delete mode 100644 docs/03_For_Hosts/03_Docker_Installation.md diff --git a/README.md b/README.md index 982cd63b..b3b12f0e 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,10 @@ Requires minimum PHP 7.4. ### How to install on traditional shared web hosting -RSS-Bridge can basically be unzipped in a web folder. Should be working instantly. +RSS-Bridge can basically be unzipped into a web folder. Should be working instantly. -Latest zip as of Sep 2023: https://github.com/RSS-Bridge/rss-bridge/archive/refs/tags/2023-09-24.zip +Latest zip: +https://github.com/RSS-Bridge/rss-bridge/archive/refs/heads/master.zip (2MB) ### How to install on Debian 12 (nginx + php-fpm) @@ -66,7 +67,7 @@ timedatectl set-timezone Europe/Oslo apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl php-intl -# Create a new user account +# Create a user account useradd --shell /bin/bash --create-home rss-bridge cd /var/www @@ -101,7 +102,10 @@ Nginx config: server { listen 80; + + # TODO: change to your own server name server_name example.com; + access_log /var/log/nginx/rss-bridge.access.log; error_log /var/log/nginx/rss-bridge.error.log; log_not_found off; @@ -150,8 +154,11 @@ listen = /run/php/rss-bridge.sock listen.owner = www-data listen.group = www-data +# Create 10 workers standing by to serve requests pm = static pm.max_children = 10 + +# Respawn worker after 500 requests (workaround for memory leaks etc.) pm.max_requests = 500 ``` @@ -179,7 +186,7 @@ Install the latest release. ```shell cd /var/www -composer create-project -v --no-dev rss-bridge/rss-bridge +composer create-project -v --no-dev --no-scripts rss-bridge/rss-bridge ``` ### How to install with Caddy @@ -192,8 +199,16 @@ Install by downloading the docker image from Docker Hub: ```bash # Create container -docker create --name=rss-bridge --publish 3000:80 rssbridge/rss-bridge +docker create --name=rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rssbridge/rss-bridge +``` +You can put custom `config.ini.php` and bridges into `./config`. + +**You must restart container for custom changes to take effect.** + +See `docker-entrypoint.sh` for details. + +```bash # Start container docker start rss-bridge ``` @@ -207,30 +222,29 @@ Browse http://localhost:3000/ docker build -t rss-bridge . # Create container -docker create --name rss-bridge --publish 3000:80 rss-bridge +docker create --name rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rss-bridge +``` +You can put custom `config.ini.php` and bridges into `./config`. + +**You must restart container for custom changes to take effect.** + +See `docker-entrypoint.sh` for details. + +```bash # Start container docker start rss-bridge ``` Browse http://localhost:3000/ -### Install with docker-compose +### Install with docker-compose (using Docker Hub) -Create a `docker-compose.yml` file locally with with the following content: -```yml -version: '2' -services: - rss-bridge: - image: rssbridge/rss-bridge:latest - volumes: - - </local/custom/path>:/config - ports: - - 3000:80 - restart: unless-stopped -``` +You can put custom `config.ini.php` and bridges into `./config`. -Then launch with `docker-compose`: +**You must restart container for custom changes to take effect.** + +See `docker-entrypoint.sh` for details. ```bash docker-compose up diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..9f178049 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,9 @@ +version: '2' +services: + rss-bridge: + image: rssbridge/rss-bridge:latest + volumes: + - ./config:/config + ports: + - 3000:80 + restart: unless-stopped diff --git a/docs/01_General/03_Requirements.md b/docs/01_General/03_Requirements.md index 1ae5aa26..617cfadc 100644 --- a/docs/01_General/03_Requirements.md +++ b/docs/01_General/03_Requirements.md @@ -1,6 +1,4 @@ -**RSS-Bridge** requires either of the following: -## A Web server* with: - PHP 7.4 (or higher) - [`openssl`](https://secure.php.net/manual/en/book.openssl.php) extension @@ -14,11 +12,3 @@ - [`sqlite3`](http://php.net/manual/en/book.sqlite3.php) extension (only when using SQLiteCache) Enable extensions by un-commenting the corresponding line in your PHP configuration (`php.ini`). - - -## A Linux server with: - - - Docker server configured (Any recent version should do) - - 100MB of disk space - -To setup RSS Bridge using Docker, see the [Docker Guide](../03_For_Hosts/03_Docker_Installation.md) on installing RSS Bridge. \ No newline at end of file diff --git a/docs/03_For_Hosts/01_Installation.md b/docs/03_For_Hosts/01_Installation.md index 39df7918..729e6abb 100644 --- a/docs/03_For_Hosts/01_Installation.md +++ b/docs/03_For_Hosts/01_Installation.md @@ -7,6 +7,4 @@ In order to install RSS-Bridge on your own web server* do as follows: For linux hosts: * Grant read-write-access for `www-data` to the `./cache` directory (`chown -R www-data ./cache`) -You have successfully installed RSS-Bridge. - -Instructions for Docker setups are at [Docker Installation](../03_For_Hosts/03_Docker_Installation.md) \ No newline at end of file +You have successfully installed RSS-Bridge. \ No newline at end of file diff --git a/docs/03_For_Hosts/02_Updating.md b/docs/03_For_Hosts/02_Updating.md index 3ec98049..3484c6dc 100644 --- a/docs/03_For_Hosts/02_Updating.md +++ b/docs/03_For_Hosts/02_Updating.md @@ -8,10 +8,6 @@ Updating an existing installation is very simple, depending on your type of inst This will update all core files to the latest version. Your custom configuration and bridges are left untouched. Keep in mind that changes to any core file of RSS-Bridge will be replaced. -## Docker - -Simply get the latest Docker build via `:latest` or specific builds via `:<tag-name>`. - ## Heroku ### If you didn't fork the repo before diff --git a/docs/03_For_Hosts/03_Docker_Installation.md b/docs/03_For_Hosts/03_Docker_Installation.md deleted file mode 100644 index d895e748..00000000 --- a/docs/03_For_Hosts/03_Docker_Installation.md +++ /dev/null @@ -1,49 +0,0 @@ -This guide is for people who want to run RSS Bridge using Docker. If you want to run it a simple PHP Webhost environment, see [Installation](../03_For_Hosts/01_Installation.md) instead. - -## Setup - -### Create the container - -```bash -docker create \ ---name=rss-bridge \ ---volume </local/custom/path>:/config \ ---publish 3000:80 \ -rssbridge/rss-bridge:latest -``` -### Run it -```bash -docker start rss-bridge -``` - -Access it using `http://IP_Address:3000`. If you'd like to run a specific version, you can run it by changing the ':latest' on the image to a tag listed [here](https://hub.docker.com/r/rssbridge/rss-bridge/tags/) - -The server runs on port 80 internally, map any port of your choice (in this example 3000). - -You can run it using a `docker-compose.yml` as well: - -```yml -version: '2' -services: - rss-bridge: - image: rssbridge/rss-bridge:latest - volumes: - - </local/custom/path>:/config - ports: - - 3000:80 - restart: unless-stopped -``` - -# Container access and information - -|Function|Command| -|----|----| -|Shell access (live container)|`docker exec -it rss-bridge /bin/sh`| -|Realtime container logs|`docker logs -f rss-bridge`| - -# Adding custom bridges and configurations -If you want to add a bridge that is not part of [`/bridges`](https://github.com/RSS-Bridge/rss-bridge/tree/master/bridges), you can map a folder to the `/config` folder of the `rss-bridge` container. - -1. Create a folder in the location of your docker-compose.yml or your general docker working area (in this example it will be `/home/docker/rssbridge/config` ). -2. Copy your [custom bridges](../05_Bridge_API/01_How_to_create_a_new_bridge.md) to the `/home/docker/rssbridge/config` folder. Applies also to [config.ini.php](../03_For_Hosts/08_Custom_Configuration.md). -3. Map the folder to `/config` inside the container. To do that, replace the `</local/custom/path>` from the previous examples with `/home/docker/rssbridge/config` \ No newline at end of file diff --git a/docs/03_For_Hosts/index.md b/docs/03_For_Hosts/index.md index 1529cb37..b89f321a 100644 --- a/docs/03_For_Hosts/index.md +++ b/docs/03_For_Hosts/index.md @@ -1,11 +1,14 @@ This section is directed at **hosts** and **server administrators**. -To install RSS-Bridge, please follow the [installation instructions](../03_For_Hosts/01_Installation.md). You must have access to a web server with a working PHP environment! +To install RSS-Bridge, please follow the [installation instructions](../03_For_Hosts/01_Installation.md). +You must have access to a web server with a working PHP environment! -RSS-Bridge comes with a large amount of bridges. Only few bridges are enabled by default. Unlock more bridges by adding them to the [whitelist](../03_For_Hosts/05_Whitelisting.md). +RSS-Bridge comes with a large amount of bridges. -Some bridges could be implemented more efficiently by actually using proprietary APIs, but there are reasons against it: +Some bridges could be implemented more efficiently by actually using proprietary APIs, +but there are reasons against it: -- RSS-Bridge exists in the first place to NOT use APIs. See [the rant](https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md#Rant) +- RSS-Bridge exists in the first place to NOT use APIs. +- See [the rant](https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md#Rant) -- APIs require private keys that could be stored on servers running RSS-Bridge, which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution. \ No newline at end of file +- APIs require private keys that could be stored on servers running RSS-Bridge,which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution. \ No newline at end of file diff --git a/docs/04_For_Developers/07_Development_Environment_Setup.md b/docs/04_For_Developers/07_Development_Environment_Setup.md index 23a4b101..d3a5ee8d 100644 --- a/docs/04_For_Developers/07_Development_Environment_Setup.md +++ b/docs/04_For_Developers/07_Development_Environment_Setup.md @@ -1,39 +1,5 @@ -These are examples of how to setup a local development environment to add bridges, improve the docs, etc. -## Docker - -The following can serve as an example for using docker: - -``` -# create a new directory -mkdir rss-bridge-contribution -cd rss-bridge-contribution - -# clone the project into a subfolder -git clone https://github.com/RSS-Bridge/rss-bridge -``` - -Then add a `docker-compose.yml` file: - -```yml -version: '3' - -services: - rss-bridge: - build: - context: ./rss-bridge - ports: - - 3000:80 - volumes: - - ./config:/config - - ./rss-bridge/bridges:/app/bridges -``` - -You can then access RSS-Bridge at `localhost:3000` and [add your bridge](../05_Bridge_API/How_to_create_a_new_bridge) to the `rss-bridge/bridges` folder. - -If you need to edit any other files, like from the `lib` folder add this to the `volumes` section: `./rss-bridge/lib:/app/lib`. - -### Docs with Docker +## Docs with Docker If you want to edit the docs add this to your docker-compose.yml: From 401cc187b71661cad81a27381763059c047c69e4 Mon Sep 17 00:00:00 2001 From: Eugene Molotov <eugene.molotov@yandex.ru> Date: Fri, 2 Aug 2024 20:44:46 +0500 Subject: [PATCH 146/423] [RutubeBridge] Fix playlist mode returning empty result (#4184) --- bridges/RutubeBridge.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bridges/RutubeBridge.php b/bridges/RutubeBridge.php index 6ec830aa..39577575 100644 --- a/bridges/RutubeBridge.php +++ b/bridges/RutubeBridge.php @@ -80,8 +80,10 @@ class RutubeBridge extends BridgeAbstract $videos = $reduxState->userChannel->videos->results; $this->title = $reduxState->userChannel->info->name; } elseif ($this->getInput('p')) { - $videos = $reduxState->playlist->data->results; - $this->title = $reduxState->playlist->title; + $playListVideosMethod = 'getPlaylistVideos(' . $this->getInput('p') . ')'; + $videos = $reduxState->api->queries->$playListVideosMethod->data->results; + $playListMethod = 'getPlaylist(' . $this->getInput('p') . ')'; + $this->title = $reduxState->api->queries->$playListMethod->data->title; } elseif ($this->getInput('s')) { $this->title = 'Поиск ' . $this->getInput('s'); } From 6ec91935462b2c24daf89b68283e1c413887f2bd Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 7 Aug 2024 00:21:06 +0200 Subject: [PATCH 147/423] yuop (#4193) --- actions/ConnectivityAction.php | 4 ++-- actions/DetectAction.php | 2 +- actions/DisplayAction.php | 2 +- actions/FindfeedAction.php | 2 +- actions/FrontpageAction.php | 6 +++--- actions/HealthAction.php | 2 +- actions/ListAction.php | 2 +- lib/ActionInterface.php | 5 +---- lib/RssBridge.php | 5 +---- 9 files changed, 12 insertions(+), 18 deletions(-) diff --git a/actions/ConnectivityAction.php b/actions/ConnectivityAction.php index eb9edeb1..9732d0aa 100644 --- a/actions/ConnectivityAction.php +++ b/actions/ConnectivityAction.php @@ -19,7 +19,7 @@ class ConnectivityAction implements ActionInterface $this->bridgeFactory = new BridgeFactory(); } - public function execute(Request $request) + public function __invoke(Request $request): Response { if (!Debug::isEnabled()) { return new Response('This action is only available in debug mode!', 403); @@ -27,7 +27,7 @@ class ConnectivityAction implements ActionInterface $bridgeName = $request->get('bridge'); if (!$bridgeName) { - return render_template('connectivity.html.php'); + return new Response(render_template('connectivity.html.php')); } $bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName); if (!$bridgeClassName) { diff --git a/actions/DetectAction.php b/actions/DetectAction.php index 0c61f1b6..cebbc307 100644 --- a/actions/DetectAction.php +++ b/actions/DetectAction.php @@ -2,7 +2,7 @@ class DetectAction implements ActionInterface { - public function execute(Request $request) + public function __invoke(Request $request): Response { $url = $request->get('url'); $format = $request->get('format'); diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index d39d1812..aee47483 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -11,7 +11,7 @@ class DisplayAction implements ActionInterface $this->logger = RssBridge::getLogger(); } - public function execute(Request $request) + public function __invoke(Request $request): Response { $bridgeName = $request->get('bridge'); $format = $request->get('format'); diff --git a/actions/FindfeedAction.php b/actions/FindfeedAction.php index 94dc6b72..6654ca6d 100644 --- a/actions/FindfeedAction.php +++ b/actions/FindfeedAction.php @@ -7,7 +7,7 @@ */ class FindfeedAction implements ActionInterface { - public function execute(Request $request) + public function __invoke(Request $request): Response { $url = $request->get('url'); $format = $request->get('format'); diff --git a/actions/FrontpageAction.php b/actions/FrontpageAction.php index 32795c45..c72dfd57 100644 --- a/actions/FrontpageAction.php +++ b/actions/FrontpageAction.php @@ -2,7 +2,7 @@ final class FrontpageAction implements ActionInterface { - public function execute(Request $request) + public function __invoke(Request $request): Response { $messages = []; $activeBridges = 0; @@ -26,13 +26,13 @@ final class FrontpageAction implements ActionInterface } // todo: cache this renderered template? - return render(__DIR__ . '/../templates/frontpage.html.php', [ + return new Response(render(__DIR__ . '/../templates/frontpage.html.php', [ 'messages' => $messages, 'admin_email' => Configuration::getConfig('admin', 'email'), 'admin_telegram' => Configuration::getConfig('admin', 'telegram'), 'bridges' => $body, 'active_bridges' => $activeBridges, 'total_bridges' => count($bridgeClassNames), - ]); + ])); } } diff --git a/actions/HealthAction.php b/actions/HealthAction.php index a38879c2..13365a3c 100644 --- a/actions/HealthAction.php +++ b/actions/HealthAction.php @@ -4,7 +4,7 @@ declare(strict_types=1); class HealthAction implements ActionInterface { - public function execute(Request $request) + public function __invoke(Request $request): Response { $response = [ 'code' => 200, diff --git a/actions/ListAction.php b/actions/ListAction.php index 3d9cdd73..3dd8f441 100644 --- a/actions/ListAction.php +++ b/actions/ListAction.php @@ -2,7 +2,7 @@ class ListAction implements ActionInterface { - public function execute(Request $request) + public function __invoke(Request $request): Response { $list = new \stdClass(); $list->bridges = []; diff --git a/lib/ActionInterface.php b/lib/ActionInterface.php index c0ddcf9f..d2e1c709 100644 --- a/lib/ActionInterface.php +++ b/lib/ActionInterface.php @@ -2,8 +2,5 @@ interface ActionInterface { - /** - * @return string|Response - */ - public function execute(Request $request); + public function __invoke(Request $request): Response; } diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 1bb5f5ea..87b11f52 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -100,11 +100,8 @@ final class RssBridge $className = '\\' . $actionName; $actionObject = new $className(); - $response = $actionObject->execute($request); + $response = $actionObject($request); - if (is_string($response)) { - $response = new Response($response); - } return $response; } From 4faaa7910174d419152993bc268fb214dda5242e Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 7 Aug 2024 03:15:43 +0200 Subject: [PATCH 148/423] refactor: change the way dependencies are wired (#4194) * refactor: change the way dependencies are setup * lint --- bin/cache-clear | 17 ++++++- bin/cache-prune | 17 ++++++- caches/ArrayCache.php | 3 ++ docs/04_For_Developers/05_Debug_mode.md | 14 +----- index.php | 63 +++++++++++++------------ lib/Configuration.php | 3 ++ lib/Debug.php | 3 ++ lib/RssBridge.php | 35 ++++++-------- lib/bootstrap.php | 6 --- lib/logger.php | 13 ++--- 10 files changed, 93 insertions(+), 81 deletions(-) diff --git a/bin/cache-clear b/bin/cache-clear index 3563abad..635f41d5 100755 --- a/bin/cache-clear +++ b/bin/cache-clear @@ -7,8 +7,21 @@ require __DIR__ . '/../lib/bootstrap.php'; -$rssBridge = new RssBridge(); +$config = []; +if (file_exists(__DIR__ . '/../config.ini.php')) { + $config = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); + if (!$config) { + http_response_code(500); + exit("Error parsing config.ini.php\n"); + } +} +Configuration::loadConfiguration($config, getenv()); -$cache = RssBridge::getCache(); +$logger = new SimpleLogger('rssbridge'); + +$logger->addHandler(new StreamHandler('php://stderr', Logger::INFO)); + +$cacheFactory = new CacheFactory($logger); +$cache = $cacheFactory->create(); $cache->clear(); diff --git a/bin/cache-prune b/bin/cache-prune index 7b7a6031..281c019d 100755 --- a/bin/cache-prune +++ b/bin/cache-prune @@ -7,8 +7,21 @@ require __DIR__ . '/../lib/bootstrap.php'; -$rssBridge = new RssBridge(); +$config = []; +if (file_exists(__DIR__ . '/../config.ini.php')) { + $config = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); + if (!$config) { + http_response_code(500); + exit("Error parsing config.ini.php\n"); + } +} +Configuration::loadConfiguration($config, getenv()); -$cache = RssBridge::getCache(); +$logger = new SimpleLogger('rssbridge'); + +$logger->addHandler(new StreamHandler('php://stderr', Logger::INFO)); + +$cacheFactory = new CacheFactory($logger); +$cache = $cacheFactory->create(); $cache->prune(); diff --git a/caches/ArrayCache.php b/caches/ArrayCache.php index efce4f35..55b18519 100644 --- a/caches/ArrayCache.php +++ b/caches/ArrayCache.php @@ -2,6 +2,9 @@ declare(strict_types=1); +/** + * Also known as an in-memory/runtime cache + */ class ArrayCache implements CacheInterface { private array $data = []; diff --git a/docs/04_For_Developers/05_Debug_mode.md b/docs/04_For_Developers/05_Debug_mode.md index 6bdb1d48..7d503acd 100644 --- a/docs/04_For_Developers/05_Debug_mode.md +++ b/docs/04_For_Developers/05_Debug_mode.md @@ -1,6 +1,7 @@ <h1 align="center">Warning!</h1> -Enabling debug mode on a public server may result in malicious clients retrieving sensitive data about your server and possibly gaining access to it. Do not enable debug mode on a public server, unless you understand the implications of your doing! +Enabling debug mode on a public server may result in malicious clients retrieving sensitive data about your server and possibly gaining access to it. +Do not enable debug mode on a public server, unless you understand the implications of your doing! *** @@ -20,14 +21,3 @@ _Notice_: * The bridge whitelist still applies! (debug mode does **not** enable all bridges) RSS-Bridge will give you a visual feedback when debug mode is enabled. - -While debug mode is active, RSS-Bridge will write additional data to your servers `error.log`. - -Debug mode is controlled by the static class `Debug`. It provides three core functions: - -* `Debug::isEnabled()`: Returns `true` if debug mode is enabled. -* `Debug::log($message)`: Adds a message to `error.log`. It takes one parameter, which can be anything. - -Example: `Debug::log('Hello World!');` - -**Notice**: `Debug::log($message)` calls `Debug::isEnabled()` internally. You don't have to do that manually. \ No newline at end of file diff --git a/index.php b/index.php index f23d8441..ccb2316f 100644 --- a/index.php +++ b/index.php @@ -2,25 +2,31 @@ if (version_compare(\PHP_VERSION, '7.4.0') === -1) { http_response_code(500); - print 'RSS-Bridge requires minimum PHP version 7.4'; - exit; -} - -if (! is_readable(__DIR__ . '/lib/bootstrap.php')) { - http_response_code(500); - print 'Unable to read lib/bootstrap.php. Check file permissions.'; - exit; + exit("RSS-Bridge requires minimum PHP version 7.4\n"); } require_once __DIR__ . '/lib/bootstrap.php'; -set_exception_handler(function (\Throwable $e) { +$config = []; +if (file_exists(__DIR__ . '/config.ini.php')) { + $config = parse_ini_file(__DIR__ . '/config.ini.php', true, INI_SCANNER_TYPED); + if (!$config) { + http_response_code(500); + exit("Error parsing config.ini.php\n"); + } +} +Configuration::loadConfiguration($config, getenv()); + +$logger = new SimpleLogger('rssbridge'); + +set_exception_handler(function (\Throwable $e) use ($logger) { $response = new Response(render(__DIR__ . '/templates/exception.html.php', ['e' => $e]), 500); $response->send(); - RssBridge::getLogger()->error('Uncaught Exception', ['e' => $e]); + $logger->error('Uncaught Exception', ['e' => $e]); }); -set_error_handler(function ($code, $message, $file, $line) { +set_error_handler(function ($code, $message, $file, $line) use ($logger) { + // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); if ((error_reporting() & $code) === 0) { // Deprecation messages and other masked errors are typically ignored here return false; @@ -35,11 +41,12 @@ set_error_handler(function ($code, $message, $file, $line) { sanitize_root($file), $line ); - RssBridge::getLogger()->warning($text); + $logger->warning($text); + // todo: return false to prevent default error handler from running? }); // There might be some fatal errors which are not caught by set_error_handler() or \Throwable. -register_shutdown_function(function () { +register_shutdown_function(function () use ($logger) { $error = error_get_last(); if ($error) { $message = sprintf( @@ -49,33 +56,29 @@ register_shutdown_function(function () { sanitize_root($error['file']), $error['line'] ); - RssBridge::getLogger()->error($message); - if (Debug::isEnabled()) { - // This output can interfere with json output etc - // This output is written at the bottom - print sprintf("<pre>%s</pre>\n", e($message)); - } + $logger->error($message); } }); -$errors = Configuration::checkInstallation(); -if ($errors) { - http_response_code(500); - print '<pre>' . implode("\n", $errors) . '</pre>'; - exit; +$cacheFactory = new CacheFactory($logger); +if (Debug::isEnabled()) { + $logger->addHandler(new StreamHandler('php://stderr', Logger::DEBUG)); + $cache = $cacheFactory->create('array'); +} else { + $logger->addHandler(new StreamHandler('php://stderr', Logger::INFO)); + $cache = $cacheFactory->create(); } - -// Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); +$httpClient = new CurlHttpClient(); date_default_timezone_set(Configuration::getConfig('system', 'timezone')); try { - $rssBridge = new RssBridge(); + $rssBridge = new RssBridge($logger, $cache, $httpClient); $response = $rssBridge->main($argv ?? []); $response->send(); } catch (\Throwable $e) { // Probably an exception inside an action - RssBridge::getLogger()->error('Exception in RssBridge::main()', ['e' => $e]); - http_response_code(500); - print render(__DIR__ . '/templates/exception.html.php', ['e' => $e]); + $logger->error('Exception in RssBridge::main()', ['e' => $e]); + $response = new Response(render(__DIR__ . '/templates/exception.html.php', ['e' => $e]), 500); + $response->send(); } diff --git a/lib/Configuration.php b/lib/Configuration.php index 63f67a3c..b104a251 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -198,6 +198,9 @@ final class Configuration public static function getConfig(string $section, string $key, $default = null) { + if (self::$config === []) { + throw new \Exception('Config has not been loaded'); + } return self::$config[strtolower($section)][strtolower($key)] ?? $default; } diff --git a/lib/Debug.php b/lib/Debug.php index 4333b3a5..ba9e787e 100644 --- a/lib/Debug.php +++ b/lib/Debug.php @@ -16,6 +16,9 @@ class Debug return false; } + /** + * @deprecated Use $this->logger->debug() + */ public static function log($message) { $e = new \Exception(); diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 87b11f52..e80e6f0a 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -2,25 +2,18 @@ final class RssBridge { - private static CacheInterface $cache; private static Logger $logger; + private static CacheInterface $cache; private static HttpClient $httpClient; - public function __construct() - { - self::$logger = new SimpleLogger('rssbridge'); - if (Debug::isEnabled()) { - self::$logger->addHandler(new StreamHandler(Logger::DEBUG)); - } else { - self::$logger->addHandler(new StreamHandler(Logger::INFO)); - } - self::$httpClient = new CurlHttpClient(); - $cacheFactory = new CacheFactory(self::$logger); - if (Debug::isEnabled()) { - self::$cache = $cacheFactory->create('array'); - } else { - self::$cache = $cacheFactory->create(); - } + public function __construct( + Logger $logger, + CacheInterface $cache, + HttpClient $httpClient + ) { + self::$logger = $logger; + self::$cache = $cache; + self::$httpClient = $httpClient; } public function main(array $argv = []): Response @@ -105,16 +98,16 @@ final class RssBridge return $response; } - public static function getCache(): CacheInterface - { - return self::$cache; - } - public static function getLogger(): Logger { return self::$logger; } + public static function getCache(): CacheInterface + { + return self::$cache; + } + public static function getHttpClient(): HttpClient { return self::$httpClient; diff --git a/lib/bootstrap.php b/lib/bootstrap.php index bfc7be39..1d866067 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -45,9 +45,3 @@ spl_autoload_register(function ($className) { } } }); - -$customConfig = []; -if (file_exists(__DIR__ . '/../config.ini.php')) { - $customConfig = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); -} -Configuration::loadConfiguration($customConfig, getenv()); diff --git a/lib/logger.php b/lib/logger.php index e579915d..916e65ed 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -83,10 +83,12 @@ final class SimpleLogger implements Logger final class StreamHandler { + private $stream; private int $level; - public function __construct(int $level = Logger::DEBUG) + public function __construct(string $stream, int $level = Logger::DEBUG) { + $this->stream = $stream; $this->level = $level; } @@ -147,13 +149,8 @@ final class StreamHandler $record['message'], $context ); - error_log($text); - if ($record['level'] < Logger::ERROR && Debug::isEnabled()) { - // The record level is INFO or WARNING here - // Not a good idea to print here because http headers might not have been sent - print sprintf("<pre>%s</pre>\n", e($text)); - } - //$bytes = file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); + + $bytes = file_put_contents($this->stream, $text, FILE_APPEND | LOCK_EX); } } From 313be4c5125158f1413afc1269135d939d976cc0 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger <chris.schabesberger@mailbox.org> Date: Wed, 7 Aug 2024 15:51:44 +0200 Subject: [PATCH 149/423] replace self:: with -> for methodcalls in Nordbayern bridge (#4195) --- bridges/NordbayernBridge.php | 41 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/bridges/NordbayernBridge.php b/bridges/NordbayernBridge.php index a5f34817..48157921 100644 --- a/bridges/NordbayernBridge.php +++ b/bridges/NordbayernBridge.php @@ -53,6 +53,19 @@ class NordbayernBridge extends BridgeAbstract ] ]]; + public function collectData() + { + $region = $this->getInput('region'); + if ($region === 'rothenburg-o-d-t') { + $region = 'rothenburg-ob-der-tauber'; + } + $url = self::URI . '/region/' . $region; + $listSite = getSimpleHTMLDOM($url); + + $this->handleNewsblock($listSite); + } + + private function getValidImage($picture) { $img = $picture->find('img', 0); @@ -75,23 +88,23 @@ class NordbayernBridge extends BridgeAbstract ) { $content .= $element; } elseif ($element->tag === 'main') { - $content .= self::getUseFullContent($element->find('article', 0)); + $content .= $this->getUseFullContent($element->find('article', 0)); } elseif ($element->tag === 'header') { - $content .= self::getUseFullContent($element); + $content .= $this->getUseFullContent($element); } elseif ( $element->tag === 'div' && !str_contains($element->class, 'article__infobox') && !str_contains($element->class, 'authorinfo') ) { - $content .= self::getUseFullContent($element); + $content .= $this->getUseFullContent($element); } elseif ( $element->tag === 'section' && (str_contains($element->class, 'article__richtext') || str_contains($element->class, 'article__context')) ) { - $content .= self::getUseFullContent($element); + $content .= $this->getUseFullContent($element); } elseif ($element->tag === 'picture') { - $content .= self::getValidImage($element); + $content .= $this->getValidImage($element); } elseif ($element->tag === 'ul') { $content .= $element; } @@ -146,8 +159,8 @@ class NordbayernBridge extends BridgeAbstract // of the title image. If we didn't do this some rss programs // would show the subtitle of the title image as teaser instead // of the actuall article teaser. - $item['content'] .= self::getTeaser($content); - $item['content'] .= self::getUseFullContent($content); + $item['content'] .= $this->getTeaser($content); + $item['content'] .= $this->getUseFullContent($content); } @@ -169,7 +182,7 @@ class NordbayernBridge extends BridgeAbstract continue; } - $item = self::getArticle($url); + $item = $this->getArticle($url); // exclude police reports if desired if ( @@ -190,16 +203,4 @@ class NordbayernBridge extends BridgeAbstract $this->items[] = $item; } } - - public function collectData() - { - $region = $this->getInput('region'); - if ($region === 'rothenburg-o-d-t') { - $region = 'rothenburg-ob-der-tauber'; - } - $url = self::URI . '/region/' . $region; - $listSite = getSimpleHTMLDOM($url); - - self::handleNewsblock($listSite); - } } From c11bc184caec2537f80161ca4d792fa238ddeceb Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 7 Aug 2024 18:09:44 +0200 Subject: [PATCH 150/423] fix: restore php error_log writing (#4196) --- index.php | 5 +-- lib/logger.php | 83 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/index.php b/index.php index ccb2316f..d29900f7 100644 --- a/index.php +++ b/index.php @@ -61,11 +61,12 @@ register_shutdown_function(function () use ($logger) { }); $cacheFactory = new CacheFactory($logger); + if (Debug::isEnabled()) { - $logger->addHandler(new StreamHandler('php://stderr', Logger::DEBUG)); + $logger->addHandler(new ErrorLogHandler(Logger::DEBUG)); $cache = $cacheFactory->create('array'); } else { - $logger->addHandler(new StreamHandler('php://stderr', Logger::INFO)); + $logger->addHandler(new ErrorLogHandler(Logger::INFO)); $cache = $cacheFactory->create(); } $httpClient = new CurlHttpClient(); diff --git a/lib/logger.php b/lib/logger.php index 916e65ed..3ebe3b0a 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -68,6 +68,16 @@ final class SimpleLogger implements Logger private function log(int $level, string $message, array $context = []): void { + $ignoredMessages = [ + 'Format name invalid', + 'Unknown format given', + 'Unable to find channel', + ]; + foreach ($ignoredMessages as $ignoredMessage) { + if (str_starts_with($message, $ignoredMessage)) { + return; + } + } foreach ($this->handlers as $handler) { $handler([ 'name' => $this->name, @@ -83,7 +93,7 @@ final class SimpleLogger implements Logger final class StreamHandler { - private $stream; + private string $stream; private int $level; public function __construct(string $stream, int $level = Logger::DEBUG) @@ -108,28 +118,6 @@ final class StreamHandler $record['context']['line'] = $e->getLine(); $record['context']['url'] = get_current_url(); $record['context']['trace'] = trace_to_call_points(trace_from_exception($e)); - - $ignoredExceptions = [ - 'You must specify a format', - 'Format name invalid', - 'Unknown format given', - 'Bridge name invalid', - 'Invalid action', - 'twitter: No results for this query', - // telegram - 'Unable to find channel. The channel is non-existing or non-public', - // fb - 'This group is not public! RSS-Bridge only supports public groups!', - 'You must be logged in to view this page', - 'Unable to get the page id. You should consider getting the ID by hand', - // tiktok 404 - 'https://www.tiktok.com/@', - ]; - foreach ($ignoredExceptions as $ignoredException) { - if (str_starts_with($e->getMessage(), $ignoredException)) { - return; - } - } } $context = ''; if ($record['context']) { @@ -145,15 +133,60 @@ final class StreamHandler $record['created_at']->format('Y-m-d H:i:s'), $record['name'], $record['level_name'], - // Should probably sanitize message for output context $record['message'], $context ); - $bytes = file_put_contents($this->stream, $text, FILE_APPEND | LOCK_EX); } } +final class ErrorLogHandler +{ + private int $level; + + public function __construct(int $level = Logger::DEBUG) + { + $this->level = $level; + } + + public function __invoke(array $record) + { + if ($record['level'] < $this->level) { + return; + } + if (isset($record['context']['e'])) { + /** @var \Throwable $e */ + $e = $record['context']['e']; + unset($record['context']['e']); + $record['context']['type'] = get_class($e); + $record['context']['code'] = $e->getCode(); + $record['context']['message'] = sanitize_root($e->getMessage()); + $record['context']['file'] = sanitize_root($e->getFile()); + $record['context']['line'] = $e->getLine(); + $record['context']['url'] = get_current_url(); + $record['context']['trace'] = trace_to_call_points(trace_from_exception($e)); + } + $context = ''; + if ($record['context']) { + try { + $context = Json::encode($record['context']); + } catch (\JsonException $e) { + $record['context']['message'] = null; + $context = Json::encode($record['context']); + } + } + $text = sprintf( + "[%s] %s.%s %s %s\n", + $record['created_at']->format('Y-m-d H:i:s'), + $record['name'], + $record['level_name'], + $record['message'], + $context + ); + error_log($text); + } +} + final class NullLogger implements Logger { public function debug(string $message, array $context = []) From 9215b95779ff2f0bd8a74d7213f3bfa09e452088 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 7 Aug 2024 18:56:27 +0200 Subject: [PATCH 151/423] fix: bug in prior refactor (#4197) --- actions/DisplayAction.php | 2 +- lib/FeedItem.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index aee47483..c51bb7cd 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -145,7 +145,7 @@ class DisplayAction implements ActionInterface if ($errorCount >= $reportLimit) { if ($errorOutput === 'feed') { // Render the exception as a feed item - $items[] = $this->createFeedItemFromException($e, $bridge); + $items = [$this->createFeedItemFromException($e, $bridge)]; } elseif ($errorOutput === 'http') { return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 500); } elseif ($errorOutput === 'none') { diff --git a/lib/FeedItem.php b/lib/FeedItem.php index 09aab129..8a092a27 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -136,7 +136,7 @@ class FeedItem { $this->title = null; if (!is_string($title)) { - Debug::log('Title must be a string!'); + trigger_error('Title must be a string: ' . print_r($title, true)); } else { $this->title = truncate(trim($title)); } From ee54cf457623afd48538b844d1fbb8b714be7674 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger <chris.schabesberger@mailbox.org> Date: Thu, 8 Aug 2024 00:00:26 +0200 Subject: [PATCH 152/423] add NurembergerNachrichten bridge (#4185) * add NurembergerNachrichten bridge apply suggested changes and fix regions put collectData on top replace self:: with -> for methodcalls * refactor: remove unused var * refactor: order methods * fix --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/NurembergerNachrichtenBridge.php | 178 +++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 bridges/NurembergerNachrichtenBridge.php diff --git a/bridges/NurembergerNachrichtenBridge.php b/bridges/NurembergerNachrichtenBridge.php new file mode 100644 index 00000000..10644212 --- /dev/null +++ b/bridges/NurembergerNachrichtenBridge.php @@ -0,0 +1,178 @@ +<?php + +class NurembergerNachrichtenBridge extends BridgeAbstract +{ + const MAINTAINER = 'schabi.org'; + const NAME = 'Nürnberger Nachrichten'; + const CACHE_TIMEOUT = 3600; + const URI = 'https://www.nn.de'; + const DESCRIPTION = 'Bridge for Bavarian regional news site nordbayern.de'; + const PARAMETERS = [ [ + 'region' => [ + 'name' => 'region', + 'type' => 'list', + 'exampleValue' => 'Nürnberg', + 'title' => 'Select a region', + 'values' => [ + 'Ansbach' => 'ansbach', + 'Erlangen' => 'erlangen', + 'Erlangen-Höchstadt' => 'erlangen-hoechstadt', + 'Forchheim' => 'forchheim', + 'Fürth' => 'fuerth', + 'Gunzenhausen' => 'gunzenhausen', + 'Neumarkt' => 'neumarkt', + 'Neustadt/Aisch-Bad Windsheim' => 'neustadt-aisch-bad-windsheim', + 'Nürnberg' => 'nuernberg', + 'Nürnberger Land' => 'nuernberger-land', + 'Pegnitz' => 'pegnitz', + 'Roth' => 'roth', + 'Schwabach' => 'schwabach', + 'Weißenburg' => 'weissenburg' + ] + ], + 'hideNNPlus' => [ + 'name' => 'Hide NN+ articles', + 'type' => 'checkbox', + 'exampleValue' => 'unchecked', + 'title' => 'Hide all paywall articles on NN' + ], + ]]; + + public function collectData() + { + $region = $this->getInput('region'); + if ( + $region === 'neustadt-aisch-bad-windsheim' || + $region === 'erlangen-hoechstadt' || + $region === '' + ) { + $region = 'region/' . $region; + } + $url = self::URI . '/' . $region; + $listSite = getSimpleHTMLDOM($url); + + $this->handleNewsblock($listSite); + } + + private function handleNewsblock($listSite) + { + $main = $listSite->find('main', 0); + foreach ($main->find('article') as $article) { + $url = $article->find('a', 0)->href; + $url = urljoin(self::URI, $url); + + $articleContent = getSimpleHTMLDOMCached($url, 86400 * 7); + + // exclude nn+ articles if desired + if ( + $this->getInput('hideNNPlus') && + str_contains($articleContent->find('article[id=article]', 0)->find('header', 0), 'icon-nnplus') + ) { + continue; + } + + $item = $this->parseArticle($articleContent, $url); + $articleContent->clear(); + + $this->items[] = $item; + } + } + + private function parseArticle($article, $link) + { + $item = []; + defaultLinkTo($article, self::URI); + + $item['uri'] = $link; + + $author = $article->find('.article__author', 1); + if ($author !== null) { + $item['author'] = trim($author->plaintext); + } + + $createdAt = $article->find('[class=article__release]', 0); + if ($createdAt) { + $item['timestamp'] = strtotime(str_replace('Uhr', '', $createdAt->plaintext)); + } + + if ($article->find('h2', 0) === null) { + $item['title'] = $article->find('h3', 0)->innertext; + } else { + $item['title'] = $article->find('h2', 0)->innertext; + } + $item['content'] = ''; + + if ($article->find('section[class*=article__richtext]', 0) === null) { + $content = $article->find('div[class*=modul__teaser]', 0)->find('p', 0); + $item['content'] .= $content; + } else { + $content = $article->find('article', 0); + // change order of article teaser in order to show it on top + // of the title image. If we didn't do this some rss programs + // would show the subtitle of the title image as teaser instead + // of the actuall article teaser. + $item['content'] .= $this->getTeaser($content); + $item['content'] .= $this->getUseFullContent($content); + } + + return $item; + } + + private function getTeaser($content) + { + $teaser = $content->find('p[class=article__teaser]', 0); + if ($teaser === null) { + return ''; + } + $teaser = $teaser->plaintext; + $teaser = preg_replace('/[ ]{2,}/', ' ', $teaser); + $teaser = '<p class="article__teaser">' . $teaser . '</p>'; + return $teaser; + } + + private function getUseFullContent($rawContent) + { + $content = ''; + foreach ($rawContent->children as $element) { + if ( + ($element->tag === 'p' || $element->tag === 'h3') && + $element->class !== 'article__teaser' + ) { + $content .= $element; + } elseif ($element->tag === 'main') { + $content .= $this->getUseFullContent($element->find('article', 0)); + } elseif ($element->tag === 'header') { + $content .= $this->getUseFullContent($element); + } elseif ( + $element->tag === 'div' && + !str_contains($element->class, 'article__infobox') && + !str_contains($element->class, 'authorinfo') + ) { + $content .= $this->getUseFullContent($element); + } elseif ( + $element->tag === 'section' && + (str_contains($element->class, 'article__richtext') || + str_contains($element->class, 'article__context')) + ) { + $content .= $this->getUseFullContent($element); + } elseif ($element->tag === 'picture') { + $content .= $this->getValidImage($element); + } elseif ($element->tag === 'ul') { + $content .= $element; + } + } + return $content; + } + + private function getValidImage($picture) + { + $img = $picture->find('img', 0); + if ($img) { + $imgUrl = $img->src; + if (!preg_match('#/logo-.*\.png#', $imgUrl)) { + return '<br><img src="' . $imgUrl . '">'; + } + } + return ''; + } +} From b25a779d98e4ed674496e6184b424e0a594f2364 Mon Sep 17 00:00:00 2001 From: Pavel Korytov <thexcloud@gmail.com> Date: Thu, 8 Aug 2024 01:27:33 +0300 Subject: [PATCH 153/423] [TldrTechBridge] Fix bridge (#4187) * [TldrTechBridge] Fix bridge * yup --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/TldrTechBridge.php | 64 ++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/bridges/TldrTechBridge.php b/bridges/TldrTechBridge.php index d2955347..6c96dff7 100644 --- a/bridges/TldrTechBridge.php +++ b/bridges/TldrTechBridge.php @@ -1,12 +1,12 @@ <?php +declare(strict_types=1); + class TldrTechBridge extends BridgeAbstract { const MAINTAINER = 'sqrtminusone'; const NAME = 'TLDR Tech Newsletter Bridge'; const URI = 'https://tldr.tech/'; - - const CACHE_TIMEOUT = 3600; // 1 hour const DESCRIPTION = 'Return newsletter articles from TLDR Tech'; const PARAMETERS = [ @@ -41,41 +41,53 @@ class TldrTechBridge extends BridgeAbstract { $topic = $this->getInput('topic'); $limit = $this->getInput('limit'); - $url = self::URI . $topic . '/archives'; - $html = getSimpleHTMLDOM($url); - $entries_root = $html->find('div.content-center.mt-5', 0); - $added = 0; + + $url = self::URI . 'api/latest/' . $topic; + $response = getContents($url, [], [], true); + $location = $response->getHeader('Location'); + $locationUrl = Url::fromString($location); + + $this->extractItem($locationUrl); + + $archives_url = self::URI . $topic . '/archives'; + $archives_html = getSimpleHTMLDOM($archives_url); + $entries_root = $archives_html->find('div.content-center.mt-5', 0); foreach ($entries_root->children() as $child) { if ($child->tag != 'a') { continue; } - // Convert /<topic>/2023-01-01 to unix timestamp - $date_items = explode('/', $child->href); - $date = strtotime(end($date_items)); - $item_url = self::URI . ltrim($child->href, '/'); - try { - $this->items[] = [ - 'uri' => self::URI . $child->href, - 'title' => $child->plaintext, - 'timestamp' => $date, - 'content' => $this->extractContent($item_url), - ]; - } catch (HttpException $e) { - continue; - } - $added++; - if ($added >= $limit) { + $this->extractItem(Url::fromString(self::URI . $child->href)); + if (count($this->items) >= $limit) { break; } } } + private function extractItem(Url $url) + { + $pathParts = explode('/', $url->getPath()); + $date = strtotime(end($pathParts)); + try { + [$content, $title] = $this->extractContent($url); + + $this->items[] = [ + 'uri' => (string) $url, + 'title' => $title, + 'timestamp' => $date, + 'content' => $content, + ]; + } catch (HttpException $e) { + // archive occasionally returns broken URLs + return; + } + } + private function extractContent($url) { - $html = getSimpleHTMLDOM($url); + $html = getSimpleHTMLDOMCached($url); $content = $html->find('div.content-center.mt-5', 0); if (!$content) { - throw new HttpException('Could not find content', 500); + throw new \Exception('Could not find content'); } $subscribe_form = $content->find('div.mt-5 > div > form', 0); if ($subscribe_form) { @@ -112,7 +124,7 @@ class TldrTechBridge extends BridgeAbstract } } } - - return $content->innertext; + $title = $content->find('h2', 0); + return [$content->innertext, $title->plaintext]; } } From 829d570f8e4f6b38f04b2fac4a3da387e64bf403 Mon Sep 17 00:00:00 2001 From: "Quentin B." <quent1-fr@users.noreply.github.com> Date: Thu, 8 Aug 2024 00:57:40 +0200 Subject: [PATCH 154/423] [CentreFranceBridge] Add bridge (#4189) * [CentreFranceBridge] Add bridge * [CentreFranceBridge] Fix bridge * [CentreFranceBridge] Fix bridge * [CentreFranceBridge] Improved icon choice * [CentreFranceBridge] Fetch additional data from articles * [CentreFranceBridge] New parameter to allow client to control how many articles to fetch * [CentreFranceBridge] Improve bridge name based on existing parameters * [CentreFranceBridge] Fixed some edge cases * refactor: reorder * fix --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/CentreFranceBridge.php | 279 ++++++++++++++++++++++++++ lib/contents.php | 4 +- lib/simplehtmldom/simple_html_dom.php | 5 - 3 files changed, 281 insertions(+), 7 deletions(-) create mode 100644 bridges/CentreFranceBridge.php diff --git a/bridges/CentreFranceBridge.php b/bridges/CentreFranceBridge.php new file mode 100644 index 00000000..a6dea227 --- /dev/null +++ b/bridges/CentreFranceBridge.php @@ -0,0 +1,279 @@ +<?php + +class CentreFranceBridge extends BridgeAbstract +{ + const NAME = 'Centre France Newspapers'; + const URI = 'https://www.centrefrance.com/'; + const DESCRIPTION = 'Common bridge for all Centre France group newspapers.'; + const CACHE_TIMEOUT = 7200; // 2h + const MAINTAINER = 'quent1'; + const PARAMETERS = [ + 'global' => [ + 'newspaper' => [ + 'name' => 'Newspaper', + 'type' => 'list', + 'values' => [ + 'La Montagne' => 'lamontagne.fr', + 'Le Populaire du Centre' => 'lepopulaire.fr', + 'La République du Centre' => 'larep.fr', + 'Le Berry Républicain' => 'leberry.fr', + 'L\'Yonne Républicaine' => 'lyonne.fr', + 'L\'Écho Républicain' => 'lechorepublicain.fr', + 'Le Journal du Centre' => 'lejdc.fr', + 'L\'Éveil de la Haute-Loire' => 'leveil.fr', + 'Le Pays' => 'le-pays.fr' + ] + ], + 'remove-reserved-for-subscribers-articles' => [ + 'name' => 'Remove reserved for subscribers articles', + 'type' => 'checkbox', + 'title' => 'Filter out articles that are only available to subscribers' + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'title' => 'How many articles to fetch. 0 to disable.', + 'required' => true, + 'defaultValue' => 15 + ] + ], + 'Local news' => [ + 'locality-slug' => [ + 'name' => 'Locality slug', + 'type' => 'text', + 'required' => false, + 'title' => 'Fetch articles for a specific locality. If not set, headlines from the front page will be used instead.', + 'exampleValue' => 'moulins-03000' + ], + ] + ]; + + public function collectData() + { + $value = $this->getInput('limit'); + if (is_numeric($value) && (int)$value >= 0) { + $limit = $value; + } else { + $limit = static::PARAMETERS['global']['limit']['defaultValue']; + } + + if (empty($this->getInput('newspaper'))) { + return; + } + + $localitySlug = $this->getInput('locality-slug') ?? ''; + $alreadyFoundArticlesURIs = []; + + $newspaperUrl = 'https://www.' . $this->getInput('newspaper') . '/' . $localitySlug . '/'; + $html = getSimpleHTMLDOM($newspaperUrl); + + // Articles are detected through their titles + foreach ($html->find('.c-titre') as $articleTitleDOMElement) { + $articleLinkDOMElement = $articleTitleDOMElement->find('a', 0); + + // Ignore articles in the « Les + partagés » block + if (strpos($articleLinkDOMElement->id, 'les_plus_partages') !== false) { + continue; + } + + $articleURI = $articleLinkDOMElement->href; + + // If the URI has already been processed, ignore it + if (in_array($articleURI, $alreadyFoundArticlesURIs, true)) { + continue; + } + + // If news are filtered for a specific locality, filter out article for other localities + if ($localitySlug !== '' && !str_contains($articleURI, $localitySlug)) { + continue; + } + + $articleTitle = ''; + + // If article is reserved for subscribers + if ($articleLinkDOMElement->find('span.premium-picto', 0)) { + if ($this->getInput('remove-reserved-for-subscribers-articles') === true) { + continue; + } + + $articleTitle .= '🔒 '; + } + + $articleTitleDOMElement = $articleLinkDOMElement->find('span[data-tb-title]', 0); + if ($articleTitleDOMElement === null) { + continue; + } + + if ($limit > 0 && count($this->items) === $limit) { + break; + } + + $articleTitle .= $articleLinkDOMElement->find('span[data-tb-title]', 0)->innertext; + $articleFullURI = urljoin('https://www.' . $this->getInput('newspaper') . '/', $articleURI); + + $item = [ + 'title' => $articleTitle, + 'uri' => $articleFullURI, + ...$this->collectArticleData($articleFullURI) + ]; + $this->items[] = $item; + + $alreadyFoundArticlesURIs[] = $articleURI; + } + } + + private function collectArticleData($uri): array + { + $html = getSimpleHTMLDOMCached($uri, 86400 * 90); // 90d + + $item = [ + 'enclosures' => [], + ]; + + $articleInformations = $html->find('.c-article-informations p'); + if (is_array($articleInformations) && $articleInformations !== []) { + $authorPosition = 1; + + // Article publication date + if (preg_match('/(\d{2})\/(\d{2})\/(\d{4})( à (\d{2})h(\d{2}))?/', $articleInformations[0]->innertext, $articleDateParts) > 0) { + $articleDate = new \DateTime('midnight'); + $articleDate->setDate($articleDateParts[3], $articleDateParts[2], $articleDateParts[1]); + + if (count($articleDateParts) === 7) { + $articleDate->setTime($articleDateParts[5], $articleDateParts[6]); + } + + $item['timestamp'] = $articleDate->getTimestamp(); + } + + // Article update date + if (count($articleInformations) >= 2 && preg_match('/(\d{2})\/(\d{2})\/(\d{4})( à (\d{2})h(\d{2}))?/', $articleInformations[1]->innertext, $articleDateParts) > 0) { + $authorPosition = 2; + + $articleDate = new \DateTime('midnight'); + $articleDate->setDate($articleDateParts[3], $articleDateParts[2], $articleDateParts[1]); + + if (count($articleDateParts) === 7) { + $articleDate->setTime($articleDateParts[5], $articleDateParts[6]); + } + + $item['timestamp'] = $articleDate->getTimestamp(); + } + + if (count($articleInformations) === ($authorPosition + 1)) { + $item['author'] = $articleInformations[$authorPosition]->innertext; + } + } + + $articleContent = $html->find('.b-article .contenu > *'); + if (is_array($articleContent)) { + $item['content'] = ''; + + foreach ($articleContent as $contentPart) { + if (in_array($contentPart->getAttribute('id'), ['cf-audio-player', 'poool-widget'], true)) { + continue; + } + + $articleHiddenParts = $contentPart->find('.bloc, .p402_hide'); + if (is_array($articleHiddenParts)) { + foreach ($articleHiddenParts as $articleHiddenPart) { + $contentPart->removeChild($articleHiddenPart); + } + } + + $item['content'] .= $contentPart->innertext; + } + } + + $articleIllustration = $html->find('.photo-wrapper .photo-box img'); + if (is_array($articleIllustration) && count($articleIllustration) === 1) { + $item['enclosures'][] = $articleIllustration[0]->getAttribute('src'); + } + + $articleAudio = $html->find('#cf-audio-player-container audio'); + if (is_array($articleAudio) && count($articleAudio) === 1) { + $item['enclosures'][] = $articleAudio[0]->getAttribute('src'); + } + + $articleTags = $html->find('.b-article > ul.c-tags > li > a.t-simple'); + if (is_array($articleTags)) { + $item['categories'] = array_map(static fn ($articleTag) => $articleTag->innertext, $articleTags); + } + + $explode = explode('_', $uri); + $array_reverse = array_reverse($explode); + $string = $array_reverse[0]; + $uid = rtrim($string, '/'); + if (is_numeric($uid)) { + $item['uid'] = $uid; + } + + // If the article is a "grand format", we use another parsing strategy + if ($item['content'] === '' && $html->find('article') !== []) { + $articleContent = $html->find('article > section'); + foreach ($articleContent as $contentPart) { + if ($contentPart->find('#journo') !== []) { + $item['author'] = $contentPart->find('#journo')->innertext; + continue; + } + + $item['content'] .= $contentPart->innertext; + } + } + + $item['content'] = str_replace('<span class="p-premium">premium</span>', '🔒', $item['content']); + $item['content'] = trim($item['content']); + + return $item; + } + + public function getName() + { + if (empty($this->getInput('newspaper'))) { + return static::NAME; + } + + $newspaperNameByDomain = array_flip(self::PARAMETERS['global']['newspaper']['values']); + if (!isset($newspaperNameByDomain[$this->getInput('newspaper')])) { + return static::NAME; + } + + $completeTitle = $newspaperNameByDomain[$this->getInput('newspaper')]; + + if (!empty($this->getInput('locality-slug'))) { + $localityName = explode('-', $this->getInput('locality-slug')); + array_pop($localityName); + $completeTitle .= ' ' . ucfirst(implode('-', $localityName)); + } + + return $completeTitle; + } + + public function getIcon() + { + if (empty($this->getInput('newspaper'))) { + return static::URI . '/favicon.ico'; + } + + return 'https://www.' . $this->getInput('newspaper') . '/favicon.ico'; + } + + public function detectParameters($url) + { + $regex = '/^(https?:\/\/)?(www\.)?([a-z-]+\.fr)(\/)?([a-z-]+-[0-9]{5})?(\/)?$/'; + $url = strtolower($url); + + if (preg_match($regex, $url, $urlMatches) === 0) { + return null; + } + + if (!in_array($urlMatches[3], self::PARAMETERS['global']['newspaper']['values'], true)) { + return null; + } + + return [ + 'newspaper' => $urlMatches[3], + 'locality-slug' => empty($urlMatches[5]) ? null : $urlMatches[5] + ]; + } +} diff --git a/lib/contents.php b/lib/contents.php index 893a3512..cc9542a9 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -142,7 +142,6 @@ function getContents( * when returning plaintext. * @param string $defaultSpanText Specifies the replacement text for `<span />` * tags when returning plaintext. - * @return false|simple_html_dom Contents as simplehtmldom object. */ function getSimpleHTMLDOM( $url, @@ -154,11 +153,12 @@ function getSimpleHTMLDOM( $stripRN = true, $defaultBRText = DEFAULT_BR_TEXT, $defaultSpanText = DEFAULT_SPAN_TEXT -) { +): \simple_html_dom { $html = getContents($url, $header ?? [], $opts ?? []); if ($html === '') { throw new \Exception('Unable to parse dom because the http response was the empty string'); } + return str_get_html( $html, $lowercase, diff --git a/lib/simplehtmldom/simple_html_dom.php b/lib/simplehtmldom/simple_html_dom.php index 3fc95760..170f6fb0 100644 --- a/lib/simplehtmldom/simple_html_dom.php +++ b/lib/simplehtmldom/simple_html_dom.php @@ -118,11 +118,6 @@ function str_get_html( throw new \Exception('Refusing to parse too big input'); } - if (empty($str) || strlen($str) > MAX_FILE_SIZE) { - $dom->clear(); - return false; - } - return $dom->load($str, $lowercase, $stripRN); } From 8c4385e61d2a8c0f23c120e082a6b832415f22fa Mon Sep 17 00:00:00 2001 From: "Quentin B." <quent1-fr@users.noreply.github.com> Date: Thu, 8 Aug 2024 01:09:13 +0200 Subject: [PATCH 155/423] [BodaccBridge] Add bridge (#4190) * [BodaccBridge] Add bridge * [BodaccBridge] Fix bridge * [BodaccBridge] Fix API url * fix --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/BodaccBridge.php | 218 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 bridges/BodaccBridge.php diff --git a/bridges/BodaccBridge.php b/bridges/BodaccBridge.php new file mode 100644 index 00000000..38e5856a --- /dev/null +++ b/bridges/BodaccBridge.php @@ -0,0 +1,218 @@ +<?php + +class BodaccBridge extends BridgeAbstract +{ + const NAME = 'BODACC'; + const URI = 'https://bodacc-datadila.opendatasoft.com/'; + const DESCRIPTION = 'Fetches announces from the French Government "Bulletin Officiel Des Annonces Civiles et Commerciales".'; + const CACHE_TIMEOUT = 86400; + const MAINTAINER = 'quent1'; + const PARAMETERS = [ + 'Annonces commerciales' => [ + 'departement' => [ + 'name' => 'Département', + 'type' => 'list', + 'values' => [ + 'Tous' => null, + 'Ain' => '01', + 'Aisne' => '02', + 'Allier' => '03', + 'Alpes-de-Haute-Provence' => '04', + 'Hautes-Alpes' => '05', + 'Alpes-Maritimes' => '06', + 'Ardèche' => '07', + 'Ardennes' => '08', + 'Ariège' => '09', + 'Aube' => '10', + 'Aude' => '11', + 'Aveyron' => '12', + 'Bouches-du-Rhône' => '13', + 'Calvados' => '14', + 'Cantal' => '15', + 'Charente' => '16', + 'Charente-Maritime' => '17', + 'Cher' => '18', + 'Corrèze' => '19', + 'Corse-du-Sud' => '2A', + 'Haute-Corse' => '2B', + 'Côte-d\'Or' => '21', + 'Côtes-d\'Armor' => '22', + 'Creuse' => '23', + 'Dordogne' => '24', + 'Doubs' => '25', + 'Drôme' => '26', + 'Eure' => '27', + 'Eure-et-Loir' => '28', + 'Finistère' => '29', + 'Gard' => '30', + 'Haute-Garonne' => '31', + 'Gers' => '32', + 'Gironde' => '33', + 'Hérault' => '34', + 'Ille-et-Vilaine' => '35', + 'Indre' => '36', + 'Indre-et-Loire' => '37', + 'Isère' => '38', + 'Jura' => '39', + 'Landes' => '40', + 'Loir-et-Cher' => '41', + 'Loire' => '42', + 'Haute-Loire' => '43', + 'Loire-Atlantique' => '44', + 'Loiret' => '45', + 'Lot' => '46', + 'Lot-et-Garonne' => '47', + 'Lozère' => '48', + 'Maine-et-Loire' => '49', + 'Manche' => '50', + 'Marne' => '51', + 'Haute-Marne' => '52', + 'Mayenne' => '53', + 'Meurthe-et-Moselle' => '54', + 'Meuse' => '55', + 'Morbihan' => '56', + 'Moselle' => '57', + 'Nièvre' => '58', + 'Nord' => '59', + 'Oise' => '60', + 'Orne' => '61', + 'Pas-de-Calais' => '62', + 'Puy-de-Dôme' => '63', + 'Pyrénées-Atlantiques' => '64', + 'Hautes-Pyrénées' => '65', + 'Pyrénées-Orientales' => '66', + 'Bas-Rhin' => '67', + 'Haut-Rhin' => '68', + 'Rhône' => '69', + 'Haute-Saône' => '70', + 'Saône-et-Loire' => '71', + 'Sarthe' => '72', + 'Savoie' => '73', + 'Haute-Savoie' => '74', + 'Paris' => '75', + 'Seine-Maritime' => '76', + 'Seine-et-Marne' => '77', + 'Yvelines' => '78', + 'Deux-Sèvres' => '79', + 'Somme' => '80', + 'Tarn' => '81', + 'Tarn-et-Garonne' => '82', + 'Var' => '83', + 'Vaucluse' => '84', + 'Vendée' => '85', + 'Vienne' => '86', + 'Haute-Vienne' => '87', + 'Vosges' => '88', + 'Yonne' => '89', + 'Territoire de Belfort' => '90', + 'Essonne' => '91', + 'Hauts-de-Seine' => '92', + 'Seine-Saint-Denis' => '93', + 'Val-de-Marne' => '94', + 'Val-d\'Oise' => '95', + 'Guadeloupe' => '971', + 'Martinique' => '972', + 'Guyane' => '973', + 'La Réunion' => '974', + 'Saint-Pierre-et-Miquelon' => '975', + 'Mayotte' => '976', + 'Saint-Barthélemy' => '977', + 'Saint-Martin' => '978', + 'Terres australes et antarctiques françaises' => '984', + 'Wallis-et-Futuna' => '986', + 'Polynésie française' => '987', + 'Nouvelle-Calédonie' => '988', + 'Île de Clipperton' => '989' + ] + ], + 'famille' => [ + 'name' => 'Famille', + 'type' => 'list', + 'values' => [ + 'Toutes' => null, + 'Annonces diverses' => 'divers', + 'Créations' => 'creation', + 'Dépôts des comptes' => 'dpc', + 'Immatriculations' => 'immatriculation', + 'Modifications diverses' => 'modification', + 'Procédures collectives' => 'collective', + 'Procédures de conciliation' => 'conciliation', + 'Procédures de rétablissement professionnel' => 'retablissement_professionnel', + 'Radiations' => 'radiation', + 'Ventes et cessions' => 'vente' + ] + ], + 'type' => [ + 'name' => 'Type', + 'type' => 'list', + 'values' => [ + 'Tous' => null, + 'Avis initial' => 'annonce', + 'Avis d\'annulation' => 'annulation', + 'Avis rectificatif' => 'rectificatif' + ] + ] + ] + ]; + + public function collectData() + { + $parameters = [ + 'select' => 'id,dateparution,typeavis_lib,familleavis_lib,commercant,ville,cp', + 'order_by' => 'id desc', + 'limit' => 50, + ]; + + $where = []; + if (!empty($this->getInput('departement'))) { + $where[] = 'numerodepartement="' . $this->getInput('departement') . '"'; + } + + if (!empty($this->getInput('famille'))) { + $where[] = 'familleavis="' . $this->getInput('famille') . '"'; + } + + if (!empty($this->getInput('type'))) { + $where[] = 'typeavis="' . $this->getInput('type') . '"'; + } + + if ($where !== []) { + $parameters['where'] = implode(' and ', $where); + } + + $url = urljoin(self::URI, '/api/explore/v2.1/catalog/datasets/annonces-commerciales/records?' . http_build_query($parameters)); + + $data = Json::decode(getContents($url), false); + + foreach ($data->results as $result) { + if ( + !isset( + $result->id, + $result->dateparution, + $result->typeavis_lib, + $result->familleavis_lib, + $result->commercant, + $result->ville, + $result->cp + ) + ) { + continue; + } + + $title = sprintf( + '[%s] %s - %s à %s (%s)', + $result->typeavis_lib, + $result->familleavis_lib, + $result->commercant, + $result->ville, + $result->cp + ); + + $this->items[] = [ + 'uid' => $result->id, + 'timestamp' => strtotime($result->dateparution), + 'title' => $title, + ]; + } + } +} From db85015daa62b26cf1ea2aab4d94e992b40a040f Mon Sep 17 00:00:00 2001 From: "Quentin B." <quent1-fr@users.noreply.github.com> Date: Thu, 8 Aug 2024 01:20:42 +0200 Subject: [PATCH 156/423] [AnfrBridge] Add bridge (#4191) * [AnfrBridge] Add bridge * yup --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/AnfrBridge.php | 278 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 bridges/AnfrBridge.php diff --git a/bridges/AnfrBridge.php b/bridges/AnfrBridge.php new file mode 100644 index 00000000..391fde77 --- /dev/null +++ b/bridges/AnfrBridge.php @@ -0,0 +1,278 @@ +<?php + +class AnfrBridge extends BridgeAbstract +{ + const NAME = 'ANFR'; + const URI = 'https://data.anfr.fr/'; + const DESCRIPTION = 'Fetches data from the French administration "Agence Nationale des Fréquences".'; + const CACHE_TIMEOUT = 604800; // 7d + const MAINTAINER = 'quent1'; + const PARAMETERS = [ + 'Données sur les réseaux mobiles' => [ + 'departement' => [ + 'name' => 'Département', + 'type' => 'list', + 'values' => [ + 'Tous' => null, + 'Ain' => '001', + 'Aisne' => '002', + 'Allier' => '003', + 'Alpes-de-Haute-Provence' => '004', + 'Hautes-Alpes' => '005', + 'Alpes-Maritimes' => '006', + 'Ardèche' => '007', + 'Ardennes' => '008', + 'Ariège' => '009', + 'Aube' => '010', + 'Aude' => '011', + 'Aveyron' => '012', + 'Bouches-du-Rhône' => '013', + 'Calvados' => '014', + 'Cantal' => '015', + 'Charente' => '016', + 'Charente-Maritime' => '017', + 'Cher' => '018', + 'Corrèze' => '019', + 'Corse-du-Sud' => '02A', + 'Haute-Corse' => '02B', + 'Côte-d\'Or' => '021', + 'Côtes-d\'Armor' => '022', + 'Creuse' => '023', + 'Dordogne' => '024', + 'Doubs' => '025', + 'Drôme' => '026', + 'Eure' => '027', + 'Eure-et-Loir' => '028', + 'Finistère' => '029', + 'Gard' => '030', + 'Haute-Garonne' => '031', + 'Gers' => '032', + 'Gironde' => '033', + 'Hérault' => '034', + 'Ille-et-Vilaine' => '035', + 'Indre' => '036', + 'Indre-et-Loire' => '037', + 'Isère' => '038', + 'Jura' => '039', + 'Landes' => '040', + 'Loir-et-Cher' => '041', + 'Loire' => '042', + 'Haute-Loire' => '043', + 'Loire-Atlantique' => '044', + 'Loiret' => '045', + 'Lot' => '046', + 'Lot-et-Garonne' => '047', + 'Lozère' => '048', + 'Maine-et-Loire' => '049', + 'Manche' => '050', + 'Marne' => '051', + 'Haute-Marne' => '052', + 'Mayenne' => '053', + 'Meurthe-et-Moselle' => '054', + 'Meuse' => '055', + 'Morbihan' => '056', + 'Moselle' => '057', + 'Nièvre' => '058', + 'Nord' => '059', + 'Oise' => '060', + 'Orne' => '061', + 'Pas-de-Calais' => '062', + 'Puy-de-Dôme' => '063', + 'Pyrénées-Atlantiques' => '064', + 'Hautes-Pyrénées' => '065', + 'Pyrénées-Orientales' => '066', + 'Bas-Rhin' => '067', + 'Haut-Rhin' => '068', + 'Rhône' => '069', + 'Haute-Saône' => '070', + 'Saône-et-Loire' => '071', + 'Sarthe' => '072', + 'Savoie' => '073', + 'Haute-Savoie' => '074', + 'Paris' => '075', + 'Seine-Maritime' => '076', + 'Seine-et-Marne' => '077', + 'Yvelines' => '078', + 'Deux-Sèvres' => '079', + 'Somme' => '080', + 'Tarn' => '081', + 'Tarn-et-Garonne' => '082', + 'Var' => '083', + 'Vaucluse' => '084', + 'Vendée' => '085', + 'Vienne' => '086', + 'Haute-Vienne' => '087', + 'Vosges' => '088', + 'Yonne' => '089', + 'Territoire de Belfort' => '090', + 'Essonne' => '091', + 'Hauts-de-Seine' => '092', + 'Seine-Saint-Denis' => '093', + 'Val-de-Marne' => '094', + 'Val-d\'Oise' => '095', + 'Guadeloupe' => '971', + 'Martinique' => '972', + 'Guyane' => '973', + 'La Réunion' => '974', + 'Saint-Pierre-et-Miquelon' => '975', + 'Mayotte' => '976', + 'Saint-Barthélemy' => '977', + 'Saint-Martin' => '978', + 'Terres australes et antarctiques françaises' => '984', + 'Wallis-et-Futuna' => '986', + 'Polynésie française' => '987', + 'Nouvelle-Calédonie' => '988', + 'Île de Clipperton' => '989' + ] + ], + 'generation' => [ + 'name' => 'Génération', + 'type' => 'list', + 'values' => [ + 'Tous' => null, + '2G' => '2G', + '3G' => '3G', + '4G' => '4G', + '5G' => '5G', + ] + ], + 'operateur' => [ + 'name' => 'Opérateur', + 'type' => 'list', + 'values' => [ + 'Tous' => null, + 'Bouygues Télécom' => 'BOUYGUES TELECOM', + 'Dauphin Télécom' => 'DAUPHIN TELECOM', + 'Digiciel' => 'DIGICEL', + 'Free Caraïbes' => 'FREE CARAIBES', + 'Free Mobile' => 'FREE MOBILE', + 'GLOBALTEL' => 'GLOBALTEL', + 'Office des postes et télécommunications de Nouvelle Calédonie' => 'Gouv Nelle Calédonie (OPT)', + 'Maore Mobile' => 'MAORE MOBILE', + 'ONATi' => 'ONATI', + 'Orange' => 'ORANGE', + 'Outremer Telecom' => 'OUTREMER TELECOM', + 'Vodafone polynésie' => 'PMT/VODAPHONE', + 'SFR' => 'SFR', + 'SPM Télécom' => 'SPM TELECOM', + 'Service des Postes et Télécommunications de Polynésie Française' => 'Gouv Nelle Calédonie (OPT)', + 'SRR' => 'SRR', + 'Station étrangère' => 'Station étrangère', + 'Telco OI' => 'TELCO IO', + 'United Telecommunication Services Caraïbes' => 'UTS Caraibes', + 'Ora Mobile' => 'VITI SAS', + 'Zeop' => 'ZEOP' + ] + ], + 'statut' => [ + 'name' => 'Statut', + 'type' => 'list', + 'values' => [ + 'Tous' => null, + 'En service' => 'En service', + 'Projet approuvé' => 'Projet approuvé', + 'Techniquement opérationnel' => 'Techniquement opérationnel', + ] + ] + ] + ]; + + public function collectData() + { + $urlParts = [ + 'id' => 'observatoire_2g_3g_4g', + 'resource_id' => '88ef0887-6b0f-4d3f-8545-6d64c8f597da', + 'fields' => 'id,adm_lb_nom,sta_nm_dpt,emr_lb_systeme,generation,date_maj,sta_nm_anfr,adr_lb_lieu,adr_lb_add1,adr_lb_add2,adr_lb_add3,adr_nm_cp,statut', + 'rows' => 10000 + ]; + + if (!empty($this->getInput('departement'))) { + $urlParts['refine.sta_nm_dpt'] = urlencode($this->getInput('departement')); + } + + if (!empty($this->getInput('generation'))) { + $urlParts['refine.generation'] = $this->getInput('generation'); + } + + if (!empty($this->getInput('operateur'))) { + // http_build_query() already does urlencoding so this call is redundant + $urlParts['refine.adm_lb_nom'] = urlencode($this->getInput('operateur')); + } + + if (!empty($this->getInput('statut'))) { + $urlParts['refine.statut'] = urlencode($this->getInput('statut')); + } + + // API seems to not play well with urlencoded data + $url = urljoin(static::URI, '/d4c/api/records/1.0/download/?' . urldecode(http_build_query($urlParts))); + + $json = getContents($url); + $data = Json::decode($json, false); + $records = $data->records; + $frequenciesByStation = []; + foreach ($records as $record) { + if (!isset($frequenciesByStation[$record->fields->sta_nm_anfr])) { + $street = sprintf( + '%s %s %s', + $record->fields->adr_lb_add1 ?? '', + $record->fields->adr_lb_add2 ?? '', + $record->fields->adr_lb_add3 ?? '' + ); + $frequenciesByStation[$record->fields->sta_nm_anfr] = [ + 'id' => $record->fields->sta_nm_anfr, + 'operator' => $record->fields->adm_lb_nom, + 'frequencies' => [], + 'lastUpdate' => 0, + 'address' => [ + 'street' => trim($street), + 'postCode' => $record->fields->adr_nm_cp, + 'city' => $record->fields->adr_lb_lieu + ] + ]; + } + + $frequenciesByStation[$record->fields->sta_nm_anfr]['frequencies'][] = [ + 'generation' => $record->fields->generation, + 'frequency' => $record->fields->emr_lb_systeme, + 'status' => $record->fields->statut, + 'updatedAt' => strtotime($record->fields->date_maj), + ]; + + $frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'] = max( + $frequenciesByStation[$record->fields->sta_nm_anfr]['lastUpdate'], + strtotime($record->fields->date_maj) + ); + } + + usort($frequenciesByStation, static fn ($a, $b) => $b['lastUpdate'] <=> $a['lastUpdate']); + + foreach ($frequenciesByStation as $station) { + $title = sprintf( + '[%s] Mise à jour de la station n°%s à %s (%s)', + $station['operator'], + $station['id'], + $station['address']['city'], + $station['address']['postCode'] + ); + + $array_reduce = array_reduce($station['frequencies'], static function ($carry, $frequency) { + return sprintf('%s<li>%s : %s</li>', $carry, $frequency['frequency'], $frequency['status']); + }, ''); + + $content = sprintf( + '<h1>Adresse complète</h1><p>%s<br>%s<br>%s</p><h1>Fréquences</h1><p><ul>%s</ul></p>', + $station['address']['street'], + $station['address']['postCode'], + $station['address']['city'], + $array_reduce + ); + + $this->items[] = [ + 'uid' => $station['id'], + 'timestamp' => $station['lastUpdate'], + 'title' => $title, + 'content' => $content, + ]; + } + } +} \ No newline at end of file From 7073bb2f4674578b396d13b926ce585c9638a305 Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Thu, 8 Aug 2024 05:05:48 +0530 Subject: [PATCH 157/423] [NVIDIADriverBridge] Initial Commit (#4198) * [NVIDIADriverBridge] Initial Commit Fetch the latest NVIDIA Linux driver updates * Update NVIDIADriverBridge.php * refactor * rename --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/NvidiaDriverBridge.php | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 bridges/NvidiaDriverBridge.php diff --git a/bridges/NvidiaDriverBridge.php b/bridges/NvidiaDriverBridge.php new file mode 100644 index 00000000..595411bc --- /dev/null +++ b/bridges/NvidiaDriverBridge.php @@ -0,0 +1,49 @@ +<?php + +class NvidiaDriverBridge extends FeedExpander +{ + const NAME = 'NVIDIA Linux Driver Releases'; + const URI = 'https://www.nvidia.com/Download/processFind.aspx'; + const DESCRIPTION = 'Fetch the latest NVIDIA Linux driver updates'; + const MAINTAINER = 'tillcash'; + const PARAMETERS = [ + [ + 'whql' => [ + 'name' => 'Version', + 'type' => 'list', + 'values' => [ + 'All' => '', + 'Beta' => '0', + 'New Feature Branch' => '5', + 'Recommended/Certified' => '1', + ], + ], + ], + ]; + + public function collectData() + { + $whql = $this->getInput('whql'); + + $parameters = [ + 'lid' => 1, // en-us + 'psid' => 129, // GeForce + 'osid' => 12, // Linux 64-bit + 'whql' => $whql, + ]; + + $url = 'https://www.nvidia.com/Download/processFind.aspx?' . http_build_query($parameters); + $dom = getSimpleHTMLDOM($url); + + foreach ($dom->find('tr#driverList') as $element) { + $id = str_replace('img_', '', $element->find('img', 0)->id); + + $this->items[] = [ + 'timestamp' => $element->find('td.gridItem', 3)->plaintext, + 'title' => sprintf('NVIDIA Linux Driver %s', $element->find('td.gridItem', 2)->plaintext), + 'uri' => 'https://www.nvidia.com/Download/driverResults.aspx/' . $id, + 'content' => $dom->find('tr#tr_' . $id . ' span', 0)->innertext, + ]; + } + } +} From 9973f731dfb0b34ae4c1d3d86143eee346ff5641 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 8 Aug 2024 02:13:04 +0200 Subject: [PATCH 158/423] feat: introduce RateLimitException (#4199) --- actions/DisplayAction.php | 8 ++++++-- bridges/RedditBridge.php | 6 ++++-- bridges/Vk2Bridge.php | 2 +- bridges/VkBridge.php | 2 +- bridges/YoutubeBridge.php | 3 ++- lib/http.php | 10 ++++++++++ 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index c51bb7cd..55f0d2fd 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -121,8 +121,12 @@ class DisplayAction implements ActionInterface $items = $feedItems; } $feed = $bridge->getFeed(); - } catch (\Exception $e) { - // Probably an exception inside a bridge + } catch (\Throwable $e) { + if ($e instanceof RateLimitException) { + // These are internally generated by bridges + $this->logger->info(sprintf('RateLimitException in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); + return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429); + } if ($e instanceof HttpException) { // Reproduce (and log) these responses regardless of error output and report limit if ($e->getCode() === 429) { diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index ef74fdcd..03f279d8 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -93,12 +93,12 @@ class RedditBridge extends BridgeAbstract { $forbiddenKey = 'reddit_forbidden'; if ($this->cache->get($forbiddenKey)) { - throw new HttpException('403 Forbidden', 403); + throw new RateLimitException(); } $rateLimitKey = 'reddit_rate_limit'; if ($this->cache->get($rateLimitKey)) { - throw new HttpException('429 Too Many Requests', 429); + throw new RateLimitException(); } try { @@ -108,8 +108,10 @@ class RedditBridge extends BridgeAbstract // 403 Forbidden // This can possibly mean that reddit has permanently blocked this server's ip address $this->cache->set($forbiddenKey, true, 60 * 61); + throw new RateLimitException(); } elseif ($e->getCode() === 429) { $this->cache->set($rateLimitKey, true, 60 * 61); + throw new RateLimitException(); } throw $e; } diff --git a/bridges/Vk2Bridge.php b/bridges/Vk2Bridge.php index 6fecba84..62ba8e05 100644 --- a/bridges/Vk2Bridge.php +++ b/bridges/Vk2Bridge.php @@ -194,7 +194,7 @@ class Vk2Bridge extends BridgeAbstract public function collectData() { if ($this->cache->get($this->rateLimitCacheKey)) { - throw new HttpException('429 Too Many Requests', 429); + throw new RateLimitException(); } $u = $this->getInput('u'); diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 22957f26..0d62305b 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -519,7 +519,7 @@ class VkBridge extends BridgeAbstract $uri = urljoin(self::URI, $headers['location'][0]); if (str_contains($uri, '/429.html')) { - returnServerError('VK responded "Too many requests"'); + throw new RateLimitException(); } if (!preg_match('#^https?://vk.com/#', $uri)) { diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index af14c856..647b1c42 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -82,13 +82,14 @@ class YoutubeBridge extends BridgeAbstract { $cacheKey = 'youtube_rate_limit'; if ($this->cache->get($cacheKey)) { - throw new HttpException('429 Too Many Requests', 429); + throw new RateLimitException(); } try { $this->collectDataInternal(); } catch (HttpException $e) { if ($e->getCode() === 429) { $this->cache->set($cacheKey, true, 60 * 16); + throw new RateLimitException(); } throw $e; } diff --git a/lib/http.php b/lib/http.php index 39f0c727..0d21b958 100644 --- a/lib/http.php +++ b/lib/http.php @@ -1,5 +1,15 @@ <?php +/** + * Thrown by bridges + */ +final class RateLimitException extends \Exception +{ +} + +/** + * @internal Do not use this class in bridges + */ class HttpException extends \Exception { public ?Response $response; From 2a96bf19b5354ceecb417f266070b9659d08e84a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 8 Aug 2024 02:55:35 +0200 Subject: [PATCH 159/423] fix: bug in prior commit (#4200) --- actions/DisplayAction.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 55f0d2fd..2a184598 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -217,7 +217,7 @@ class DisplayAction implements ActionInterface return $report['count']; } - private static function createGithubIssueUrl(BridgeAbstract $bridge, \Exception $e, string $message): string + private static function createGithubIssueUrl(BridgeAbstract $bridge, \Throwable $e, string $message): string { $maintainer = $bridge->getMaintainer(); if (str_contains($maintainer, ',')) { From 6afd13eb06276f085f245c5075972ef36eb6740a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 8 Aug 2024 03:43:26 +0200 Subject: [PATCH 160/423] refactor: deprecate FeedItem constructor (#4201) * fix: bug in prior commit * refactor: deprecate FeedItem constructor * test: fix --- actions/DisplayAction.php | 27 +++++++------------ bridges/GULPProjekteBridge.php | 16 +++++------ bridges/NintendoBridge.php | 2 +- bridges/RutubeBridge.php | 15 ++++++----- bridges/ScalableCapitalBlogBridge.php | 18 ++++++++----- bridges/Vk2Bridge.php | 12 ++++----- lib/FeedItem.php | 20 +++----------- lib/FormatAbstract.php | 15 +++++------ lib/XPathAbstract.php | 26 +++++++++++------- tests/FeedItemTest.php | 39 ++++----------------------- tests/Formats/BaseFormatTest.php | 2 +- 11 files changed, 78 insertions(+), 114 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 2a184598..a5703e5e 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -93,7 +93,6 @@ class DisplayAction implements ActionInterface private function createResponse(Request $request, BridgeAbstract $bridge, string $format) { $items = []; - $feed = []; try { $bridge->loadConfiguration(); @@ -113,14 +112,6 @@ class DisplayAction implements ActionInterface $bridge->setInput($input); $bridge->collectData(); $items = $bridge->getItems(); - if (isset($items[0]) && is_array($items[0])) { - $feedItems = []; - foreach ($items as $item) { - $feedItems[] = FeedItem::fromArray($item); - } - $items = $feedItems; - } - $feed = $bridge->getFeed(); } catch (\Throwable $e) { if ($e instanceof RateLimitException) { // These are internally generated by bridges @@ -162,7 +153,7 @@ class DisplayAction implements ActionInterface $format = $formatFactory->create($format); $format->setItems($items); - $format->setFeed($feed); + $format->setFeed($bridge->getFeed()); $now = time(); $format->setLastModified($now); $headers = [ @@ -172,19 +163,20 @@ class DisplayAction implements ActionInterface return new Response($format->stringify(), 200, $headers); } - private function createFeedItemFromException($e, BridgeAbstract $bridge): FeedItem + private function createFeedItemFromException($e, BridgeAbstract $bridge): array { - $item = new FeedItem(); + $item = []; // Create a unique identifier every 24 hours $uniqueIdentifier = urlencode((int)(time() / 86400)); $title = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier); - $item->setTitle($title); - $item->setURI(get_current_url()); - $item->setTimestamp(time()); + + $item['title'] = $title; + $item['uri'] = get_current_url(); + $item['timestamp'] = time(); // Create an item identifier for feed readers e.g. "staysafetv twitch videos_19389" - $item->setUid($bridge->getName() . '_' . $uniqueIdentifier); + $item['uid'] = $bridge->getName() . '_' . $uniqueIdentifier; $content = render_template(__DIR__ . '/../templates/bridge-error.html.php', [ 'error' => render_template(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), @@ -192,7 +184,8 @@ class DisplayAction implements ActionInterface 'issueUrl' => self::createGithubIssueUrl($bridge, $e, create_sane_exception_message($e)), 'maintainer' => $bridge->getMaintainer(), ]); - $item->setContent($content); + $item['content'] = $content; + return $item; } diff --git a/bridges/GULPProjekteBridge.php b/bridges/GULPProjekteBridge.php index e0bb8cbe..05689bc9 100644 --- a/bridges/GULPProjekteBridge.php +++ b/bridges/GULPProjekteBridge.php @@ -129,24 +129,24 @@ class GULPProjekteBridge extends WebDriverAbstract while (true) { $items = $this->getDriver()->findElements(WebDriverBy::tagName('app-project-view')); foreach ($items as $item) { - $feedItem = new FeedItem(); + $feedItem = []; $heading = $item->findElement(WebDriverBy::xpath('.//app-heading-tag/h1/a')); - $feedItem->setTitle($heading->getText()); - $feedItem->setURI('https://www.gulp.de' . $heading->getAttribute('href')); + $feedItem['title'] = $heading->getText(); + $feedItem['uri'] = 'https://www.gulp.de' . $heading->getAttribute('href'); $info = $item->findElement(WebDriverBy::tagName('app-icon-info-list')); if ($logo = $this->getLogo($item)) { - $feedItem->setEnclosures([$logo]); + $feedItem['enclosures'] = [$logo]; } if (str_contains($info->getText(), 'Projektanbieter:')) { - $feedItem->setAuthor($info->findElement(WebDriverBy::xpath('.//li/span[2]/span'))->getText()); + $feedItem['author'] = $info->findElement(WebDriverBy::xpath('.//li/span[2]/span'))->getText(); } else { // mostly "Direkt vom Auftraggeber" or "GULP Agentur" - $feedItem->setAuthor($item->findElement(WebDriverBy::tagName('b'))->getText()); + $feedItem['author'] = $item->findElement(WebDriverBy::tagName('b'))->getText(); } - $feedItem->setContent($item->findElement(WebDriverBy::xpath('.//p[@class="description"]'))->getText()); + $feedItem['content'] = $item->findElement(WebDriverBy::xpath('.//p[@class="description"]'))->getText(); $timeAgo = $item->findElement(WebDriverBy::xpath('.//small[contains(@class, "time-ago")]'))->getText(); - $feedItem->setTimestamp($this->getTimestamp($timeAgo)); + $feedItem['timestamp'] = $this->getTimestamp($timeAgo); $this->items[] = $feedItem; } diff --git a/bridges/NintendoBridge.php b/bridges/NintendoBridge.php index 1c4ecf2b..2f6113b2 100644 --- a/bridges/NintendoBridge.php +++ b/bridges/NintendoBridge.php @@ -477,7 +477,7 @@ class NintendoBridge extends XPathAbstract return $date->getTimestamp(); } - protected function generateItemId(FeedItem $item) + protected function generateItemId(array $item) { return $this->getCurrentCategory() . '-' . $this->lastId; } diff --git a/bridges/RutubeBridge.php b/bridges/RutubeBridge.php index 39577575..f8e106c8 100644 --- a/bridges/RutubeBridge.php +++ b/bridges/RutubeBridge.php @@ -107,10 +107,10 @@ class RutubeBridge extends BridgeAbstract } foreach ($videos as $video) { - $item = new FeedItem(); - $item->setTitle($video->title); - $item->setURI($video->video_url); - $content = '<a href="' . $item->getURI() . '">'; + $item = []; + $item['title'] = $video->title; + $item['uri'] = $video->video_url; + $content = '<a href="' . $video->video_url . '">'; $content .= '<img src="' . $video->thumbnail_url . '" />'; $content .= '</a><br/>'; $content .= nl2br( @@ -122,9 +122,10 @@ class RutubeBridge extends BridgeAbstract $video->description . ' ' ) ); - $item->setTimestamp($video->created_ts); - $item->setAuthor($video->author->name); - $item->setContent($content); + $item['timestamp'] = $video->created_ts; + $item['author'] = $video->author->name; + $item['content'] = $content; + $this->items[] = $item; } } diff --git a/bridges/ScalableCapitalBlogBridge.php b/bridges/ScalableCapitalBlogBridge.php index 6f95efb3..d95431c6 100644 --- a/bridges/ScalableCapitalBlogBridge.php +++ b/bridges/ScalableCapitalBlogBridge.php @@ -41,16 +41,20 @@ class ScalableCapitalBlogBridge extends WebDriverAbstract $items = $this->getDriver()->findElements(WebDriverBy::xpath('//div[contains(@class, "articles")]//div[@class="items"]//div[contains(@class, "item")]')); foreach ($items as $item) { - $feedItem = new FeedItem(); + $feedItem = []; + + $feedItem['enclosures'] = ['https://de.scalable.capital' . $item->findElement(WebDriverBy::tagName('img'))->getAttribute('src')]; - $feedItem->setEnclosures(['https://de.scalable.capital' . $item->findElement(WebDriverBy::tagName('img'))->getAttribute('src')]); $heading = $item->findElement(WebDriverBy::tagName('a')); - $feedItem->setTitle($heading->getText()); - $feedItem->setURI('https://de.scalable.capital' . $heading->getAttribute('href')); - $feedItem->setContent($item->findElement(WebDriverBy::xpath('.//div[@class="summary"]'))->getText()); + $feedItem['title'] = $heading->getText(); + + $feedItem['uri'] = 'https://de.scalable.capital' . $heading->getAttribute('href'); + $feedItem['content'] = $item->findElement(WebDriverBy::xpath('.//div[@class="summary"]'))->getText(); + $date = $item->findElement(WebDriverBy::xpath('.//div[@class="published-date"]'))->getText(); - $feedItem->setTimestamp($this->formatItemTimestamp($date)); - $feedItem->setAuthor($item->findElement(WebDriverBy::xpath('.//div[@class="author"]'))->getText()); + $feedItem['timestamp'] = $this->formatItemTimestamp($date); + + $feedItem['author'] = $item->findElement(WebDriverBy::xpath('.//div[@class="author"]'))->getText(); $this->items[] = $feedItem; } diff --git a/bridges/Vk2Bridge.php b/bridges/Vk2Bridge.php index 62ba8e05..f52850ce 100644 --- a/bridges/Vk2Bridge.php +++ b/bridges/Vk2Bridge.php @@ -255,7 +255,7 @@ class Vk2Bridge extends BridgeAbstract if (!$ownerId) { $ownerId = $post['owner_id']; } - $item = new FeedItem(); + $item = []; $content = $this->generateContentFromPost($post); if (isset($post['copy_history'])) { if ($this->getInput('hide_reposts')) { @@ -277,11 +277,11 @@ class Vk2Bridge extends BridgeAbstract $content .= '):</p>'; $content .= $this->generateContentFromPost($originalPost); } - $item->setContent($content); - $item->setTimestamp($post['date']); - $item->setAuthor($this->ownerNames[$post['from_id']]); - $item->setTitle($this->getTitle(strip_tags($content))); - $item->setURI($this->getPostURI($post)); + $item['content'] = $content; + $item['timestamp'] = $post['date']; + $item['author'] = $this->ownerNames[$post['from_id']]; + $item['title'] = $this->getTitle(strip_tags($content)); + $item['uri'] = $this->getPostURI($post); $this->items[] = $item; } diff --git a/lib/FeedItem.php b/lib/FeedItem.php index 8a092a27..bca06c23 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -12,10 +12,6 @@ class FeedItem protected ?string $uid = null; protected array $misc = []; - public function __construct() - { - } - public static function fromArray(array $itemArray): self { $item = new self(); @@ -25,6 +21,10 @@ class FeedItem return $item; } + private function __construct() + { + } + public function __set($name, $value) { switch ($name) { @@ -89,18 +89,6 @@ class FeedItem return $this->uri; } - /** - * Set URI to the full article. - * - * Use {@see FeedItem::getURI()} to get the URI. - * - * _Note_: Removes whitespace from the beginning and end of the URI. - * - * _Remarks_: Uses the attribute "href" or "src" if the provided URI is an - * object of simple_html_dom_node. - * - * @param simple_html_dom_node|object|string $uri URI to the full article. - */ public function setURI($uri) { $this->uri = null; // Clear previous data diff --git a/lib/FormatAbstract.php b/lib/FormatAbstract.php index 28eb4bbf..9cba0d8c 100644 --- a/lib/FormatAbstract.php +++ b/lib/FormatAbstract.php @@ -6,11 +6,11 @@ abstract class FormatAbstract const MIME_TYPE = 'text/plain'; - protected string $charset = 'UTF-8'; - protected array $items = []; - protected int $lastModified; - protected array $feed = []; + protected array $items = []; + protected string $charset = 'UTF-8'; + + protected int $lastModified; abstract public function stringify(); @@ -30,12 +30,11 @@ abstract class FormatAbstract return $this->feed; } - /** - * @param FeedItem[] $items - */ public function setItems(array $items): void { - $this->items = $items; + foreach ($items as $item) { + $this->items[] = FeedItem::fromArray($item); + } } /** diff --git a/lib/XPathAbstract.php b/lib/XPathAbstract.php index 6163ca13..44cbab67 100644 --- a/lib/XPathAbstract.php +++ b/lib/XPathAbstract.php @@ -422,9 +422,18 @@ abstract class XPathAbstract extends BridgeAbstract } foreach ($entries as $entry) { - $item = new FeedItem(); - foreach (['title', 'content', 'uri', 'author', 'timestamp', 'enclosures', 'categories'] as $param) { - $expression = $this->getParam($param); + $item = []; + $parameters = [ + 'title', + 'content', + 'uri', + 'author', + 'timestamp', + 'enclosures', + 'categories', + ]; + foreach ($parameters as $parameter) { + $expression = $this->getParam($parameter); if ('' === $expression) { continue; } @@ -438,21 +447,21 @@ abstract class XPathAbstract extends BridgeAbstract continue; } - if ('categories' === $param && $typedResult instanceof \DOMNodeList) { + if ('categories' === $parameter && $typedResult instanceof \DOMNodeList) { $value = []; foreach ($typedResult as $domNode) { $value[] = $this->getItemValueOrNodeValue($domNode, false); } } else { - $value = $this->getItemValueOrNodeValue($typedResult, 'content' === $param); + $value = $this->getItemValueOrNodeValue($typedResult, 'content' === $parameter); } - $item->__set($param, $this->formatParamValue($param, $value)); + $item[$parameter] = $this->formatParamValue($parameter, $value); } $itemId = $this->generateItemId($item); if (null !== $itemId) { - $item->setUid($itemId); + $item['uid'] = $itemId; } $this->items[] = $item; @@ -646,10 +655,9 @@ abstract class XPathAbstract extends BridgeAbstract /** * Allows overriding default mechanism determining items Uid's * - * @param FeedItem $item * @return string|null */ - protected function generateItemId(FeedItem $item) + protected function generateItemId(array $item) { return null; } diff --git a/tests/FeedItemTest.php b/tests/FeedItemTest.php index 3390e7b3..fbc7a4ea 100644 --- a/tests/FeedItemTest.php +++ b/tests/FeedItemTest.php @@ -4,45 +4,16 @@ declare(strict_types=1); namespace RssBridge\Tests; -use FeedItem; use PHPUnit\Framework\TestCase; class FeedItemTest extends TestCase { public function test() { - $item = new FeedItem(); - $item->setTitle('hello'); - $this->assertSame('hello', $item->getTitle()); - - $item = FeedItem::fromArray(['title' => 'hello2']); - $this->assertSame('hello2', $item->getTitle()); - - $item = new FeedItem(); - $item->setAuthor('123'); - $this->assertSame('123', $item->getAuthor()); - - $item = new FeedItem(); - $item->title = 'aa'; - $this->assertSame('aa', $item->title); - $this->assertSame('aa', $item->getTitle()); - } - - public function testTimestamp() - { - $item = new FeedItem(); - $item->setTimestamp(5); - $this->assertSame(5, $item->getTimestamp()); - - $item->setTimestamp('5'); - $this->assertSame(5, $item->getTimestamp()); - - $item->setTimestamp('1970-01-01 18:00:00'); - $this->assertSame(64800, $item->getTimestamp()); - - $item->setTimestamp('1st jan last year'); - - // This will fail at 2025-01-01 hehe - $this->assertSame(1672531200, $item->getTimestamp()); + $item = [ + 'title' => 'kek', + ]; + $feedItem = \FeedItem::fromArray($item); + $this->assertSame('kek', $feedItem->getTitle()); } } diff --git a/tests/Formats/BaseFormatTest.php b/tests/Formats/BaseFormatTest.php index 8999e772..0907b72a 100644 --- a/tests/Formats/BaseFormatTest.php +++ b/tests/Formats/BaseFormatTest.php @@ -39,7 +39,7 @@ abstract class BaseFormatTest extends TestCase $items = []; foreach ($data['items'] as $item) { - $items[] = \FeedItem::fromArray($item); + $items[] = ($item); } return (object)[ From 2acd415475e0d1f4b621003a85ece6e47a3790f7 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 8 Aug 2024 04:31:47 +0200 Subject: [PATCH 161/423] refactor: drop usage of Debug::log (#4202) * refactor: drop usage of Debug::log * lint --- bridges/AsahiShimbunAJWBridge.php | 1 - bridges/DagensNyheterDirektBridge.php | 5 --- bridges/FicbookBridge.php | 1 - bridges/FolhaDeSaoPauloBridge.php | 4 --- bridges/HackerNewsUserThreadsBridge.php | 2 -- bridges/ItakuBridge.php | 3 +- bridges/JustETFBridge.php | 8 ----- bridges/MastodonBridge.php | 12 ++++--- bridges/PanacheDigitalGamesBridge.php | 2 +- bridges/PresidenciaPTBridge.php | 1 - bridges/SlusheBridge.php | 8 +---- bridges/ThreadsBridge.php | 5 ++- bridges/TwitterBridge.php | 2 +- bridges/TwitterV2Bridge.php | 28 ++-------------- bridges/UnraidCommunityApplicationsBridge.php | 2 -- bridges/XenForoBridge.php | 2 -- lib/BridgeCard.php | 4 +-- lib/Debug.php | 16 --------- lib/FeedItem.php | 33 ++++++++++--------- lib/RssBridge.php | 3 +- 20 files changed, 37 insertions(+), 105 deletions(-) diff --git a/bridges/AsahiShimbunAJWBridge.php b/bridges/AsahiShimbunAJWBridge.php index 03bee6ba..873eb351 100644 --- a/bridges/AsahiShimbunAJWBridge.php +++ b/bridges/AsahiShimbunAJWBridge.php @@ -45,7 +45,6 @@ class AsahiShimbunAJWBridge extends BridgeAbstract foreach ($html->find('#MainInner li a') as $element) { if ($element->parent()->class == 'HeadlineTopImage-S') { - Debug::log('Skip Headline, it is repeated below'); continue; } $item = []; diff --git a/bridges/DagensNyheterDirektBridge.php b/bridges/DagensNyheterDirektBridge.php index 4d1629fb..f0748b76 100644 --- a/bridges/DagensNyheterDirektBridge.php +++ b/bridges/DagensNyheterDirektBridge.php @@ -27,11 +27,6 @@ class DagensNyheterDirektBridge extends BridgeAbstract $url = self::BASEURL . $link; $title = $element->find('h2', 0)->plaintext; $author = $element->find('div.ds-byline__titles', 0)->plaintext; - // Debug::log($link); - // Debug::log($datetime); - // Debug::log($title); - // Debug::log($url); - // Debug::log($author); $article_content = $element->find('div.direkt-post__content', 0); $article_html = ''; diff --git a/bridges/FicbookBridge.php b/bridges/FicbookBridge.php index d11015ad..3aa7d998 100644 --- a/bridges/FicbookBridge.php +++ b/bridges/FicbookBridge.php @@ -187,7 +187,6 @@ class FicbookBridge extends BridgeAbstract $fixed_date = str_replace(' г.', '', $fixed_date); if ($fixed_date === $date) { - Debug::log('Unable to fix date: ' . $date); return null; } diff --git a/bridges/FolhaDeSaoPauloBridge.php b/bridges/FolhaDeSaoPauloBridge.php index dba86c52..27713a41 100644 --- a/bridges/FolhaDeSaoPauloBridge.php +++ b/bridges/FolhaDeSaoPauloBridge.php @@ -44,8 +44,6 @@ class FolhaDeSaoPauloBridge extends FeedExpander $item['content'] = $text; $item['uri'] = explode('*', $item['uri'])[1]; } - } else { - Debug::log('???: ' . $item['uri']); } } else { $item['uri'] = explode('*', $item['uri'])[1]; @@ -58,13 +56,11 @@ class FolhaDeSaoPauloBridge extends FeedExpander { $feed_input = $this->getInput('feed'); if (substr($feed_input, 0, strlen(self::URI)) === self::URI) { - Debug::log('Input:: ' . $feed_input); $feed_url = $feed_input; } else { /* TODO: prepend `/` if missing */ $feed_url = self::URI . '/' . $this->getInput('feed'); } - Debug::log('URL: ' . $feed_url); $limit = $this->getInput('amount'); $this->collectExpandableDatas($feed_url, $limit); } diff --git a/bridges/HackerNewsUserThreadsBridge.php b/bridges/HackerNewsUserThreadsBridge.php index fee96b61..0ab7445d 100644 --- a/bridges/HackerNewsUserThreadsBridge.php +++ b/bridges/HackerNewsUserThreadsBridge.php @@ -21,8 +21,6 @@ class HackerNewsUserThreadsBridge extends BridgeAbstract { $url = 'https://news.ycombinator.com/threads?id=' . $this->getInput('user'); $html = getSimpleHTMLDOM($url); - Debug::log('queried ' . $url); - Debug::log('found ' . $html); $item = []; $articles = $html->find('tr[class*="comtr"]'); diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index 506805f7..e6ecbdc8 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -522,7 +522,7 @@ class ItakuBridge extends BridgeAbstract { $url = self::URI . '/api/commissions/' . $id . '/?format=json'; $uri = self::URI . '/commissions/' . $id; - // Debug::log(var_dump($metadata)); + $data = $metadata ?? $this->getData($url, true, true) or returnServerError("Could not load $url"); @@ -664,7 +664,6 @@ class ItakuBridge extends BridgeAbstract private function getData(string $url, bool $cache = false, bool $getJSON = false, array $httpHeaders = [], array $curlOptions = []) { - // Debug::log($url); if ($getJSON) { //get JSON object if ($cache) { $data = $this->loadCacheValue($url); diff --git a/bridges/JustETFBridge.php b/bridges/JustETFBridge.php index 88920133..bcefe331 100644 --- a/bridges/JustETFBridge.php +++ b/bridges/JustETFBridge.php @@ -138,8 +138,6 @@ class JustETFBridge extends BridgeAbstract date_time_set($df, 0, 0); - // Debug::log(date_format($df, 'U')); - return date_format($df, 'U'); } @@ -216,8 +214,6 @@ class JustETFBridge extends BridgeAbstract $element = $article->find('div.subheadline', 0) or returnServerError('Date not found!'); - // Debug::log($element->plaintext); - $date = trim(explode('|', $element->plaintext)[0]); return $this->fixDate($date); @@ -230,8 +226,6 @@ class JustETFBridge extends BridgeAbstract $element->find('a', 0)->onclick = ''; - // Debug::log($element->innertext); - return $element->innertext; } @@ -300,8 +294,6 @@ class JustETFBridge extends BridgeAbstract $element = $html->find('div.infobox div.vallabel', 0) or returnServerError('Date not found!'); - // Debug::log($element->plaintext); - $date = trim(explode("\r\n", $element->plaintext)[1]); return $this->fixDate($date); diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index e673bf14..b98b1135 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -275,11 +275,13 @@ class MastodonBridge extends BridgeAbstract $toSign = '(request-target): get ' . $matches[2] . "\nhost: " . $matches[1] . "\ndate: " . $date; $result = openssl_sign($toSign, $signature, $pkey, 'RSA-SHA256'); if ($result) { - Debug::log($toSign); - $sig = 'Signature: keyId="' . $keyId . '",headers="(request-target) host date",signature="' . - base64_encode($signature) . '"'; - Debug::log($sig); - array_push($headers, $sig); + $sig = sprintf( + 'Signature: keyId="%s",headers="(request-target) host date",signature="%s"', + $keyId, + base64_encode($signature) + ); + + $headers[] = $sig; } } try { diff --git a/bridges/PanacheDigitalGamesBridge.php b/bridges/PanacheDigitalGamesBridge.php index 6f7d8994..bb0c00b9 100644 --- a/bridges/PanacheDigitalGamesBridge.php +++ b/bridges/PanacheDigitalGamesBridge.php @@ -38,7 +38,7 @@ class PanacheDigitalGamesBridge extends BridgeAbstract $image_html = $element->find('.news-item-thumbnail-image', 0); if ($image_html) { $image_strings = explode('\'', $image_html); - /* Debug::log('S: ' . count($image_strings) . '||' . implode('_ _', $image_strings)); */ + if (count($image_strings) == 4) { $item['content'] = '<img src="' . $image_strings[1] . '" />'; } diff --git a/bridges/PresidenciaPTBridge.php b/bridges/PresidenciaPTBridge.php index 247e8fce..2f55f262 100644 --- a/bridges/PresidenciaPTBridge.php +++ b/bridges/PresidenciaPTBridge.php @@ -55,7 +55,6 @@ class PresidenciaPTBridge extends BridgeAbstract $contexts = $this->getParameters(); foreach (array_keys($contexts['Section']) as $k) { - Debug::log('Key: ' . var_export($k, true)); if ($this->getInput($k)) { $html = getSimpleHTMLDOMCached($this->getURI() . $k); diff --git a/bridges/SlusheBridge.php b/bridges/SlusheBridge.php index 12bed13a..05be8d4f 100644 --- a/bridges/SlusheBridge.php +++ b/bridges/SlusheBridge.php @@ -118,13 +118,8 @@ class SlusheBridge extends BridgeAbstract $html = getSimpleHTMLDOM($uri, $headers); - //Debug::log($html); - //Debug::log($html->find('div.blog-item')[0]); - //Loop on each entry foreach ($html->find('div.blog-item') as $element) { - //Debug::log($element); - $title = $element->find('h3.title', 0)->first_child()->innertext; $article_uri = $element->find('h3.title', 0)->first_child()->href; $timestamp = $element->find('div.publication-date', 0)->innertext; @@ -153,7 +148,6 @@ class SlusheBridge extends BridgeAbstract // Add image thumbnail(s) foreach ($media_uris->find('img') as $media_uri) { $media_html .= '<a href="' . $article_uri . '">' . $media_uri . '</a>'; - //Debug::log('Adding to enclosures: ' . str_replace(' ', '%20', $media_uri->src)); $item['enclosures'][] = str_replace(' ', '%20', $media_uri->src); } } @@ -165,7 +159,7 @@ class SlusheBridge extends BridgeAbstract foreach ($media_uris->find('img') as $media_uri) { $media_html .= '<p>Video:</p><a href="' . $article_uri . '">' . $media_uri . '</a>'; - //Debug::log('Adding to enclosures: ' . $media_uri->src); + $item['enclosures'][] = $media_uri->src; } } diff --git a/bridges/ThreadsBridge.php b/bridges/ThreadsBridge.php index b7e5cd1a..cfcbba0e 100644 --- a/bridges/ThreadsBridge.php +++ b/bridges/ThreadsBridge.php @@ -70,9 +70,9 @@ class ThreadsBridge extends BridgeAbstract public function collectData() { $html = getSimpleHTMLDOMCached($this->getURI(), static::CACHE_TIMEOUT); - Debug::log(sprintf('Fetched: %s', $this->getURI())); + $jsonBlobs = $html->find('script[type="application/json"]'); - Debug::log(sprintf('%d JSON blobs found.', count($jsonBlobs))); + $gatheredCodes = []; $limit = $this->getInput('limit'); foreach ($jsonBlobs as $jsonBlob) { @@ -87,7 +87,6 @@ class ThreadsBridge extends BridgeAbstract } } } - Debug::log(sprintf('Candidate codes found in JSON in script tags: %s', print_r($gatheredCodes, true))); $this->feedName = html_entity_decode($html->find('meta[property=og:title]', 0)->content); // todo: meta[property=og:description] could populate the feed description diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 93301038..800fd63c 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -541,7 +541,7 @@ EOD; } break; default: - Debug::log('Missing support for media type: ' . $media->type); + break; } } } diff --git a/bridges/TwitterV2Bridge.php b/bridges/TwitterV2Bridge.php index 07af8301..19b27137 100644 --- a/bridges/TwitterV2Bridge.php +++ b/bridges/TwitterV2Bridge.php @@ -192,7 +192,6 @@ EOD . $this->getInput('u'), $authHeaders, $params); if (isset($user->errors)) { - Debug::log('User JSON: ' . json_encode($user)); returnServerError('Requested username can\'t be found.'); } @@ -266,7 +265,6 @@ EOD (isset($data->errors) && !isset($data->data)) || (isset($data->meta) && $data->meta->result_count === 0) ) { - Debug::log('Data JSON: ' . json_encode($data)); switch ($this->queriedContext) { case 'By keyword or hashtag': returnServerError('No results for this query.'); @@ -311,7 +309,6 @@ EOD foreach ($includesTweets as $includesTweet) { $includesTweetsIds[] = $includesTweet->id; } - Debug::log('includesTweetsIds: ' . join(',', $includesTweetsIds)); // Set default params for API query $params = [ @@ -336,8 +333,6 @@ EOD // Create output array with all required elements for each tweet foreach ($tweets as $tweet) { - //Debug::log('Tweet JSON: ' . json_encode($tweet)); - // Skip pinned tweet (if selected) if ($hidePinned && $tweet->id === $pinnedTweetId) { continue; @@ -376,12 +371,10 @@ EOD $cleanedQuotedTweet = null; $quotedUser = null; if ($isQuote) { - Debug::log('Tweet is quote'); foreach ($includesTweets as $includesTweet) { if ($includesTweet->id === $tweet->referenced_tweets[0]->id) { $quotedTweet = $includesTweet; $cleanedQuotedTweet = nl2br($quotedTweet->text); - //Debug::log('Found quoted tweet'); break; } } @@ -389,7 +382,6 @@ EOD $quotedUser = $this->getTweetUser($quotedTweet, $retweetedUsers, $includesUsers); } if ($isRetweet || is_null($user)) { - Debug::log('Tweet is retweet, or $user is null'); // Replace tweet object with original retweeted object if ($isRetweet) { foreach ($includesTweets as $includesTweet) { @@ -430,7 +422,6 @@ EOD . $this->item['username'] . ')'; $cleanedTweet = nl2br($tweet->text); - //Debug::log('cleanedTweet: ' . $cleanedTweet); // Perform optional keyword filtering (only keep tweet if keyword is found) if (! empty($tweetFilter)) { @@ -452,7 +443,6 @@ EOD // Search for and replace URLs in Tweet text $cleanedTweet = $this->replaceTweetURLs($tweet, $cleanedTweet); if (isset($cleanedQuotedTweet)) { - Debug::log('Replacing URLs in Quoted Tweet text'); $cleanedQuotedTweet = $this->replaceTweetURLs($quotedTweet, $cleanedQuotedTweet); } @@ -478,9 +468,7 @@ EOD // Get external link info $extURL = null; if (isset($tweet->entities->urls) && strpos($tweet->entities->urls[0]->expanded_url, 'twitter.com') === false) { - Debug::log('Found an external link!'); $extURL = $tweet->entities->urls[0]->expanded_url; - Debug::log($extURL); $extDisplayURL = $tweet->entities->urls[0]->display_url; $extTitle = $tweet->entities->urls[0]->title; $extDesc = $tweet->entities->urls[0]->description; @@ -513,15 +501,12 @@ EOD; $ext_media_html = ''; if (!$hideImages) { if (isset($tweet->attachments->media_keys)) { - Debug::log('Generating HTML for tweet media'); $media_html = $this->createTweetMediaHTML($tweet, $includesMedia, $retweetedMedia); } if (isset($quotedTweet->attachments->media_keys)) { - Debug::log('Generating HTML for quoted tweet media'); $quoted_media_html = $this->createTweetMediaHTML($quotedTweet, $includesMedia, $retweetedMedia); } if (isset($extURL)) { - Debug::log('Generating HTML for external link media'); if ($this->getInput('noimgscaling')) { $extMediaURL = $extMediaOrig; } else { @@ -562,7 +547,6 @@ QUOTE; // Add External Link HTML, if relevant if (isset($extURL) && !$this->getInput('noexternallink')) { - Debug::log('Adding HTML for external link'); $ext_html = <<<EXTERNAL <div style="display: table; border-style: solid; border-width: 1px; border-radius: 5px; padding: 5px;"> $ext_media_html<br> @@ -653,21 +637,18 @@ EXTERNAL; { $originalUser = new stdClass(); // make the linters stop complaining if (isset($retweetedUsers)) { - Debug::log('Searching for tweet author_id in $retweetedUsers'); foreach ($retweetedUsers as $retweetedUser) { if ($retweetedUser->id === $tweetObject->author_id) { $matchedUser = $retweetedUser; - Debug::log('Found author_id match in $retweetedUsers'); break; } } } if (!isset($matchedUser->username) && isset($includesUsers)) { - Debug::log('Searching for tweet author_id in $includesUsers'); foreach ($includesUsers as $includesUser) { if ($includesUser->id === $tweetObject->author_id) { $matchedUser = $includesUser; - Debug::log('Found author_id match in $includesUsers'); + break; } } @@ -689,7 +670,6 @@ EXTERNAL; $tweetMedia = []; // Start by checking the original list of tweet Media includes if (isset($includesMedia)) { - Debug::log('Searching for media_key in $includesMedia'); foreach ($includesMedia as $includesMedium) { if ( in_array( @@ -697,14 +677,12 @@ EXTERNAL; $tweetObject->attachments->media_keys ) ) { - Debug::log('Found media_key in $includesMedia'); $tweetMedia[] = $includesMedium; } } } // If no matches found, check the retweet Media includes if (empty($tweetMedia) && isset($retweetedMedia)) { - Debug::log('Searching for media_key in $retweetedMedia'); foreach ($retweetedMedia as $retweetedMedium) { if ( in_array( @@ -712,7 +690,6 @@ EXTERNAL; $tweetObject->attachments->media_keys ) ) { - Debug::log('Found media_key in $retweetedMedia'); $tweetMedia[] = $retweetedMedium; } } @@ -760,8 +737,7 @@ EOD; EOD; break; default: - Debug::log('Missing support for media type: ' - . $media->type); + break; } } diff --git a/bridges/UnraidCommunityApplicationsBridge.php b/bridges/UnraidCommunityApplicationsBridge.php index 441edb65..1295e827 100644 --- a/bridges/UnraidCommunityApplicationsBridge.php +++ b/bridges/UnraidCommunityApplicationsBridge.php @@ -14,14 +14,12 @@ class UnraidCommunityApplicationsBridge extends BridgeAbstract private function fetchApps() { - Debug::log('Fetching all applications/plugins'); $this->apps = getContents(self::APPSURI); $this->apps = json_decode($this->apps, true)['applist']; } private function sortApps() { - Debug::log('Sorting applications/plugins'); usort($this->apps, function ($app1, $app2) { return $app1['FirstSeen'] < $app2['FirstSeen'] ? 1 : -1; }); diff --git a/bridges/XenForoBridge.php b/bridges/XenForoBridge.php index 1ecb1d74..d1ecea74 100644 --- a/bridges/XenForoBridge.php +++ b/bridges/XenForoBridge.php @@ -436,8 +436,6 @@ class XenForoBridge extends BridgeAbstract break; } - // Debug::log(date_format($df, 'U')); - return date_format($df, 'U'); } } diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index d15ac865..27285558 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -192,7 +192,7 @@ final class BridgeCard { $required = $entry['required'] ?? null; if ($required) { - Debug::log('The "required" attribute is not supported for lists.'); + trigger_error('The required attribute is not supported for lists'); unset($entry['required']); } @@ -235,7 +235,7 @@ final class BridgeCard { $required = $entry['required'] ?? null; if ($required) { - Debug::log('The "required" attribute is not supported for checkboxes.'); + trigger_error('The required attribute is not supported for checkboxes'); unset($entry['required']); } diff --git a/lib/Debug.php b/lib/Debug.php index ba9e787e..630fd8ec 100644 --- a/lib/Debug.php +++ b/lib/Debug.php @@ -15,20 +15,4 @@ class Debug } return false; } - - /** - * @deprecated Use $this->logger->debug() - */ - public static function log($message) - { - $e = new \Exception(); - $trace = trace_from_exception($e); - // Drop the current frame - array_pop($trace); - $lastFrame = $trace[array_key_last($trace)]; - $text = sprintf('%s(%s): %s', $lastFrame['file'], $lastFrame['line'], $message); - - $logger = RssBridge::getLogger(); - $logger->debug($text); - } } diff --git a/lib/FeedItem.php b/lib/FeedItem.php index bca06c23..8c9a60b1 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -12,6 +12,8 @@ class FeedItem protected ?string $uid = null; protected array $misc = []; + private Logger $logger; + public static function fromArray(array $itemArray): self { $item = new self(); @@ -23,6 +25,7 @@ class FeedItem private function __construct() { + $this->logger = RssBridge::getLogger(); } public function __set($name, $value) @@ -99,17 +102,17 @@ class FeedItem } elseif ($uri->hasAttribute('src')) { // Image $uri = $uri->src; } else { - Debug::log('The item provided as URI is unknown!'); + $this->logger->debug('The item provided as URI is unknown!'); } } if (!is_string($uri)) { - Debug::log(sprintf('Expected $uri to be string but got %s', gettype($uri))); + $this->logger->debug(sprintf('Expected $uri to be string but got %s', gettype($uri))); return; } $uri = trim($uri); // Intentionally doing a weak url validation here because FILTER_VALIDATE_URL is too strict if (!preg_match('#^https?://#i', $uri)) { - Debug::log(sprintf('Not a valid url: "%s"', $uri)); + $this->logger->debug(sprintf('Not a valid url: "%s"', $uri)); return; } $this->uri = $uri; @@ -124,7 +127,7 @@ class FeedItem { $this->title = null; if (!is_string($title)) { - trigger_error('Title must be a string: ' . print_r($title, true)); + $this->logger->debug('Title must be a string: ' . print_r($title, true)); } else { $this->title = truncate(trim($title)); } @@ -143,11 +146,11 @@ class FeedItem } else { $timestamp = strtotime($datetime); if ($timestamp === false) { - Debug::log('Unable to parse timestamp!'); + $this->logger->debug('Unable to parse timestamp!'); } } if ($timestamp <= 0) { - Debug::log('Timestamp must be greater than zero!'); + $this->logger->debug('Timestamp must be greater than zero!'); } else { $this->timestamp = $timestamp; } @@ -162,7 +165,7 @@ class FeedItem { $this->author = null; if (!is_string($author)) { - Debug::log('Author must be a string!'); + $this->logger->debug('Author must be a string!'); } else { $this->author = $author; } @@ -190,7 +193,7 @@ class FeedItem if (is_string($content)) { $this->content = $content; } else { - Debug::log(sprintf('Unable to convert feed content to string: %s', gettype($content))); + $this->logger->debug(sprintf('Unable to convert feed content to string: %s', gettype($content))); } } @@ -204,7 +207,7 @@ class FeedItem $this->enclosures = []; if (!is_array($enclosures)) { - Debug::log('Enclosures must be an array!'); + $this->logger->debug('Enclosures must be an array!'); return; } foreach ($enclosures as $enclosure) { @@ -215,7 +218,7 @@ class FeedItem FILTER_FLAG_PATH_REQUIRED ) ) { - Debug::log('Each enclosure must contain a scheme, host and path!'); + $this->logger->debug('Each enclosure must contain a scheme, host and path!'); } elseif (!in_array($enclosure, $this->enclosures)) { $this->enclosures[] = $enclosure; } @@ -232,14 +235,14 @@ class FeedItem $this->categories = []; if (!is_array($categories)) { - Debug::log('Categories must be an array!'); + $this->logger->debug('Categories must be an array!'); return; } foreach ($categories as $category) { if (is_string($category)) { $this->categories[] = $category; } else { - Debug::log('Category must be a string!'); + $this->logger->debug('Category must be a string!'); } } } @@ -253,7 +256,7 @@ class FeedItem { $this->uid = null; if (!is_string($uid)) { - Debug::log(sprintf('uid must be string: %s (%s)', (string) $uid, var_export($uid, true))); + $this->logger->debug(sprintf('uid must be string: %s (%s)', (string) $uid, var_export($uid, true))); return; } if (preg_match('/^[a-f0-9]{40}$/', $uid)) { @@ -267,9 +270,9 @@ class FeedItem public function addMisc($name, $value) { if (!is_string($name)) { - Debug::log('Key must be a string!'); + $this->logger->debug('Key must be a string!'); } elseif (in_array($name, get_object_vars($this))) { - Debug::log('Key must be unique!'); + $this->logger->debug('Key must be unique!'); } else { $this->misc[$name] = $value; } diff --git a/lib/RssBridge.php b/lib/RssBridge.php index e80e6f0a..23f65cf0 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -100,7 +100,8 @@ final class RssBridge public static function getLogger(): Logger { - return self::$logger; + // null logger is only for the tests not to fail + return self::$logger ?? new NullLogger(); } public static function getCache(): CacheInterface From f358f1abec8278c4e5659b311bf8eea5f34802d8 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 8 Aug 2024 17:47:04 +0200 Subject: [PATCH 162/423] refactor: loadCacheValue/saveCacheValue (#4205) --- bridges/BugzillaBridge.php | 2 +- bridges/FurAffinityBridge.php | 2 +- bridges/ItakuBridge.php | 2 +- bridges/MastodonBridge.php | 2 +- bridges/PixivBridge.php | 2 +- bridges/WordPressMadaraBridge.php | 2 +- docs/05_Bridge_API/02_BridgeAbstract.md | 324 +++++++++++++----------- lib/BridgeAbstract.php | 10 +- 8 files changed, 188 insertions(+), 158 deletions(-) diff --git a/bridges/BugzillaBridge.php b/bridges/BugzillaBridge.php index c2dc8d40..23e93eb8 100644 --- a/bridges/BugzillaBridge.php +++ b/bridges/BugzillaBridge.php @@ -164,7 +164,7 @@ class BugzillaBridge extends BridgeAbstract } $cache = $this->loadCacheValue($this->instance . $user); - if (!is_null($cache)) { + if ($cache) { return $cache; } diff --git a/bridges/FurAffinityBridge.php b/bridges/FurAffinityBridge.php index 087c3ded..f7d830bb 100644 --- a/bridges/FurAffinityBridge.php +++ b/bridges/FurAffinityBridge.php @@ -676,7 +676,7 @@ class FurAffinityBridge extends BridgeAbstract $name = parent::getName(); if ($this->getOption('aCookie') !== null) { $username = $this->loadCacheValue('username'); - if ($username !== null) { + if ($username) { $name = $username . '\'s ' . parent::getName(); } } diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index e6ecbdc8..b231b143 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -667,7 +667,7 @@ class ItakuBridge extends BridgeAbstract if ($getJSON) { //get JSON object if ($cache) { $data = $this->loadCacheValue($url); - if (is_null($data)) { + if (!$data) { $data = getContents($url, $httpHeaders, $curlOptions); $this->saveCacheValue($url, $data); } diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index b98b1135..e180fdb4 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -105,7 +105,7 @@ class MastodonBridge extends BridgeAbstract break; } $rtUser = $this->loadCacheValue($rtContent['attributedTo']); - if (!isset($rtUser)) { + if (!$rtUser) { // We fetch the author, since we cannot always assume the format of the URL. $user = $this->fetchAP($rtContent['attributedTo']); preg_match('/https?:\/\/([a-z0-9-\.]{0,})\//', $rtContent['attributedTo'], $matches); diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php index 820b3a7c..e464b12d 100644 --- a/bridges/PixivBridge.php +++ b/bridges/PixivBridge.php @@ -314,7 +314,7 @@ class PixivBridge extends BridgeAbstract { // checks if cookie is set, if not initialise it with the cookie from the config $value = $this->loadCacheValue('cookie'); - if (!isset($value)) { + if (!$value) { $value = $this->getOption('cookie'); // 30 days + 1 day to let cookie chance to renew diff --git a/bridges/WordPressMadaraBridge.php b/bridges/WordPressMadaraBridge.php index 4325075c..9af44ef4 100644 --- a/bridges/WordPressMadaraBridge.php +++ b/bridges/WordPressMadaraBridge.php @@ -118,7 +118,7 @@ The default URI shows the Madara demo page.'; { $url_cache = 'TitleInfo_' . preg_replace('/[^\w]/', '.', rtrim($url, '/')); $cache = $this->loadCacheValue($url_cache); - if (isset($cache)) { + if ($cache) { return $cache; } diff --git a/docs/05_Bridge_API/02_BridgeAbstract.md b/docs/05_Bridge_API/02_BridgeAbstract.md index a8e9db42..b6813a16 100644 --- a/docs/05_Bridge_API/02_BridgeAbstract.md +++ b/docs/05_Bridge_API/02_BridgeAbstract.md @@ -1,4 +1,5 @@ -`BridgeAbstract` is a base class for standard bridges. It implements the most common functions to simplify the process of adding new bridges. +`BridgeAbstract` is a base class for standard bridges. +It implements the most common functions to simplify the process of adding new bridges. *** @@ -11,7 +12,9 @@ You need four basic steps in order to create a new bridge: [**Step 3**](#step-3---add-general-constants-to-the-class) - Add general constants to the class [**Step 4**](#step-4---implement-a-function-to-collect-feed-data) - Implement a function to collect feed data -These steps are described in more detail below. At the end of this document you'll find a complete [template](#template) based on these instructions. The pictures below show an example based on these instructions: +These steps are described in more detail below. +At the end of this document you'll find a complete [template](#template) based on these instructions. +The pictures below show an example based on these instructions: <details><summary>Show pictures</summary><div> @@ -23,7 +26,12 @@ These steps are described in more detail below. At the end of this document you' </div></details><br> -Make sure to read these instructions carefully. Please don't hesitate to open an [Issue](https://github.com/RSS-Bridge/rss-bridge/issues) if you have further questions (or suggestions). Once your bridge is finished, please open a [Pull Request](https://github.com/RSS-Bridge/rss-bridge/pulls), in order to get your bridge merge into RSS-Bridge. +Make sure to read these instructions carefully. +Please don't hesitate to open an +[Issue](https://github.com/RSS-Bridge/rss-bridge/issues) +if you have further questions (or suggestions). +Once your bridge is finished, please open a [Pull Request](https://github.com/RSS-Bridge/rss-bridge/pulls), +in order to get your bridge merge into RSS-Bridge. *** @@ -33,7 +41,8 @@ Please read [these instructions](./01_How_to_create_a_new_bridge.md) on how to c ## Step 2 - Add a class, extending `BridgeAbstract` -Your bridge needs to be a class, which extends `BridgeAbstract`. The class name must **exactly** match the name of the file, without the file extension. +Your bridge needs to be a class, which extends `BridgeAbstract`. +The class name must **exactly** match the name of the file, without the file extension. For example: `MyBridge.php` => `MyBridge` @@ -65,43 +74,46 @@ const CACHE_TIMEOUT // (optional) Defines the maximum duration for the cache in <details><summary>Show example</summary><div> ```PHP -<?PHP +<?php + class MyBridge extends BridgeAbstract { const NAME = 'My Bridge'; const URI = 'https://rss-bridge.github.io/rss-bridge/Bridge_API/BridgeAbstract.html'; const DESCRIPTION = 'Returns "Hello World!"'; const MAINTAINER = 'ghost'; } -// This line is empty (just imagine it!) ``` </div></details><br> -**Notice**: `const PARAMETERS` can be used to request information from the user. Refer to [these instructions](#parameters) for more information. +**Notice**: `const PARAMETERS` can be used to request information from the user. +Refer to [these instructions](#parameters) for more information. ## Step 4 - Implement a function to collect feed data -In order for RSS-Bridge to collect data, you must implement the **public** function `collectData`. This function takes no arguments and returns nothing. It generates a list of feed elements, which must be placed into the variable `$this->items`. +In order for RSS-Bridge to collect data, you must implement the **public** function `collectData`. +This function takes no arguments and returns nothing. +It generates a list of feed elements, which must be placed into the variable `$this->items`. <details><summary>Show example</summary><div> ```PHP -<?PHP -class MyBridge extends BridgeAbstract { - const NAME = 'My Bridge'; - const URI = 'https://rss-bridge.github.io/rss-bridge/Bridge_API/BridgeAbstract.html'; - const DESCRIPTION = 'Returns "Hello World!"'; - const MAINTAINER = 'ghost'; +<?php - public function collectData() { - $item = []; // Create an empty item +class MyBridge extends BridgeAbstract +{ + const NAME = 'My Bridge'; + const URI = 'https://rss-bridge.github.io/rss-bridge/Bridge_API/BridgeAbstract.html'; + const DESCRIPTION = 'Returns "Hello World!"'; + const MAINTAINER = 'ghost'; - $item['title'] = 'Hello World!'; - - $this->items[] = $item; // Add item to the list - } + public function collectData() + { + $item = []; + $item['title'] = 'Hello World!'; + $this->items[] = $item; + } } -// This line is empty (just imagine it!) ``` </div></details><br> @@ -112,32 +124,36 @@ For more details on the `collectData` function refer to [these instructions](#co # Template -Use this template to create your own bridge. Please remove any unnecessary comments and parameters. +Use this template to create your own bridge. +Please remove any unnecessary comments and parameters. ```php <?php -class MyBridge extends BridgeAbstract { - const NAME = 'Unnamed bridge'; - const URI = ''; - const DESCRIPTION = 'No description provided'; - const MAINTAINER = 'No maintainer'; - const PARAMETERS = []; // Can be omitted! - const CACHE_TIMEOUT = 3600; // Can be omitted! - public function collectData() { - $item = []; // Create an empty item - - $item['title'] = 'Hello World!'; - - $this->items[] = $item; // Add item to the list - } +class MyBridge extends BridgeAbstract +{ + const NAME = 'Unnamed bridge'; + const URI = ''; + const DESCRIPTION = 'No description provided'; + const MAINTAINER = 'No maintainer'; + const PARAMETERS = []; // Can be omitted! + const CACHE_TIMEOUT = 3600; // Can be omitted! + + public function collectData() + { + $item = []; // Create an empty item + + $item['title'] = 'Hello World!'; + + $this->items[] = $item; // Add item to the list + } } -// This line is empty (just imagine it!) ``` # PARAMETERS -You can specify additional parameters in order to customize the bridge (i.e. to specify how many items to return). This document explains how to specify those parameters and which options are available to you. +You can specify additional parameters in order to customize the bridge (i.e. to specify how many items to return). +This document explains how to specify those parameters and which options are available to you. For information on how to read parameter values during execution, please refer to the [getInput](../06_Helper_functions/index.md#getinput) function. @@ -145,12 +161,14 @@ For information on how to read parameter values during execution, please refer t ## Adding parameters to a bridge -Parameters are specified as part of the bridge class. An empty list of parameters is defined as `const PARAMETERS = [];` +Parameters are specified as part of the bridge class. +An empty list of parameters is defined as `const PARAMETERS = [];` <details><summary>Show example</summary><div> ```PHP -<?PHP +<?php + class MyBridge extends BridgeAbstract { /* ... */ const PARAMETERS = []; // Empty list of parameters (can be omitted) @@ -167,7 +185,8 @@ Parameters are organized in two levels: ## Level 1 - Context -A context is defined as a associative array of parameters. The name of a context is displayed by RSS-Bridge. +A context is defined as a associative array of parameters. +The name of a context is displayed by RSS-Bridge. <details><summary>Show example</summary><div> @@ -196,7 +215,8 @@ const PARAMETERS = [ </div></details><br> -You can also define a set of parameters that will be applied to every possible context of your bridge. To do this, specify a context named `global`. +You can also define a set of parameters that will be applied to every possible context of your bridge. +To do this, specify a context named `global`. <details><summary>Show example</summary><div> @@ -219,9 +239,9 @@ where `n` is the name with which the bridge can access the parameter during exec ```PHP const PARAMETERS = [ - 'My Context' => [ - 'n' => [] - ] + 'My Context' => [ + 'n' => [], + ] ]; ``` @@ -232,17 +252,17 @@ The parameter specification consists of various fields, listed in the table belo <details><summary>Show example</summary><div> ```PHP -const PARAMETERS = array( - 'My Context' => array( - 'n' => array( +const PARAMETERS = [ + 'My Context' => [ + 'n' => [ 'name' => 'Limit', 'type' => 'number', 'required' => false, 'title' => 'Maximum number of items to return', - 'defaultValue' => 10 - ) - ) -); + 'defaultValue' => 10, + ] + ] +]; ``` **Output** @@ -271,29 +291,30 @@ List values are defined in an associative array where keys are the string displa ```PHP ... 'type' => 'list', - 'values' => array( + 'values' => [ 'Item A' => 'itemA' 'Item B' => 'itemB' - ) + ] ... ``` If a more complex organization is required to display the values, the above key/value can be used to set a title as a key and another array as a value: + ```PHP ... 'type' => 'list', - 'values' => array( + 'values' => [ 'Item A' => 'itemA', - 'List 1' => array( + 'List 1' => [ 'Item C' => 'itemC', 'Item D' => 'itemD' - ), - 'List 2' => array( + ], + 'List 2' => [ 'Item E' => 'itemE', 'Item F' => 'itemF' - ), + ], 'Item B' => 'itemB' - ) + ] ... ``` @@ -316,20 +337,18 @@ It provides a way to identify which context the bridge is called with. Example: ```PHP -... - const PARAMETERS = array( - 'By user name' => array( - 'u' => array('name' => 'Username') - ), - 'By user ID' => array( - 'id' => array('name' => 'User ID') - ) - ); - -... +const PARAMETERS = [ + 'By user name' => [ + 'u' => ['name' => 'Username'] + ], + 'By user ID' => [ + 'id' => ['name' => 'User ID'] + ] +]; ``` -In this example `$this->queriedContext` will either return **By user name** or **By user ID**. The queried context might return no value, so the best way to handle it is by using a case-structure: +In this example `$this->queriedContext` will either return **By user name** or **By user ID**. +The queried context might return no value, so the best way to handle it is by using a case-structure: ```PHP switch($this->queriedContext){ @@ -342,32 +361,36 @@ switch($this->queriedContext){ ``` # collectData -The `collectData` function is responsible for collecting data and adding items to generate feeds from. If you are unsure how to solve a specific problem, please don't hesitate to open an [Issue](https://github.com/RSS-Bridge/rss-bridge/issues) on GitHub. Existing bridges are also a good source to learn implementing your own bridge. + +The `collectData` function is responsible for collecting data and adding items to generate feeds from. +If you are unsure how to solve a specific problem, please don't hesitate to open an [Issue](https://github.com/RSS-Bridge/rss-bridge/issues) on GitHub. +Existing bridges are also a good source to learn implementing your own bridge. ## Implementing the `collectData` function -Implementation for the `collectData` function is specific to each bridge. However, there are certain reoccurring elements, described below. RSS-Bridge also provides functions to simplify the process of collecting and parsing HTML data (see "Helper Functions" on the sidebar) +Implementation for the `collectData` function is specific to each bridge. +However, there are certain reoccurring elements, described below. RSS-Bridge also provides functions to simplify the process of collecting and parsing HTML data (see "Helper Functions" on the sidebar) -Elements collected by this function must be stored in `$this->items`. The `items` variable is an array of item elements, each of which is an associative array that may contain arbitrary keys. RSS-Bridge specifies common keys which are used to generate most common feed formats. +Elements collected by this function must be stored in `$this->items`. +The `items` variable is an array of item elements, each of which is an associative array that may contain arbitrary keys. +RSS-Bridge specifies common keys which are used to generate most common feed formats. <details><summary>Show example</summary><div> ```PHP - -$item = []; // Create a new item - +$item = []; $item['title'] = 'Hello World!'; - -$this->items[] = $item; // Add item to the list - +$this->items[] = $item; ``` </div></details><br> + Additional keys may be added for custom APIs (ignored by RSS-Bridge). ## Item parameters -The item array should provide as much information as possible for RSS-Bridge to generate feature rich feeds. Find below list of keys supported by RSS-Bridge. +The item array should provide as much information as possible for RSS-Bridge to generate feature rich feeds. +Find below list of keys supported by RSS-Bridge. ```PHP $item['uri'] // URI to reach the subject ("https://...") @@ -379,65 +402,81 @@ $item['enclosures'] // Array of URIs to an attachments (pictures, files, etc...) $item['categories'] // Array of categories / tags / topics $item['uid'] // A unique ID to identify the current item ``` + All formats support these parameters. The formats `Plaintext` and `JSON` also support custom parameters. # getDescription The `getDescription` function returns the description for a bridge. -**Notice:** By default **RSS-Bridge** returns the contents of `const DESCRIPTION`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns the contents of `const DESCRIPTION`, +so you only have to implement this function if you require different behavior! ```PHP - public function getDescription(){ - return self::DESCRIPTION; - } +public function getDescription() +{ + return self::DESCRIPTION; +} ``` # getMaintainer The `getMaintainer` function returns the name of the maintainer for a bridge. -**Notice:** By default **RSS-Bridge** returns `const MAINTAINER`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns `const MAINTAINER`, +so you only have to implement this function if you require different behavior! ```PHP - public function getMaintainer(){ - return self::MAINTAINER; - } +public function getMaintainer() +{ + return self::MAINTAINER; +} ``` # getName + The `getName` function returns the name of a bridge. -**Notice:** By default **RSS-Bridge** returns `const NAME`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns `const NAME`, +so you only have to implement this function if you require different behavior! ```PHP - public function getName(){ - return self::NAME; - } +public function getName() +{ + return self::NAME; +} ``` # getURI + The `getURI` function returns the base URI for a bridge. -**Notice:** By default **RSS-Bridge** returns `const URI`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns `const URI`, +so you only have to implement this function if you require different behavior! ```PHP - public function getURI(){ - return self::URI; - } +public function getURI() +{ + return self::URI; +} ``` # getIcon + The `getIcon` function returns the URI for an icon, used as favicon in feeds. -If no icon is specified by the bridge, RSS-Bridge will use a default location: `static::URI . '/favicon.ico'` (i.e. "https://github.com/favicon.ico") which may or may not exist. +If no icon is specified by the bridge, +RSS-Bridge will use a default location: `static::URI . '/favicon.ico'` (i.e. "https://github.com/favicon.ico") which may or may not exist. ```PHP - public function getIcon(){ - return static::URI . '/favicon.ico'; - } +public function getIcon() +{ + return static::URI . '/favicon.ico'; +} ``` + # detectParameters + The `detectParameters` function takes a URL and attempts to extract a valid set of parameters for the current bridge. If the passed URL is valid for this bridge, the function should return an array of parameter -> value pairs that can be used by this bridge, including context if available, or an empty array if the bridge requires no parameters. If the URL is not relevant for this bridge, the function should return `null`. @@ -445,31 +484,34 @@ If the passed URL is valid for this bridge, the function should return an array **Notice:** Implementing this function is optional. By default, **RSS-Bridge** tries to match the supplied URL to the `URI` constant defined in the bridge, which may be enough for bridges without any parameters defined. ```PHP -public function detectParameters($url){ - $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/'; - if(empty(static::PARAMETERS) - && preg_match($regex, $url, $urlMatches) > 0 - && preg_match($regex, static::URI, $bridgeUriMatches) > 0 - && $urlMatches[3] === $bridgeUriMatches[3]) { - return []; - } else { - return null; - } +public function detectParameters($url) +{ + $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/'; + if (empty(static::PARAMETERS) + && preg_match($regex, $url, $urlMatches) > 0 + && preg_match($regex, static::URI, $bridgeUriMatches) > 0 + && $urlMatches[3] === $bridgeUriMatches[3] + ) { + return []; + } else { + return null; + } } ``` -**Notice:** This function is also used by the [findFeed](../04_For_Developers/04_Actions.md#findfeed) action. This action allows an user to get a list of all feeds corresponding to an URL. +**Notice:** This function is also used by the [findFeed](../04_For_Developers/04_Actions.md#findfeed) action. +This action allows an user to get a list of all feeds corresponding to an URL. You can implement automated tests for the `detectParameters` function by adding the `TEST_DETECT_PARAMETERS` constant to your bridge class constant. `TEST_DETECT_PARAMETERS` is an array, with as key the URL passed to the `detectParameters`function and as value, the array of parameters returned by `detectParameters` ```PHP - const TEST_DETECT_PARAMETERS = [ - 'https://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], - 'https://instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], - 'http://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], - ]; +const TEST_DETECT_PARAMETERS = [ + 'https://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'https://instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'http://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], +]; ``` **Notice:** Adding this constant is optional. If the constant is not present, no automated test will be executed. @@ -478,57 +520,47 @@ You can implement automated tests for the `detectParameters` function by adding *** # Helper Methods -`BridgeAbstract` implements helper methods to make it easier for bridge maintainers to create bridges. Use these methods whenever possible instead of writing your own. -- [saveCacheValue](#savecachevalue) -- [loadCacheValue](#loadcachevalue) +`BridgeAbstract` implements helper methods to make it easier for bridge maintainers to create bridges. +Use these methods whenever possible instead of writing your own. ## saveCacheValue -Within the context of the current bridge, stores a value by key in the cache. The value can later be retrieved with [loadCacheValue](#loadcachevalue). + +Within the context of the current bridge, stores a value by key in the cache. +The value can later be retrieved with [loadCacheValue](#loadcachevalue). ```php -protected function saveCacheValue($key, $value) +protected function saveCacheValue($key, $value, $ttl = null) ``` -- `$key` - the name under which the value is stored in the cache. -- `$value` - the value to store in the cache. - -Usage example: +Example: ```php -const MY_KEY = 'MyKey'; - public function collectData() { - $value = 'my value'; - $this->saveCacheValue(MY_KEY, $value); + $this->saveCacheValue('my_key', 'my_value', 3600); // 1h } ``` ## loadCacheValue -Within the context of the current bridge, loads a value by key from cache. Optionally specifies the cache duration for the key. Returns `null` if the key doesn't exist or the value is expired. + +Within the context of the current bridge, loads a value by key from cache. +Optionally specifies the cache duration for the key. +Returns `null` if the key doesn't exist or the value is expired. ```php -protected function loadCacheValue($key, $duration = null) +protected function loadCacheValue($key, $default = null) ``` -- `$key` - the name under which the value is stored in the cache. -- `$duration` - the maximum time in seconds after which the value expires. - -Usage example: +Example: ```php -const MY_KEY = 'MyKey'; - public function collectData() { - $value = $this->loadCacheValue(MY_KEY, 1800 /* 30 minutes */); + $value = $this->loadCacheValue('my_key'); - if (!isset($value)){ - // load value - $this->saveCacheValue(MY_KEY, $value); - } - - // ... + if (! $value) { + $this->saveCacheValue('my_key', 'foobar'); + } } ``` diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 2467dec6..6444b6aa 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -322,16 +322,14 @@ abstract class BridgeAbstract return null; } - protected function loadCacheValue(string $key) + protected function loadCacheValue(string $key, $default = null) { - $cacheKey = $this->getShortName() . '_' . $key; - return $this->cache->get($cacheKey); + return $this->cache->get($this->getShortName() . '_' . $key, $default); } - protected function saveCacheValue(string $key, $value, $ttl = 86400) + protected function saveCacheValue(string $key, $value, int $ttl = null) { - $cacheKey = $this->getShortName() . '_' . $key; - $this->cache->set($cacheKey, $value, $ttl); + $this->cache->set($this->getShortName() . '_' . $key, $value, $ttl); } public function getShortName(): string From adcc8e371d7b369f46945e01ce57b7561f383726 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:20:10 +0200 Subject: [PATCH 163/423] [TarnkappeBridge] changed "unwanted stuff" (#4206) * [TarnkappeBridge] changed "unwanted stuff" em was removed because the annoying affiliate info, but it also deleted the text from blockquotes. The p-element with the affiliate info has no attributes like class, but it is the only p-element with a style-attribute, so I used this to identify it. * Update TarnkappeBridge.php removed whitespace * Update TarnkappeBridge.php don't know why I did it twice before --- bridges/TarnkappeBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/TarnkappeBridge.php b/bridges/TarnkappeBridge.php index c04c9546..374a79b3 100644 --- a/bridges/TarnkappeBridge.php +++ b/bridges/TarnkappeBridge.php @@ -65,10 +65,11 @@ class TarnkappeBridge extends FeedExpander // remove unwanted stuff foreach ( - $article->find('em, section, div.menu') as $element + $article->find('section, div.menu, p[style]') as $element ) { $element->remove(); } + // reload html, as remove() is buggy $article = str_get_html($article->outertext); From 4ef5ca50c6f9080481e34fa305af952187b39d88 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Sat, 10 Aug 2024 11:36:58 -0400 Subject: [PATCH 164/423] [KemonoBridge] Add KemonoBridge (#4192) * [KemonoBridge] Add KemonoBridge * refactor * [KemonoBridge] fix categories in cases where it's a proper json array --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/KemonoBridge.php | 93 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 bridges/KemonoBridge.php diff --git a/bridges/KemonoBridge.php b/bridges/KemonoBridge.php new file mode 100644 index 00000000..53a820dc --- /dev/null +++ b/bridges/KemonoBridge.php @@ -0,0 +1,93 @@ +<?php + +class KemonoBridge extends BridgeAbstract +{ + const NAME = 'Kemono'; + const MAINTAINER = 'phantop'; + const URI = 'https://kemono.su/'; + const DESCRIPTION = 'Returns posts from Kemono.'; + const PARAMETERS = [[ + 'service' => [ + 'name' => 'Content service', + 'type' => 'list', + 'defaultValue' => 'patreon', + 'values' => [ + 'Patreon' => 'patreon', + 'Pixiv Fanbox' => 'fanbox', + 'Fantia' => 'fantia', + 'Boosty' => 'boosty', + 'Gumroad' => 'gumroad', + 'SubscribeStar' => 'subscribestar', + ] + ], + 'user' => [ + 'name' => 'User ID/Name', + 'exampleValue' => '9069743', # Thomas Joy + 'required' => true, + ] + ]]; + + private $title; + + public function collectData() + { + $api = parent::getURI() . 'api/v1/'; + $url = $api . $this->getInput('service') . '/user/' . $this->getInput('user'); + $api_response = getContents($url); + $json = Json::decode($api_response); + + $url .= '/profile'; + $api_response = getContents($url); + $profile = Json::decode($api_response); + $this->title = ucfirst($profile['name']); + + foreach ($json as $element) { + $item = []; + $item['author'] = $this->title; + $item['content'] = $element['content']; + $item['timestamp'] = strtotime($element['published']); + $item['title'] = $element['title']; + $item['uid'] = $element['id']; + $item['uri'] = $this->getURI() . '/post/' . $item['uid']; + + if ($element['tags']) { + $tags = $element['tags']; + if (is_array($tags)) { + $item['categories'] = $tags; + } else { + $tags = preg_replace('/^{"?/', '["', $tags); + $tags = preg_replace('/"?}$/', '"]', $tags); + $item['categories'] = Json::decode($tags); + } + } + + $item['enclosures'] = []; + if (array_key_exists('url', $element['embed'])) { + $item['enclosures'][] = $element['embed']['url']; + } + if (array_key_exists('path', $element['file'])) { + $element['attachments'][] = $element['file']; + } + foreach ($element['attachments'] as $file) { + $item['enclosures'][] = parent::getURI() . $file['path']; + } + + $this->items[] = $item; + } + } + + public function getName() + { + $name = parent::getName(); + if (isset($this->title)) { + $name .= ' - ' . $this->title; + } + return $name; + } + + public function getURI() + { + $uri = parent::getURI() . $this->getInput('service') . '/user/' . $this->getInput('user'); + return $uri; + } +} From 129b8a3a5aee04b15edf5dcdc4c44f2c062fc4a8 Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sat, 10 Aug 2024 23:10:37 +0200 Subject: [PATCH 165/423] [ModifyBridge] New bridge to modify feeds (#4164) * [ModifyBridge] New bridge to modify feeds Create a general bridge that can modify the common fields of feeds with regular expressions. * [ModifyBridge] Also modify <enclosure> element Additionally to the list of <enclosures>. --- bridges/ModifyBridge.php | 181 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 bridges/ModifyBridge.php diff --git a/bridges/ModifyBridge.php b/bridges/ModifyBridge.php new file mode 100644 index 00000000..bf9fa262 --- /dev/null +++ b/bridges/ModifyBridge.php @@ -0,0 +1,181 @@ +<?php + +class ModifyBridge extends FeedExpander +{ + const MAINTAINER = 'Mynacol'; + const NAME = 'Modify Feed'; + const CACHE_TIMEOUT = 3600; // 1h + const DESCRIPTION = 'Modifies a feed of your choice with regexes'; + const URI = 'https://github.com/RSS-Bridge/rss-bridge'; + + const PARAMETERS = [[ + 'url' => [ + 'name' => 'Feed URL', + 'type' => 'text', + 'exampleValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day', + 'required' => true, + ], + 'title_pattern' => [ + 'name' => 'Title find pattern (regular expression!)', + 'type' => 'text', + 'exampleValue' => 'Unwanted part in title', + 'required' => false, + ], + 'title_replacement' => [ + 'name' => 'Title replacement for the find pattern', + 'type' => 'text', + 'exampleValue' => '${0}', + 'required' => false, + ], + 'author_pattern' => [ + 'name' => 'Author find pattern (regular expression!)', + 'type' => 'text', + 'exampleValue' => '^(author)\s*|\s*publisher$', + 'required' => false, + ], + 'author_replacement' => [ + 'name' => 'Author replacement for the find pattern', + 'type' => 'text', + 'exampleValue' => '${1}', + 'required' => false, + ], + 'content_pattern' => [ + 'name' => 'Content find pattern (regular expression!)', + 'type' => 'text', + 'exampleValue' => '(content)\s+advertisement\s+(content)', + 'required' => false, + ], + 'content_replacement' => [ + 'name' => 'Content replacement for the find pattern', + 'type' => 'text', + 'exampleValue' => '${1} ${2}', + 'required' => false, + ], + 'uri_pattern' => [ + 'name' => 'URI/URL find pattern (regular expression!)', + 'type' => 'text', + 'exampleValue' => '^https?://(.*)/(.*)$', + 'required' => false, + ], + 'uri_replacement' => [ + 'name' => 'URI/URL replacement for the find pattern', + 'type' => 'text', + 'exampleValue' => 'https://${1}/foo/${2}', + 'required' => false, + ], + 'enclosure_pattern' => [ + 'name' => 'Enclosure URI/URL find pattern (regular expression!)', + 'type' => 'text', + 'exampleValue' => '^https?://(.*)/(.*)$', + 'required' => false, + ], + 'enclosure_replacement' => [ + 'name' => 'Enclosure URI/URL replacement for the find pattern', + 'type' => 'text', + 'exampleValue' => 'https://${1}/foo/${2}', + 'required' => false, + ], + 'case_insensitive' => [ + 'name' => 'Case-insensitive find patterns', + 'type' => 'checkbox', + 'required' => false, + ], + ]]; + + public function collectData() + { + $url = $this->getInput('url'); + if (!Url::validate($url)) { + throw new \Exception('The url parameter must either refer to http or https protocol.'); + } + $this->collectExpandableDatas($this->getURI()); + } + + protected function parseItem(array $item) + { + // Title + $pattern = $this->buildPattern($this->getInput('title_pattern')); + $replacement = $this->getInput('title_replacement'); + $res = preg_replace($pattern, $replacement, $item['title']); + if ($res !== null) { + $item['title'] = $res; + } + + // Author + $pattern = $this->buildPattern($this->getInput('author_pattern')); + $replacement = $this->getInput('author_replacement'); + $res = preg_replace($pattern, $replacement, $item['author']); + if ($res !== null) { + $item['author'] = $res; + } + + // Content + $pattern = $this->buildPattern($this->getInput('content_pattern')); + $replacement = $this->getInput('content_replacement'); + $res = preg_replace($pattern, $replacement, $item['content']); + if ($res !== null) { + $item['content'] = $res; + } + + // URI + $pattern = $this->buildPattern($this->getInput('uri_pattern')); + $replacement = $this->getInput('uri_replacement'); + $res = preg_replace($pattern, $replacement, $item['uri']); + if ($res !== null) { + $item['uri'] = $res; + } + + // Enclosures + if (array_key_exists('enclosures', $item)) { + $pattern = $this->buildPattern($this->getInput('enclosure_pattern')); + $replacement = $this->getInput('enclosure_replacement'); + foreach ($item['enclosures'] as $key => $val) { + $res = preg_replace($pattern, $replacement, $val); + if ($res !== null) { + $item['enclosures'][$key] = $res; + } + } + } + if (array_key_exists('enclosure', $item)) { + $pattern = $this->buildPattern($this->getInput('enclosure_pattern')); + $replacement = $this->getInput('enclosure_replacement'); + $res = preg_replace($pattern, $replacement, $item['enclosure']['url']); + if ($res !== null) { + $item['enclosure']['url'] = $res; + } + } + + return $item; + } + + private function buildPattern($pattern) + { + if (! str_contains($pattern, '#')) { + $delimiter = '#'; + } elseif (! str_contains($pattern, '/')) { + $delimiter = '/'; + } else { + throw new \Exception('Cannot use both / and # inside filter'); + } + + $regex = $delimiter . $pattern . $delimiter; + if ($this->getInput('case_insensitive')) { + $regex .= 'i'; + } + return $regex; + } + + public function getURI() + { + $url = $this->getInput('url'); + if ($url) { + return $url; + } + return parent::getURI(); + } + + public function getName() + { + return parent::getName(); + } +} From 2e6e2467598c8400cf476cb5914b3a8bd1a3601d Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Sat, 10 Aug 2024 17:11:43 -0400 Subject: [PATCH 166/423] [KemonoBridge] attempt to fix malformed tag responses (#4209) --- bridges/KemonoBridge.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bridges/KemonoBridge.php b/bridges/KemonoBridge.php index 53a820dc..e333b574 100644 --- a/bridges/KemonoBridge.php +++ b/bridges/KemonoBridge.php @@ -55,9 +55,10 @@ class KemonoBridge extends BridgeAbstract if (is_array($tags)) { $item['categories'] = $tags; } else { - $tags = preg_replace('/^{"?/', '["', $tags); - $tags = preg_replace('/"?}$/', '"]', $tags); - $item['categories'] = Json::decode($tags); + $tags = preg_replace('/^{/', '', $tags); + $tags = preg_replace('/}$/', '', $tags); + $tags = preg_replace('/"/', '', $tags); + $item['categories'] = explode(',', $tags); } } From 133dbf87c50c70f8644cfd7916aff3daa8fac520 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 11 Aug 2024 01:23:10 +0200 Subject: [PATCH 167/423] fix(telegram): add note if content is omitted from preview page (#4210) * fix(telegram): add note if content is omitted from preview page * lint --- bridges/TelegramBridge.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bridges/TelegramBridge.php b/bridges/TelegramBridge.php index a3c910e8..81c5aeb9 100644 --- a/bridges/TelegramBridge.php +++ b/bridges/TelegramBridge.php @@ -70,8 +70,14 @@ class TelegramBridge extends BridgeAbstract { $message = ''; + $notSupported = $messageDiv->find('div.message_media_not_supported_wrap', 0); + if ($notSupported) { + // For unknown reasons, the telegram preview page omits the content of this post + $message = 'RSS-Bridge was unable to find the content of this post.<br><br>' . $notSupported->innertext; + } + if ($messageDiv->find('div.tgme_widget_message_forwarded_from', 0)) { - $message = $messageDiv->find('div.tgme_widget_message_forwarded_from', 0)->innertext . '<br><br>'; + $message .= $messageDiv->find('div.tgme_widget_message_forwarded_from', 0)->innertext . '<br><br>'; } if ($messageDiv->find('a.tgme_widget_message_reply', 0)) { From 4424ea54e93da9e9ea125e07037bc3db7e8af590 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 11 Aug 2024 02:31:50 +0200 Subject: [PATCH 168/423] chore: increase linter speed (#4211) --- composer.json | 2 +- index.php | 3 +++ lib/BridgeAbstract.php | 6 +++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index 34f1a80f..cafcd085 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ }, "scripts": { "test": "./vendor/bin/phpunit", - "lint": "./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./", + "lint": "./vendor/bin/phpcs --parallel=2 --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./", "compat": "./vendor/bin/phpcs --standard=phpcompatibility.xml --warning-severity=0 --extensions=php -p ./" } } diff --git a/index.php b/index.php index d29900f7..1efda44a 100644 --- a/index.php +++ b/index.php @@ -62,6 +62,9 @@ register_shutdown_function(function () use ($logger) { $cacheFactory = new CacheFactory($logger); +// Uncomment this for debug logging +// $logger->addHandler(new StreamHandler('/tmp/rss-bridge.txt', Logger::DEBUG)); + if (Debug::isEnabled()) { $logger->addHandler(new ErrorLogHandler(Logger::DEBUG)); $cache = $cacheFactory->create('array'); diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 6444b6aa..23e90e13 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -2,8 +2,8 @@ abstract class BridgeAbstract { - const NAME = 'Unnamed bridge'; - const URI = ''; + const NAME = null; + const URI = null; const DONATION_URI = ''; const DESCRIPTION = 'No description provided'; @@ -61,7 +61,7 @@ abstract class BridgeAbstract public function getName() { - return static::NAME; + return static::NAME ?? $this->getShortName(); } public function getURI() From 307c22204d643308b8cb22ba024a5b6b95fbcb84 Mon Sep 17 00:00:00 2001 From: Tobias Alexander Franke <thefranke@users.noreply.github.com> Date: Sun, 11 Aug 2024 23:20:20 +0200 Subject: [PATCH 169/423] [ActivisionResearchBridge] New bridge for the Activision Research blog (#4213) * [ActivisionResearchBridge] New bridge for the Activision Research blog * [ActivisionResearchBridge] Fix linting issues --- bridges/ActivisionResearchBridge.php | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 bridges/ActivisionResearchBridge.php diff --git a/bridges/ActivisionResearchBridge.php b/bridges/ActivisionResearchBridge.php new file mode 100644 index 00000000..88af4b46 --- /dev/null +++ b/bridges/ActivisionResearchBridge.php @@ -0,0 +1,45 @@ +<?php + +class ActivisionResearchBridge extends BridgeAbstract +{ + const NAME = 'Activision Research Blog'; + const URI = 'https://research.activision.com'; + const DESCRIPTION = 'Posts from the Activision Research blog'; + const MAINTAINER = 'thefranke'; + const CACHE_TIMEOUT = 86400; // 24h + + public function collectData() + { + $dom = getSimpleHTMLDOM(static::URI); + $dom = $dom->find('div[id="home-blog-feed"]', 0); + if (!$dom) { + throw new \Exception(sprintf('Unable to find css selector on `%s`', $url)); + } + $dom = defaultLinkTo($dom, $this->getURI()); + foreach ($dom->find('div[class="blog-entry"]') as $article) { + $a = $article->find('a', 0); + + $blogimg = extractFromDelimiters($article->find('div[class="blog-img"]', 0)->style, 'url(', ')'); + + $title = htmlspecialchars_decode($article->find('div[class="title"]', 0)->plaintext); + $author = htmlspecialchars_decode($article->find('div[class="author]', 0)->plaintext); + $date = $article->find('div[class="pubdate"]', 0)->plaintext; + + $entry = getSimpleHTMLDOMCached($a->href, static::CACHE_TIMEOUT * 7 * 4); + $entry = defaultLinkTo($entry, $this->getURI()); + + $content = $entry->find('div[class="blog-body"]', 0); + $tagsremove = ['script', 'iframe', 'input', 'form']; + $content = sanitize($content, $tagsremove); + $content = '<img src="' . static::URI . $blogimg . '" alt="">' . $content; + + $this->items[] = [ + 'title' => $title, + 'author' => $author, + 'uri' => $a->href, + 'content' => $content, + 'timestamp' => strtotime($date), + ]; + } + } +} From e9d3a657bab09c6fbe8e7d4002cdcb5215054bf4 Mon Sep 17 00:00:00 2001 From: Tobias Alexander Franke <thefranke@users.noreply.github.com> Date: Thu, 15 Aug 2024 00:47:39 +0200 Subject: [PATCH 170/423] [EASeedBridge] New bridge for the EA Seed blog (#4216) * [EASeedBridge] New bridge for the EA Seed blog * Fix linter issues --- bridges/EASeedBridge.php | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 bridges/EASeedBridge.php diff --git a/bridges/EASeedBridge.php b/bridges/EASeedBridge.php new file mode 100644 index 00000000..bb5fa41d --- /dev/null +++ b/bridges/EASeedBridge.php @@ -0,0 +1,42 @@ +<?php + +class EASeedBridge extends BridgeAbstract +{ + const NAME = 'EA Seed Blog'; + const URI = 'https://www.ea.com/seed'; + const DESCRIPTION = 'Posts from the EA Seed blog'; + const MAINTAINER = 'thefranke'; + const CACHE_TIMEOUT = 86400; // 24h + + public function collectData() + { + $dom = getSimpleHTMLDOM(static::URI); + $dom = $dom->find('ea-grid', 0); + if (!$dom) { + throw new \Exception(sprintf('Unable to find css selector on `%s`', $url)); + } + $dom = defaultLinkTo($dom, $this->getURI()); + foreach ($dom->find('ea-tile') as $article) { + $a = $article->find('a', 0); + $date = $article->find('div', 1)->plaintext; + $title = $article->find('h3', 0)->plaintext; + $author = $article->find('div', 0)->plaintext; + + $entry = getSimpleHTMLDOMCached($a->href, static::CACHE_TIMEOUT * 7 * 4); + + $content = $entry->find('main', 0); + + // remove header and links to other posts + $content->find('ea-header', 0)->outertext = ''; + $content->find('ea-section', -1)->outertext = ''; + + $this->items[] = [ + 'title' => $title, + 'author' => $author, + 'uri' => $a->href, + 'content' => $content, + 'timestamp' => strtotime($date), + ]; + } + } +} From c0e37bcf35d4eb040138423de82ce934d53ff093 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 18 Aug 2024 19:11:11 +0200 Subject: [PATCH 171/423] refactor: frontpage and proxy setting (#4214) --- actions/DisplayAction.php | 1 + actions/FrontpageAction.php | 22 +++++++++++++--------- config.default.ini.php | 4 ++-- lib/BridgeCard.php | 15 +++++++++------ lib/http.php | 1 + 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index a5703e5e..d111e69e 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -51,6 +51,7 @@ class DisplayAction implements ActionInterface return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400); } + // Disable proxy (if enabled and per user's request) if ( Configuration::getConfig('proxy', 'url') && Configuration::getConfig('proxy', 'by_bridge') diff --git a/actions/FrontpageAction.php b/actions/FrontpageAction.php index c72dfd57..6ab18d29 100644 --- a/actions/FrontpageAction.php +++ b/actions/FrontpageAction.php @@ -4,6 +4,8 @@ final class FrontpageAction implements ActionInterface { public function __invoke(Request $request): Response { + $token = $request->attribute('token'); + $messages = []; $activeBridges = 0; @@ -20,19 +22,21 @@ final class FrontpageAction implements ActionInterface $body = ''; foreach ($bridgeClassNames as $bridgeClassName) { if ($bridgeFactory->isEnabled($bridgeClassName)) { - $body .= BridgeCard::render($bridgeClassName, $request); + $body .= BridgeCard::render($bridgeClassName, $token); $activeBridges++; } } - // todo: cache this renderered template? - return new Response(render(__DIR__ . '/../templates/frontpage.html.php', [ - 'messages' => $messages, - 'admin_email' => Configuration::getConfig('admin', 'email'), - 'admin_telegram' => Configuration::getConfig('admin', 'telegram'), - 'bridges' => $body, - 'active_bridges' => $activeBridges, - 'total_bridges' => count($bridgeClassNames), + $response = new Response(render(__DIR__ . '/../templates/frontpage.html.php', [ + 'messages' => $messages, + 'admin_email' => Configuration::getConfig('admin', 'email'), + 'admin_telegram' => Configuration::getConfig('admin', 'telegram'), + 'bridges' => $body, + 'active_bridges' => $activeBridges, + 'total_bridges' => count($bridgeClassNames), ])); + + // TODO: The rendered template could be cached, but beware config changes that changes the html + return $response; } } diff --git a/config.default.ini.php b/config.default.ini.php index c6b0779d..c23372d9 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -86,8 +86,8 @@ telegram = "" donations = true [proxy] - -; Sets the proxy url (i.e. "tcp://192.168.0.0:32") +; The HTTP proxy to tunnel requests through +; https://curl.se/libcurl/c/CURLOPT_PROXY.html ; "" = Proxy disabled (default) url = "" diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index 27285558..f270c1a3 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -2,7 +2,7 @@ final class BridgeCard { - public static function render(string $bridgeClassName, Request $request): string + public static function render(string $bridgeClassName, ?string $token): string { $bridgeFactory = new BridgeFactory(); @@ -14,10 +14,15 @@ final class BridgeCard $description = $bridge->getDescription(); $contexts = $bridge->getParameters(); - if (Configuration::getConfig('proxy', 'url') && Configuration::getConfig('proxy', 'by_bridge')) { + // Checkbox for disabling of proxy (if enabled) + if ( + Configuration::getConfig('proxy', 'url') + && Configuration::getConfig('proxy', 'by_bridge') + ) { + $proxyName = Configuration::getConfig('proxy', 'name') ?: Configuration::getConfig('proxy', 'url'); $contexts['global']['_noproxy'] = [ - 'name' => 'Disable proxy (' . (Configuration::getConfig('proxy', 'name') ?: Configuration::getConfig('proxy', 'url')) . ')', - 'type' => 'checkbox' + 'name' => sprintf('Disable proxy (%s)', $proxyName), + 'type' => 'checkbox', ]; } @@ -47,8 +52,6 @@ final class BridgeCard CARD; - $token = $request->attribute('token'); - if (count($contexts) === 0) { // The bridge has zero parameters $card .= self::renderForm($bridgeClassName, '', [], $token); diff --git a/lib/http.php b/lib/http.php index 0d21b958..d1043b33 100644 --- a/lib/http.php +++ b/lib/http.php @@ -113,6 +113,7 @@ final class CurlHttpClient implements HttpClient if ($config['proxy']) { curl_setopt($ch, CURLOPT_PROXY, $config['proxy']); } + if (curl_setopt_array($ch, $config['curl_options']) === false) { throw new \Exception('Tried to set an illegal curl option'); } From 320afc3f324ded84f56169dce70bc7721201ea0a Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Mon, 19 Aug 2024 22:47:42 +0530 Subject: [PATCH 172/423] [MaalaimalarBridge] fix image (#4222) * [NvidiaDriverBridge] Added Windows support * Update NvidiaDriverBridge.php * Update NvidiaDriverBridge.php * [MaalaimalarBridge] fix image * [MaalaimalarBridge] fix lint --- bridges/MaalaimalarBridge.php | 8 +++- bridges/NvidiaDriverBridge.php | 86 ++++++++++++++++++++++++++++------ 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/bridges/MaalaimalarBridge.php b/bridges/MaalaimalarBridge.php index 87f85694..59c76b01 100644 --- a/bridges/MaalaimalarBridge.php +++ b/bridges/MaalaimalarBridge.php @@ -103,8 +103,12 @@ class MaalaimalarBridge extends BridgeAbstract { $content = ''; $imageElement = $article->find('div.ignore-autoplay img', 0); - if ($imageElement) { - $content .= '<p><img src="' . $imageElement->{'data-src'} . '"></p>'; + if ($imageElement && isset($imageElement->{'data-src'})) { + $url = str_replace('500x300_', '', $imageElement->{'data-src'}); + + if (filter_var($url, FILTER_VALIDATE_URL)) { + $content = sprintf('<p><img src="%s"></p>', htmlspecialchars($url, ENT_QUOTES, 'UTF-8')); + } } $storyElement = $article->find('div.story-content', 0); diff --git a/bridges/NvidiaDriverBridge.php b/bridges/NvidiaDriverBridge.php index 595411bc..6664fc77 100644 --- a/bridges/NvidiaDriverBridge.php +++ b/bridges/NvidiaDriverBridge.php @@ -2,36 +2,83 @@ class NvidiaDriverBridge extends FeedExpander { - const NAME = 'NVIDIA Linux Driver Releases'; + const NAME = 'NVIDIA Driver Releases'; const URI = 'https://www.nvidia.com/Download/processFind.aspx'; - const DESCRIPTION = 'Fetch the latest NVIDIA Linux driver updates'; + const DESCRIPTION = 'Fetch the latest NVIDIA driver updates'; const MAINTAINER = 'tillcash'; + const PARAMETERS = [ - [ - 'whql' => [ - 'name' => 'Version', + 'Windows' => [ + 'wwhql' => [ + 'name' => 'Driver Type', 'type' => 'list', 'values' => [ - 'All' => '', - 'Beta' => '0', - 'New Feature Branch' => '5', - 'Recommended/Certified' => '1', + 'All' => '', + 'Certified' => '1', + 'Studio' => '4', ], + 'defaultValue' => '1', + ], + ], + 'Linux' => [ + 'lwhql' => [ + 'name' => 'Driver Type', + 'type' => 'list', + 'values' => [ + 'All' => '', + 'Beta' => '0', + 'Branch' => '5', + 'Certified' => '1', + ], + 'defaultValue' => '1', + ], + ], + 'FreeBSD' => [ + 'fwhql' => [ + 'name' => 'Driver Type', + 'type' => 'list', + 'values' => [ + 'All' => '', + 'Beta' => '0', + 'Branch' => '5', + 'Certified' => '1', + ], + 'defaultValue' => '1', ], ], ]; + private $operatingSystem = ''; + public function collectData() { - $whql = $this->getInput('whql'); - $parameters = [ 'lid' => 1, // en-us 'psid' => 129, // GeForce - 'osid' => 12, // Linux 64-bit - 'whql' => $whql, ]; + switch ($this->queriedContext) { + case 'Windows': + $whql = $this->getInput('wwhql'); + $parameters['osid'] = 57; + $parameters['dtcid'] = 1; // Windows Driver DCH + $parameters['whql'] = $whql; + $this->operatingSystem = 'Windows'; + break; + case 'Linux': + $whql = $this->getInput('lwwhql'); + $parameters['osid'] = 12; + $parameters['whql'] = $whql; + $this->operatingSystem = 'Linux'; + break; + case 'FreeBSD': + $whql = $this->getInput('fwwhql'); + $parameters['osid'] = 22; + $parameters['whql'] = $whql; + $this->operatingSystem = 'FreeBSD'; + break; + } + $url = 'https://www.nvidia.com/Download/processFind.aspx?' . http_build_query($parameters); $dom = getSimpleHTMLDOM($url); @@ -40,10 +87,21 @@ class NvidiaDriverBridge extends FeedExpander $this->items[] = [ 'timestamp' => $element->find('td.gridItem', 3)->plaintext, - 'title' => sprintf('NVIDIA Linux Driver %s', $element->find('td.gridItem', 2)->plaintext), + 'title' => sprintf('NVIDIA Driver %s', $element->find('td.gridItem', 2)->plaintext), 'uri' => 'https://www.nvidia.com/Download/driverResults.aspx/' . $id, 'content' => $dom->find('tr#tr_' . $id . ' span', 0)->innertext, ]; } } + + public function getIcon() + { + return 'https://www.nvidia.com/favicon.ico'; + } + + public function getName() + { + $version = $this->getKey('whql') ?? ''; + return sprintf('NVIDIA %s %s Driver Releases', $this->operatingSystem, $version); + } } From 2d5d2f5017e9ba75f562683ee84cff392b700d43 Mon Sep 17 00:00:00 2001 From: tillcash <tillcash@users.noreply.github.com> Date: Tue, 20 Aug 2024 21:02:15 +0530 Subject: [PATCH 173/423] [NvidiaDriverBridge] fix typo (#4224) --- bridges/NvidiaDriverBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/NvidiaDriverBridge.php b/bridges/NvidiaDriverBridge.php index 6664fc77..3c478697 100644 --- a/bridges/NvidiaDriverBridge.php +++ b/bridges/NvidiaDriverBridge.php @@ -66,13 +66,13 @@ class NvidiaDriverBridge extends FeedExpander $this->operatingSystem = 'Windows'; break; case 'Linux': - $whql = $this->getInput('lwwhql'); + $whql = $this->getInput('lwhql'); $parameters['osid'] = 12; $parameters['whql'] = $whql; $this->operatingSystem = 'Linux'; break; case 'FreeBSD': - $whql = $this->getInput('fwwhql'); + $whql = $this->getInput('fwhql'); $parameters['osid'] = 22; $parameters['whql'] = $whql; $this->operatingSystem = 'FreeBSD'; From 3a327503ee26bc8efeb1b3dd43ac4e1d58334229 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Wed, 21 Aug 2024 16:10:03 -0400 Subject: [PATCH 174/423] [NPRBridge] add bridge for NPR stories (#4225) * [NPRBridge] add bridge for NPR stories * [NPRBridge] Use better selectors for multiple items --- bridges/NPRBridge.php | 214 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 bridges/NPRBridge.php diff --git a/bridges/NPRBridge.php b/bridges/NPRBridge.php new file mode 100644 index 00000000..aca9f653 --- /dev/null +++ b/bridges/NPRBridge.php @@ -0,0 +1,214 @@ +<?php + +class NPRBridge extends FeedExpander +{ + const MAINTAINER = 'phantop'; + const NAME = 'NPR'; + const URI = 'https://www.npr.org/'; + const DESCRIPTION = 'Returns the latest articles from NPR'; + const PARAMETERS = [[ + 'section' => [ + 'name' => 'Site section', + 'type' => 'list', + 'defaultValue' => '1002', + // Obtained from https://legacy.npr.org/list?date=2024-05-05&id= + // With ids: 3002 (Topics), 3004 (Programs), 3006 (Series) + // Feeds cleaned up to exclude all that hadn't updated this year + 'values' => [ + 'All Things Considered' => '2', + 'Morning Edition' => '3', + 'Weekend Edition Saturday' => '7', + 'Weekend Edition Sunday' => '10', + 'Fresh Air' => '13', + 'Wait Wait...Don\'t Tell Me!' => '35', + 'TED Radio Hour' => '57', + 'News' => '1001', + 'Home Page Top Stories' => '1002', + 'National' => '1003', + 'World' => '1004', + 'Business' => '1006', + 'Science' => '1007', + 'Culture' => '1008', + 'Middle East' => '1009', + 'Education' => '1013', + 'Politics' => '1014', + 'Race' => '1015', + 'Religion' => '1016', + 'Economy' => '1017', + 'Your Money' => '1018', + 'Technology' => '1019', + 'Media' => '1020', + 'Research News' => '1024', + 'Environment' => '1025', + 'Space' => '1026', + 'Health Care' => '1027', + 'On Aging' => '1028', + 'Mental Health' => '1029', + 'Children\'s Health' => '1030', + 'Global Health' => '1031', + 'Books' => '1032', + 'Author Interviews' => '1033', + 'Book Reviews' => '1034', + 'Music' => '1039', + 'Movies' => '1045', + 'Performing Arts' => '1046', + 'Art & Design' => '1047', + 'Pop Culture' => '1048', + 'Humor & Fun' => '1052', + 'Food' => '1053', + 'Sports' => '1055', + 'Opinion' => '1057', + 'Analysis' => '1059', + 'Obituaries' => '1062', + 'Your Health' => '1066', + 'Law' => '1070', + 'Studio Sessions' => '1103', + 'Music Reviews' => '1104', + 'Music Interviews' => '1105', + 'Music News' => '1106', + 'Music Lists' => '1107', + 'New Music' => '1108', + 'Concerts' => '1109', + 'Music Videos' => '1110', + 'National Security' => '1122', + 'Europe' => '1124', + 'Asia' => '1125', + 'Africa' => '1126', + 'The Americas' => '1127', + 'Health' => '1128', + 'Energy' => '1131', + 'Animals' => '1132', + 'On Disabilities' => '1133', + 'Fitness & Nutrition' => '1134', + 'Medical Treatments' => '1135', + 'History' => '1136', + 'Movie Interviews' => '1137', + 'Television' => '1138', + 'Recipes' => '1139', + 'Fine Art' => '1141', + 'Architecture' => '1142', + 'Photography' => '1143', + 'Theater' => '1144', + 'Dance' => '1145', + 'Strange News' => '1146', + 'Investigations' => '1150', + 'Music Quizzes' => '1151', + 'Book News & Features' => '1161', + 'TV Reviews' => '1163', + 'Family' => '1164', + 'Weather' => '1165', + 'Perspective' => '1166', + 'Climate' => '1167', + 'Press Releases and Statements' => '750003', + 'Movie Reviews' => '4467349', + 'Sunday Puzzle' => '4473090', + 'Simon Says' => '4495795', + 'StoryCorps' => '4516989', + '\'Not My Job\'' => '5163715', + 'Tiny Desk' => '92071316', + 'Jazz' => '92756586', + 'Pop Culture Happy Hour' => '93568166', + 'Planet Money' => '94427042', + 'The Thistle & Shamrock' => '103063413', + 'Fresh Air Weekend' => '139029251', + 'Elections' => '139482413', + 'Presidential Race' => '139544303', + 'World Cafe: Sense Of Place' => '142680413', + 'Jazz Night In America' => '347139849', + 'Jazz Night In America: The Radio Program' => '347174538', + 'Planet Money Buys Gold' => '377029766', + 'Music Features' => '613820055', + 'Bill Of The Month' => '651784144', + 'Student Podcast Challenge' => '662609200', + 'Life Kit' => '676529561', + 'Picture This' => '787467815', + 'Gaming' => '820266919', + 'Games' => '820593993', + 'Health Reporting in the States' => '914131100', + 'Untangling Disinformation' => '973275370', + 'Pride Month' => '1002248299', + 'Planet Money Summer School' => '1015448333', + 'What\'s Making Us Happy' => '1019281468', + 'Native American Heritage Month' => '1047406725', + 'Podcast Recommendations' => '1068304478', + 'Tiny Desk Contest' => '1072544367', + 'Ukraine invasion — explained' => '1082539802', + 'Reproductive rights in America' => '1096684820', + 'My Unsung Hero' => '1134955065', + 'The NPR news quiz' => '1146192567', + 'Video Game Reviews' => '1175242824', + 'Gaming Culture' => '1175243560', + 'Up First Newsletter' => '1180232252', + 'Up First' => '1182407811', + 'Body Electric' => '1199526213', + 'Interview highlights' => '1200383155', + 'Middle East Crisis — explained' => '1205445976', + 'The Sunday Story from Up First' => '1213771050', + 'Life Kit\'s guide to emergency preparedness' => '1217925264', + 'Code Switch: Perspectives' => '1223739304', + 'How to Thrive as You Age' => '1225474023', + 'Time Machine: The Throughline History Quiz' => '1233646427', + 'We, The Voters' => '1241382501', + 'The Science of Siblings' => '1241438370', + 'Throughline: Constitutional Amendments' => '1242285011', + 'NPR Investigations: Off The Mark' => '1245316423', + 'Campus protests over the Gaza war' => '1248184956', + 'UAW Goes South' => '1250012704', + 'Books We Love' => '1251857292', + 'NPR\'s Embedded: Supermajority ' => '1254807812', + 'Throughline: The Middle East Conflict' => '1255058395', + ] + ] + ]]; + + public function getIcon() + { + return 'https://media.npr.org/chrome/favicon/favicon.ico'; + } + + public function collectData() + { + $url = 'https://feeds.npr.org/' . $this->getInput('section') . '/rss.xml'; + $this->collectExpandableDatas($url, 10); + } + + protected function parseItem(array $item) + { + $html = getSimpleHTMLDOMCached($item['uri']); + $html = defaultLinkTo($html, self::URI); + $text = $html->find('#storytext', 0); + + // a bit of a cheat to offer the text-only alternative url + $item['comments'] = preg_replace('/www/', 'text', $item['uri']); + + // clean up related articles, duplicate image credit and enlarged versions + $ads = 'aside.ad-wrap, span.credit, .bucket.img'; + $enlarge = '.enlarge-options, .enlarge_measure, .enlarge_html'; + foreach ($text->find("$ads, $enlarge") as $ad) { + $ad->remove(); + } + + $item['content'] = preg_replace('/(hide|toggle) caption/', '', $text); + + // get tags, program/series names + $item['categories'] = []; + $tags = '.tag, .program-block > a, .branding__title'; + foreach ($html->find($tags) as $tag) { + $item['categories'][] = $tag->innertext; + } + $item['categories'] = array_unique($item['categories']); + + // fetch audios and transcripts + $item['enclosures'] = []; + foreach ($html->find('.audio-tool > a') as $audio) { + $item['enclosures'][] = $audio->href; + } + foreach ($html->find('[data-audio]') as $audio) { + $json_text = $audio->getAttribute('data-audio'); + $json = Json::decode(html_entity_decode($json_text), true); + $item['enclosures'][] = base64_decode($json['audioUrl']); + } + + return $item; + } +} From d379f3e575e9b84f3e65e7571532f11a6d298279 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Wed, 21 Aug 2024 16:57:02 -0400 Subject: [PATCH 175/423] [CubariProxyBridge] add bridge for cubari manga proxies (#4220) * [CubariProxyBridge] add bridge for cubari manga proxies * [CubariProxyBridge] add limit and use isset --- bridges/CubariProxyBridge.php | 124 ++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 bridges/CubariProxyBridge.php diff --git a/bridges/CubariProxyBridge.php b/bridges/CubariProxyBridge.php new file mode 100644 index 00000000..492d5fbb --- /dev/null +++ b/bridges/CubariProxyBridge.php @@ -0,0 +1,124 @@ +<?php + +class CubariProxyBridge extends BridgeAbstract +{ + const NAME = 'Cubari Proxy'; + const MAINTAINER = 'phantop'; + const URI = 'https://cubari.moe'; + const DESCRIPTION = 'Returns chapters from Cubari.'; + const PARAMETERS = [[ + 'service' => [ + 'name' => 'Content service', + 'type' => 'list', + 'defaultValue' => 'mangadex', + 'values' => [ + 'MangAventure' => 'mangadventure', + 'MangaDex' => 'mangadex', + 'MangaKatana' => 'mangakatana', + 'MangaSee' => 'mangasee', + ] + ], + 'series' => [ + 'name' => 'Series ID/Name', + 'exampleValue' => '8c1d7d0c-e0b7-4170-941d-29f652c3c19d', # KnH + 'required' => true, + ], + 'fetch' => [ + 'name' => 'Fetch chapter page images', + 'type' => 'list', + 'title' => 'Places chapter images in feed contents. Entries will consume more bandwidth.', + 'defaultValue' => 'c', + 'values' => [ + 'None' => 'n', + 'Content' => 'c', + 'Enclosure' => 'e' + ] + ], + 'limit' => self::LIMIT + ]]; + + private $title; + + public function collectData() + { + $limit = $this->getInput('limit') ?? 10; + + $url = parent::getURI() . '/read/api/' . $this->getInput('service') . '/series/' . $this->getInput('series'); + $json = Json::decode(getContents($url)); + $this->title = $json['title']; + + $chapters = $json['chapters']; + krsort($chapters); + + $count = 0; + foreach ($chapters as $number => $element) { + $item = []; + $item['uri'] = $this->getURI() . '/' . $number; + + if ($element['title']) { + $item['title'] = $number . ' - ' . $element['title']; + } else { + $item['title'] = 'Volume ' . $element['volume'] . ' Chapter ' . $number; + } + + $group = '1'; + if (isset($element['release_date'])) { + $dates = $element['release_date']; + $date = max($dates); + $item['timestamp'] = $date; + $group = array_keys($dates, $date)[0]; + } + $page = $element['groups'][$group]; + $item['author'] = $json['groups'][$group]; + $api = parent::getURI() . $page; + $item['uid'] = $page; + $item['comments'] = $api; + + if ($this->getInput('fetch') != 'n') { + $pages = []; + try { + $jsonp = getContents($api); + $pages = Json::decode($jsonp); + } catch (HttpException $e) { + // allow error 500, as it's effectively a 429 + if ($e->getCode() != 500) { + throw $e; + } + } + if ($this->getInput('fetch') == 'e') { + $item['enclosures'] = $pages; + } + if ($this->getInput('fetch') == 'c') { + $item['content'] = ''; + foreach ($pages as $img) { + $item['content'] .= '<img src="' . $img . '"/>'; + } + } + } + + if ($count++ == $limit) { + break; + } + + $this->items[] = $item; + } + } + + public function getName() + { + $name = parent::getName(); + if (isset($this->title)) { + $name .= ' - ' . $this->title; + } + return $name; + } + + public function getURI() + { + $uri = parent::getURI(); + if ($this->getInput('service')) { + $uri .= '/read/' . $this->getInput('service') . '/' . $this->getInput('series'); + } + return $uri; + } +} From 06a88960002cd8b8dba10b06bf0af1e71cecf512 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Wed, 21 Aug 2024 16:58:26 -0400 Subject: [PATCH 176/423] [PriviblurBridge] Add Priviblur (Tumblr frontend) bridge (#4221) * [PriviblurBridge] Add Priviblur (Tumblr frontend) bridge * [PriviblurBridge] prevent error if post has no tags --- bridges/PriviblurBridge.php | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 bridges/PriviblurBridge.php diff --git a/bridges/PriviblurBridge.php b/bridges/PriviblurBridge.php new file mode 100644 index 00000000..198c38bc --- /dev/null +++ b/bridges/PriviblurBridge.php @@ -0,0 +1,69 @@ +<?php + +class PriviblurBridge extends BridgeAbstract +{ + const NAME = 'Priviblur'; + const MAINTAINER = 'phantop'; + const URI = 'https://github.com/syeopite/priviblur'; + const DESCRIPTION = 'Returns Tumblr posts from a Priviblur link'; + const PARAMETERS = [ + [ + 'url' => [ + 'name' => 'URL', + 'exampleValue' => 'https://priviblur.fly.dev', + 'required' => true, + ] + ] + ]; + + private $title; + + public function collectData() + { + $url = $this->getURI(); + $html = getSimpleHTMLDOM($url); + $html = defaultLinkTo($html, $url); + $this->title = $html->find('head title', 0)->innertext; + + $elements = $html->find('.post'); + foreach ($elements as $element) { + $item = []; + $item['author'] = $element->find('.primary-post-author .blog-name', 0)->innertext; + $item['comments'] = $element->find('.interaction-buttons > a', 1)->href; + $item['content'] = $element->find('.post-body', 0); + $item['timestamp'] = $element->find('.primary-post-author time', 0)->innertext; + $item['title'] = $item['author'] . ': ' . $item['timestamp']; + $item['uid'] = $item['comments']; // tumblr url is canonical + $item['uri'] = $element->find('.interaction-buttons > a', 0)->href; + + if ($element->find('.post-tags', 0)) { + $tags = html_entity_decode($element->find('.post-tags', 0)->plaintext); + $tags = explode('#', $tags); + $tags = array_map('trim', $tags); + array_shift($tags); + $item['categories'] = $tags; + } + + $heading = $element->find('h1', 0); + if ($heading) { + $item['title'] = $heading->innertext; + } + + $this->items[] = $item; + } + } + + public function getName() + { + $name = parent::getName(); + if (isset($this->title)) { + $name = $this->title; + } + return $name; + } + + public function getURI() + { + return $this->getInput('url') ? $this->getInput('url') : parent::getURI(); + } +} From 4a3919c1a35083ade3dca0a49dd9c78cd8fc4f28 Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Wed, 21 Aug 2024 17:05:29 -0400 Subject: [PATCH 177/423] [NPRBridge] Add missing tag and remove extra HTML elements (#4227) --- bridges/NPRBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/NPRBridge.php b/bridges/NPRBridge.php index aca9f653..644ede68 100644 --- a/bridges/NPRBridge.php +++ b/bridges/NPRBridge.php @@ -192,9 +192,9 @@ class NPRBridge extends FeedExpander // get tags, program/series names $item['categories'] = []; - $tags = '.tag, .program-block > a, .branding__title'; + $tags = '.tag, .program-block > a, .branding__title, article h3.slug'; foreach ($html->find($tags) as $tag) { - $item['categories'][] = $tag->innertext; + $item['categories'][] = $tag->plaintext; } $item['categories'] = array_unique($item['categories']); From 05e2c350b7de8480a1a36a7b4ec284c7479adfb5 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 22 Aug 2024 00:33:35 +0200 Subject: [PATCH 178/423] refactor: less reliance on super globals (#4228) --- actions/DisplayAction.php | 12 +++++++----- formats/HtmlFormat.php | 4 ++++ index.php | 10 +++++++++- lib/RssBridge.php | 9 +-------- templates/base.html.php | 2 +- templates/html-format.html.php | 2 +- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index d111e69e..26f1cb40 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -21,12 +21,13 @@ class DisplayAction implements ActionInterface /** @var Response $cachedResponse */ $cachedResponse = $this->cache->get($cacheKey); if ($cachedResponse) { - $ifModifiedSince = $_SERVER['HTTP_IF_MODIFIED_SINCE'] ?? null; + $ifModifiedSince = $request->server('HTTP_IF_MODIFIED_SINCE'); $lastModified = $cachedResponse->getHeader('last-modified'); if ($ifModifiedSince && $lastModified) { $lastModified = new \DateTimeImmutable($lastModified); $lastModifiedTimestamp = $lastModified->getTimestamp(); $modifiedSince = strtotime($ifModifiedSince); + // TODO: \DateTimeImmutable can be compared directly if ($lastModifiedTimestamp <= $modifiedSince) { $modificationTimeGMT = gmdate('D, d M Y H:i:s ', $lastModifiedTimestamp); return new Response('', 304, ['last-modified' => $modificationTimeGMT . 'GMT']); @@ -182,7 +183,7 @@ class DisplayAction implements ActionInterface $content = render_template(__DIR__ . '/../templates/bridge-error.html.php', [ 'error' => render_template(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 'searchUrl' => self::createGithubSearchUrl($bridge), - 'issueUrl' => self::createGithubIssueUrl($bridge, $e, create_sane_exception_message($e)), + 'issueUrl' => self::createGithubIssueUrl($bridge, $e), 'maintainer' => $bridge->getMaintainer(), ]); $item['content'] = $content; @@ -211,7 +212,7 @@ class DisplayAction implements ActionInterface return $report['count']; } - private static function createGithubIssueUrl(BridgeAbstract $bridge, \Throwable $e, string $message): string + private static function createGithubIssueUrl(BridgeAbstract $bridge, \Throwable $e): string { $maintainer = $bridge->getMaintainer(); if (str_contains($maintainer, ',')) { @@ -221,13 +222,14 @@ class DisplayAction implements ActionInterface } $maintainers = array_map('trim', $maintainers); + $queryString = $_SERVER['QUERY_STRING'] ?? ''; $query = [ 'title' => $bridge->getName() . ' failed with: ' . $e->getMessage(), 'body' => sprintf( "```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```\nMaintainer: @%s", - $message, + create_sane_exception_message($e), implode("\n", trace_to_call_points(trace_from_exception($e))), - $_SERVER['QUERY_STRING'] ?? '', + $queryString, Configuration::getVersion(), PHP_OS_FAMILY, phpversion() ?: 'Unknown', diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index 37ef3a93..a5bcc451 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -9,6 +9,9 @@ class HtmlFormat extends FormatAbstract // This query string is url encoded $queryString = $_SERVER['QUERY_STRING']; + // TODO: this should be the proper bridge short name and not user provided string + $bridgeName = $_GET['bridge']; + $feedArray = $this->getFeed(); $formatFactory = new FormatFactory(); $formats = []; @@ -48,6 +51,7 @@ class HtmlFormat extends FormatAbstract } $html = render_template(__DIR__ . '/../templates/html-format.html.php', [ + 'bridge_name' => $bridgeName, 'charset' => $this->getCharset(), 'title' => $feedArray['name'], 'formats' => $formats, diff --git a/index.php b/index.php index 1efda44a..c4fe104f 100644 --- a/index.php +++ b/index.php @@ -76,9 +76,17 @@ $httpClient = new CurlHttpClient(); date_default_timezone_set(Configuration::getConfig('system', 'timezone')); +$argv = $argv ?? null; +if ($argv) { + parse_str(implode('&', array_slice($argv, 1)), $cliArgs); + $request = Request::fromCli($cliArgs); +} else { + $request = Request::fromGlobals(); +} + try { $rssBridge = new RssBridge($logger, $cache, $httpClient); - $response = $rssBridge->main($argv ?? []); + $response = $rssBridge->main($request); $response->send(); } catch (\Throwable $e) { // Probably an exception inside an action diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 23f65cf0..35318c5b 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -16,15 +16,8 @@ final class RssBridge self::$httpClient = $httpClient; } - public function main(array $argv = []): Response + public function main(Request $request): Response { - if ($argv) { - parse_str(implode('&', array_slice($argv, 1)), $cliArgs); - $request = Request::fromCli($cliArgs); - } else { - $request = Request::fromGlobals(); - } - foreach ($request->toArray() as $key => $value) { if (!is_string($value)) { return new Response(render(__DIR__ . '/../templates/error.html.php', [ diff --git a/templates/base.html.php b/templates/base.html.php index d2557599..a8ff7660 100644 --- a/templates/base.html.php +++ b/templates/base.html.php @@ -4,7 +4,7 @@ <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="description" content="RSS-Bridge" /> - <title><?= e($_title ?? 'RSS-Bridge') ?> + RSS-Bridge diff --git a/templates/html-format.html.php b/templates/html-format.html.php index bc95c5d0..a05acdde 100644 --- a/templates/html-format.html.php +++ b/templates/html-format.html.php @@ -30,7 +30,7 @@
    - + From b0674d7b19f2bba901e8aab75760c021d0495e9d Mon Sep 17 00:00:00 2001 From: Clemens Neubauer Date: Thu, 22 Aug 2024 11:36:58 +0200 Subject: [PATCH 179/423] [BMDSystemhausBlogBridge] rework detectParameters (#4138) * bridge BMDSystemhausBlog: rework of detectParameters * fix lint phpcs error * Update BMDSystemhausBlogBridge.php * Update BMDSystemhausBlogBridge.php --- bridges/BMDSystemhausBlogBridge.php | 55 +++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/bridges/BMDSystemhausBlogBridge.php b/bridges/BMDSystemhausBlogBridge.php index c80f3ff3..12f3ca5e 100644 --- a/bridges/BMDSystemhausBlogBridge.php +++ b/bridges/BMDSystemhausBlogBridge.php @@ -8,6 +8,7 @@ class BMDSystemhausBlogBridge extends BridgeAbstract const URI = 'https://www.bmd.com'; const DONATION_URI = 'https://paypal.me/cntools'; const DESCRIPTION = 'BMD Systemhaus - We make business easy'; + const BMD_FAV_ICON = 'https://www.bmd.com/favicon.ico'; const ITEMSTYLE = [ 'ilcr' => '
    {data_img}{data_content}
    ', @@ -148,18 +149,58 @@ class BMDSystemhausBlogBridge extends BridgeAbstract return null; } - if ($parsedUrl->getHost() != 'www.bmd.com') { + if (!in_array($parsedUrl->getHost(), ['www.bmd.com', 'bmd.com'])) { return null; } - $path = explode('/', $parsedUrl->getPath()); + $lang = ''; - if ($this->getURIbyCountry($path[1]) == '') { - return null; + // extract language from url + $path = explode('/', $parsedUrl->getPath()); + if (count($path) > 1) { + $lang = $path[1]; + + // validate data + if ($this->getURIbyCountry($lang) == '') { + $lang = ''; + } + } + + // if no country available, find language by browser + if ($lang == '') { + $srvLanguages = explode(';', $_SERVER['HTTP_ACCEPT_LANGUAGE']); + if (count($srvLanguages) > 0) { + $languages = explode(',', $srvLanguages[0]); + if (count($languages) > 0) { + for ($i = 0; $i < count($languages); $i++) { + $langDetails = explode('-', $languages[$i]); + if (count($langDetails) > 1) { + $lang = $langDetails[1]; + } else { + $lang = substr($srvLanguages[0], 0, 2); + } + + // validate data + if ($this->getURIbyCountry($lang) == '') { + $lang = ''; + } + + if ($lang != '') { + break; + } + } + } + } + } + + // if no URL found by language, use AT as default + if ($this->getURIbyCountry($lang) == '') { + $lang = 'at'; } $params = []; - $params['country'] = $path[1]; + $params['country'] = strtolower($lang); + return $params; } @@ -173,7 +214,7 @@ class BMDSystemhausBlogBridge extends BridgeAbstract //----------------------------------------------------- public function getIcon() { - return 'https://www.bmd.com/favicon.ico'; + return self::BMD_FAV_ICON; } //----------------------------------------------------- @@ -192,7 +233,7 @@ class BMDSystemhausBlogBridge extends BridgeAbstract //----------------------------------------------------- private function getURIbyCountry($country) { - switch ($country) { + switch (strtolower($country)) { case 'at': return 'https://www.bmd.com/at/ueber-bmd/blog-ohne-filter.html'; case 'de': From c849576c936bf25982fe91f531301d57d787b285 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 23 Aug 2024 17:09:17 +0200 Subject: [PATCH 180/423] fix(rumble): fix guid bug (#4232) Remove tracking parameter in query to avoid feed readers to interpret these as new items --- bridges/RumbleBridge.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bridges/RumbleBridge.php b/bridges/RumbleBridge.php index a8841e00..8d92db3b 100644 --- a/bridges/RumbleBridge.php +++ b/bridges/RumbleBridge.php @@ -60,12 +60,19 @@ class RumbleBridge extends BridgeAbstract $dom = getSimpleHTMLDOM($url); foreach ($dom->find('ol.thumbnail__grid div.thumbnail__grid--item') as $video) { + $itemUrlString = self::URI . $video->find('a', 0)->href; + $itemUrl = Url::fromString($itemUrlString); + $item = [ 'title' => $video->find('h3', 0)->plaintext, - 'uri' => self::URI . $video->find('a', 0)->href, + + // Remove tracking parameter in query string + 'uri' => $itemUrl->withQueryString(null)->__toString(), + 'author' => $account . '@rumble.com', 'content' => defaultLinkTo($video, self::URI)->innertext, ]; + $time = $video->find('time', 0); if ($time) { $publishedAt = new \DateTimeImmutable($time->getAttribute('datetime')); From 6516e31c1bc78867820fead69ad65ba4c1d350ea Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 23 Aug 2024 17:34:06 +0200 Subject: [PATCH 181/423] refactor: format rendering (#4229) --- actions/DisplayAction.php | 10 ++++++++-- formats/AtomFormat.php | 10 ++-------- formats/HtmlFormat.php | 6 +----- formats/JsonFormat.php | 2 +- formats/MrssFormat.php | 7 ++----- formats/PlaintextFormat.php | 5 +---- formats/SfeedFormat.php | 9 +-------- index.php | 7 +++++-- lib/FormatAbstract.php | 13 +------------ templates/html-format.html.php | 2 +- tests/FormatTest.php | 2 +- tests/Formats/BaseFormatTest.php | 2 +- 12 files changed, 25 insertions(+), 50 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 26f1cb40..5265abd8 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -160,9 +160,15 @@ class DisplayAction implements ActionInterface $format->setLastModified($now); $headers = [ 'last-modified' => gmdate('D, d M Y H:i:s ', $now) . 'GMT', - 'content-type' => $format->getMimeType() . '; charset=' . $format->getCharset(), + 'content-type' => $format->getMimeType() . '; charset=UTF-8', ]; - return new Response($format->stringify(), 200, $headers); + $body = $format->render(); + + // This is supposed to remove non-utf8 byte sequences, but I'm unsure if it works + ini_set('mbstring.substitute_character', 'none'); + $body = mb_convert_encoding($body, 'UTF-8', 'UTF-8'); + + return new Response($body, 200, $headers); } private function createFeedItemFromException($e, BridgeAbstract $bridge): array diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index 5c9f2b6a..7908eb90 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -14,9 +14,9 @@ class AtomFormat extends FormatAbstract protected const ATOM_NS = 'http://www.w3.org/2005/Atom'; protected const MRSS_NS = 'http://search.yahoo.com/mrss/'; - public function stringify() + public function render(): string { - $document = new \DomDocument('1.0', $this->getCharset()); + $document = new \DomDocument('1.0', 'UTF-8'); $document->formatOutput = true; $feedUrl = get_current_url(); @@ -86,8 +86,6 @@ class AtomFormat extends FormatAbstract $author->appendChild($authorName); $authorName->appendChild($document->createTextNode($feedAuthor)); - - foreach ($this->getItems() as $item) { $itemArray = $item->toArray(); $entryTimestamp = $item->getTimestamp(); @@ -204,10 +202,6 @@ class AtomFormat extends FormatAbstract } $xml = $document->saveXML(); - - // Remove invalid characters - ini_set('mbstring.substitute_character', 'none'); - $xml = mb_convert_encoding($xml, $this->getCharset(), 'UTF-8'); return $xml; } } diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index a5bcc451..04721ae1 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -4,7 +4,7 @@ class HtmlFormat extends FormatAbstract { const MIME_TYPE = 'text/html'; - public function stringify() + public function render(): string { // This query string is url encoded $queryString = $_SERVER['QUERY_STRING']; @@ -52,16 +52,12 @@ class HtmlFormat extends FormatAbstract $html = render_template(__DIR__ . '/../templates/html-format.html.php', [ 'bridge_name' => $bridgeName, - 'charset' => $this->getCharset(), 'title' => $feedArray['name'], 'formats' => $formats, 'uri' => $feedArray['uri'], 'items' => $items, 'donation_uri' => $donationUri, ]); - // Remove invalid characters - ini_set('mbstring.substitute_character', 'none'); - $html = mb_convert_encoding($html, $this->getCharset(), 'UTF-8'); return $html; } } diff --git a/formats/JsonFormat.php b/formats/JsonFormat.php index 586aae0a..3548ef6e 100644 --- a/formats/JsonFormat.php +++ b/formats/JsonFormat.php @@ -23,7 +23,7 @@ class JsonFormat extends FormatAbstract 'uid', ]; - public function stringify() + public function render(): string { $feedArray = $this->getFeed(); diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index aaa1d0cd..f7b11949 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -32,9 +32,9 @@ class MrssFormat extends FormatAbstract protected const ATOM_NS = 'http://www.w3.org/2005/Atom'; protected const MRSS_NS = 'http://search.yahoo.com/mrss/'; - public function stringify() + public function render(): string { - $document = new \DomDocument('1.0', $this->getCharset()); + $document = new \DomDocument('1.0', 'UTF-8'); $document->formatOutput = true; $feed = $document->createElement('rss'); @@ -198,9 +198,6 @@ class MrssFormat extends FormatAbstract } $xml = $document->saveXML(); - // Remove invalid non-UTF8 characters - ini_set('mbstring.substitute_character', 'none'); - $xml = mb_convert_encoding($xml, $this->getCharset(), 'UTF-8'); return $xml; } } diff --git a/formats/PlaintextFormat.php b/formats/PlaintextFormat.php index 4e18caa6..e93c94b5 100644 --- a/formats/PlaintextFormat.php +++ b/formats/PlaintextFormat.php @@ -4,16 +4,13 @@ class PlaintextFormat extends FormatAbstract { const MIME_TYPE = 'text/plain'; - public function stringify() + public function render(): string { $feed = $this->getFeed(); foreach ($this->getItems() as $item) { $feed['items'][] = $item->toArray(); } $text = print_r($feed, true); - // Remove invalid non-UTF8 characters - ini_set('mbstring.substitute_character', 'none'); - $text = mb_convert_encoding($text, $this->getCharset(), 'UTF-8'); return $text; } } diff --git a/formats/SfeedFormat.php b/formats/SfeedFormat.php index 33740aaa..063e4543 100644 --- a/formats/SfeedFormat.php +++ b/formats/SfeedFormat.php @@ -4,7 +4,7 @@ class SfeedFormat extends FormatAbstract { const MIME_TYPE = 'text/plain'; - public function stringify() + public function render(): string { $text = ''; foreach ($this->getItems() as $item) { @@ -26,13 +26,6 @@ class SfeedFormat extends FormatAbstract ); } - // Remove invalid non-UTF8 characters - ini_set('mbstring.substitute_character', 'none'); - $text = mb_convert_encoding( - $text, - $this->getCharset(), - 'UTF-8' - ); return $text; } diff --git a/index.php b/index.php index c4fe104f..2a613363 100644 --- a/index.php +++ b/index.php @@ -62,8 +62,11 @@ register_shutdown_function(function () use ($logger) { $cacheFactory = new CacheFactory($logger); -// Uncomment this for debug logging -// $logger->addHandler(new StreamHandler('/tmp/rss-bridge.txt', Logger::DEBUG)); +// Uncomment this for info logging to fs +// $logger->addHandler(new StreamHandler('/tmp/rss-bridge.txt', Logger::INFO)); + +// Uncomment this for debug logging to fs +// $logger->addHandler(new StreamHandler('/tmp/rss-bridge-debug.txt', Logger::DEBUG)); if (Debug::isEnabled()) { $logger->addHandler(new ErrorLogHandler(Logger::DEBUG)); diff --git a/lib/FormatAbstract.php b/lib/FormatAbstract.php index 9cba0d8c..17e733a7 100644 --- a/lib/FormatAbstract.php +++ b/lib/FormatAbstract.php @@ -8,11 +8,10 @@ abstract class FormatAbstract protected array $feed = []; protected array $items = []; - protected string $charset = 'UTF-8'; protected int $lastModified; - abstract public function stringify(); + abstract public function render(): string; public function setFeed(array $feed) { @@ -50,16 +49,6 @@ abstract class FormatAbstract return static::MIME_TYPE; } - public function setCharset(string $charset) - { - $this->charset = $charset; - } - - public function getCharset(): string - { - return $this->charset; - } - public function setLastModified(int $lastModified) { $this->lastModified = $lastModified; diff --git a/templates/html-format.html.php b/templates/html-format.html.php index a05acdde..8e8abfcf 100644 --- a/templates/html-format.html.php +++ b/templates/html-format.html.php @@ -1,7 +1,7 @@ - + <?= e($title) ?> diff --git a/tests/FormatTest.php b/tests/FormatTest.php index b5df395c..33e02769 100644 --- a/tests/FormatTest.php +++ b/tests/FormatTest.php @@ -58,7 +58,7 @@ class FormatTest extends TestCase class TestFormat extends \FormatAbstract { - public function stringify() + public function render(): string { } } diff --git a/tests/Formats/BaseFormatTest.php b/tests/Formats/BaseFormatTest.php index 0907b72a..4b6a549b 100644 --- a/tests/Formats/BaseFormatTest.php +++ b/tests/Formats/BaseFormatTest.php @@ -64,6 +64,6 @@ abstract class BaseFormatTest extends TestCase $format->setFeed($sample->meta); $format->setLastModified(strtotime('2000-01-01 12:00:00 UTC')); - return $format->stringify(); + return $format->render(); } } From d51cc8f1a7e17cc07cecc1979beae9618902997f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kol=C3=A1=C5=99?= Date: Wed, 28 Aug 2024 19:43:40 +0200 Subject: [PATCH 182/423] Fixed path in CeskaTelevizeBridge (#4236) --- bridges/CeskaTelevizeBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/CeskaTelevizeBridge.php b/bridges/CeskaTelevizeBridge.php index be00d664..026e8c2a 100644 --- a/bridges/CeskaTelevizeBridge.php +++ b/bridges/CeskaTelevizeBridge.php @@ -60,7 +60,7 @@ class CeskaTelevizeBridge extends BridgeAbstract foreach ($html->find('#episodeListSection a[data-testid=card]') as $element) { $itemTitle = $element->find('h3', 0); $itemContent = $element->find('p[class^=content-]', 0); - $itemDate = $element->find('div[class^=playTime-] span', 0); + $itemDate = $element->find('div[class^=playTime-] span, [data-testid=episode-item-broadcast] span', 0); $itemThumbnail = $element->find('img', 0); $itemUri = self::URI . $element->getAttribute('href'); From e010fd4d52e3273d6ba2d2d6f45e5a58880ba043 Mon Sep 17 00:00:00 2001 From: tillcash Date: Wed, 28 Aug 2024 23:15:54 +0530 Subject: [PATCH 183/423] [HinduTamilBridge] fix image (#4237) --- bridges/HinduTamilBridge.php | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/bridges/HinduTamilBridge.php b/bridges/HinduTamilBridge.php index d12f5131..50b9b8e6 100644 --- a/bridges/HinduTamilBridge.php +++ b/bridges/HinduTamilBridge.php @@ -4,6 +4,7 @@ class HinduTamilBridge extends FeedExpander { const NAME = 'HinduTamil'; const URI = 'https://www.hindutamil.in'; + const FEED_BASE_URL = 'https://feeds.feedburner.com/Hindu_Tamil_'; const DESCRIPTION = 'Retrieve full articles from hindutamil.in feeds'; const MAINTAINER = 'tillcash'; const PARAMETERS = [ @@ -45,8 +46,6 @@ class HinduTamilBridge extends FeedExpander ], ]; - const FEED_BASE_URL = 'https://feeds.feedburner.com/Hindu_Tamil_'; - public function getName() { $topic = $this->getKey('topic'); @@ -69,34 +68,30 @@ class HinduTamilBridge extends FeedExpander return $item; } - $date = $dom->find('p span.date', 1); - if ($date) { - $item['timestamp'] = $this->toRFC3339($date->plaintext); - } - - $image = $dom->find('#LoadArticle figure', 0) ?? ''; - $item['content'] = $image . $this->cleanContent($content); + $item['timestamp'] = $this->getTimestamp($dom) ?? $item['timestamp']; + $item['content'] = $this->getImage($dom) . $this->cleanContent($content); return $item; } - private function cleanContent($content) + private function cleanContent($content): string { - foreach ($content->find('div[align="center"], script') as $remove) { + foreach ($content->find('div[align="center"], script, .adsplacement') as $remove) { $remove->outertext = ''; } - return $content; + return $content->innertext; } - private function toRFC3339($dateString) + private function getTimestamp($dom): ?string { - $timestamp = strtotime(trim($dateString)); + $date = $dom->find('meta[property="article:published_time"]', 0); + return $date ? $date->getAttribute('content') : null; + } - if ($timestamp === false) { - return null; - } - - return date('Y-m-d\TH:i:s', $timestamp) . '+05:30'; + private function getImage($dom): string + { + $image = $dom->find('meta[property="og:image"]', 0); + return $image ? sprintf('

    ', $image->getAttribute('content')) : ''; } } From 58544cd61a465102bebc315b4467e4e45d587723 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 29 Aug 2024 22:48:59 +0200 Subject: [PATCH 184/423] refactor: introduce DI container (#4238) * refactor: introduce DI container * add bin/test --- actions/ConnectivityAction.php | 7 +-- actions/DetectAction.php | 16 ++++--- actions/DisplayAction.php | 20 +++++---- actions/FindfeedAction.php | 16 ++++--- actions/FrontpageAction.php | 17 +++++--- actions/ListAction.php | 16 ++++--- bin/cache-clear | 7 +-- bin/cache-prune | 7 +-- bin/test | 29 +++++++++++++ bridges/TwitchBridge.php | 2 +- index.php | 23 ++-------- lib/BridgeCard.php | 9 ++-- lib/BridgeFactory.php | 10 +++-- lib/CacheFactory.php | 4 -- lib/Container.php | 33 ++++++++++++++ lib/RssBridge.php | 23 ++++------ lib/dependencies.php | 78 ++++++++++++++++++++++++++++++++++ lib/logger.php | 3 +- 18 files changed, 231 insertions(+), 89 deletions(-) create mode 100755 bin/test create mode 100644 lib/Container.php create mode 100644 lib/dependencies.php diff --git a/actions/ConnectivityAction.php b/actions/ConnectivityAction.php index 9732d0aa..e4e1e7c2 100644 --- a/actions/ConnectivityAction.php +++ b/actions/ConnectivityAction.php @@ -14,9 +14,10 @@ class ConnectivityAction implements ActionInterface { private BridgeFactory $bridgeFactory; - public function __construct() - { - $this->bridgeFactory = new BridgeFactory(); + public function __construct( + BridgeFactory $bridgeFactory + ) { + $this->bridgeFactory = $bridgeFactory; } public function __invoke(Request $request): Response diff --git a/actions/DetectAction.php b/actions/DetectAction.php index cebbc307..8d3d6263 100644 --- a/actions/DetectAction.php +++ b/actions/DetectAction.php @@ -2,6 +2,14 @@ class DetectAction implements ActionInterface { + private BridgeFactory $bridgeFactory; + + public function __construct( + BridgeFactory $bridgeFactory + ) { + $this->bridgeFactory = $bridgeFactory; + } + public function __invoke(Request $request): Response { $url = $request->get('url'); @@ -14,14 +22,12 @@ class DetectAction implements ActionInterface return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format'])); } - $bridgeFactory = new BridgeFactory(); - - foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) { - if (!$bridgeFactory->isEnabled($bridgeClassName)) { + foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) { + if (!$this->bridgeFactory->isEnabled($bridgeClassName)) { continue; } - $bridge = $bridgeFactory->create($bridgeClassName); + $bridge = $this->bridgeFactory->create($bridgeClassName); $bridgeParams = $bridge->detectParameters($url); diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 5265abd8..c00c0d5e 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -4,11 +4,16 @@ class DisplayAction implements ActionInterface { private CacheInterface $cache; private Logger $logger; + private BridgeFactory $bridgeFactory; - public function __construct() - { - $this->cache = RssBridge::getCache(); - $this->logger = RssBridge::getLogger(); + public function __construct( + CacheInterface $cache, + Logger $logger, + BridgeFactory $bridgeFactory + ) { + $this->cache = $cache; + $this->logger = $logger; + $this->bridgeFactory = $bridgeFactory; } public function __invoke(Request $request): Response @@ -39,8 +44,7 @@ class DisplayAction implements ActionInterface if (!$bridgeName) { return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge parameter']), 400); } - $bridgeFactory = new BridgeFactory(); - $bridgeClassName = $bridgeFactory->createBridgeClassName($bridgeName); + $bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName); if (!$bridgeClassName) { return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Bridge not found']), 404); } @@ -48,7 +52,7 @@ class DisplayAction implements ActionInterface if (!$format) { return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']), 400); } - if (!$bridgeFactory->isEnabled($bridgeClassName)) { + if (!$this->bridgeFactory->isEnabled($bridgeClassName)) { return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400); } @@ -62,7 +66,7 @@ class DisplayAction implements ActionInterface define('NOPROXY', true); } - $bridge = $bridgeFactory->create($bridgeClassName); + $bridge = $this->bridgeFactory->create($bridgeClassName); $response = $this->createResponse($request, $bridge, $format); diff --git a/actions/FindfeedAction.php b/actions/FindfeedAction.php index 6654ca6d..e18c3e1d 100644 --- a/actions/FindfeedAction.php +++ b/actions/FindfeedAction.php @@ -7,6 +7,14 @@ */ class FindfeedAction implements ActionInterface { + private BridgeFactory $bridgeFactory; + + public function __construct( + BridgeFactory $bridgeFactory + ) { + $this->bridgeFactory = $bridgeFactory; + } + public function __invoke(Request $request): Response { $url = $request->get('url'); @@ -19,15 +27,13 @@ class FindfeedAction implements ActionInterface return new Response('You must specify a format', 400); } - $bridgeFactory = new BridgeFactory(); - $results = []; - foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) { - if (!$bridgeFactory->isEnabled($bridgeClassName)) { + foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) { + if (!$this->bridgeFactory->isEnabled($bridgeClassName)) { continue; } - $bridge = $bridgeFactory->create($bridgeClassName); + $bridge = $this->bridgeFactory->create($bridgeClassName); $bridgeParams = $bridge->detectParameters($url); diff --git a/actions/FrontpageAction.php b/actions/FrontpageAction.php index 6ab18d29..824441b2 100644 --- a/actions/FrontpageAction.php +++ b/actions/FrontpageAction.php @@ -2,6 +2,14 @@ final class FrontpageAction implements ActionInterface { + private BridgeFactory $bridgeFactory; + + public function __construct( + BridgeFactory $bridgeFactory + ) { + $this->bridgeFactory = $bridgeFactory; + } + public function __invoke(Request $request): Response { $token = $request->attribute('token'); @@ -9,10 +17,9 @@ final class FrontpageAction implements ActionInterface $messages = []; $activeBridges = 0; - $bridgeFactory = new BridgeFactory(); - $bridgeClassNames = $bridgeFactory->getBridgeClassNames(); + $bridgeClassNames = $this->bridgeFactory->getBridgeClassNames(); - foreach ($bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) { + foreach ($this->bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) { $messages[] = [ 'body' => sprintf('Warning : Bridge "%s" not found', $missingEnabledBridge), 'level' => 'warning' @@ -21,8 +28,8 @@ final class FrontpageAction implements ActionInterface $body = ''; foreach ($bridgeClassNames as $bridgeClassName) { - if ($bridgeFactory->isEnabled($bridgeClassName)) { - $body .= BridgeCard::render($bridgeClassName, $token); + if ($this->bridgeFactory->isEnabled($bridgeClassName)) { + $body .= BridgeCard::render($this->bridgeFactory, $bridgeClassName, $token); $activeBridges++; } } diff --git a/actions/ListAction.php b/actions/ListAction.php index 3dd8f441..f6347f9c 100644 --- a/actions/ListAction.php +++ b/actions/ListAction.php @@ -2,19 +2,25 @@ class ListAction implements ActionInterface { + private BridgeFactory $bridgeFactory; + + public function __construct( + BridgeFactory $bridgeFactory + ) { + $this->bridgeFactory = $bridgeFactory; + } + public function __invoke(Request $request): Response { $list = new \stdClass(); $list->bridges = []; $list->total = 0; - $bridgeFactory = new BridgeFactory(); - - foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) { - $bridge = $bridgeFactory->create($bridgeClassName); + foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) { + $bridge = $this->bridgeFactory->create($bridgeClassName); $list->bridges[$bridgeClassName] = [ - 'status' => $bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive', + 'status' => $this->bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive', 'uri' => $bridge->getURI(), 'donationUri' => $bridge->getDonationURI(), 'name' => $bridge->getName(), diff --git a/bin/cache-clear b/bin/cache-clear index 635f41d5..71466360 100755 --- a/bin/cache-clear +++ b/bin/cache-clear @@ -17,11 +17,8 @@ if (file_exists(__DIR__ . '/../config.ini.php')) { } Configuration::loadConfiguration($config, getenv()); -$logger = new SimpleLogger('rssbridge'); +$container = require __DIR__ . '/../lib/dependencies.php'; -$logger->addHandler(new StreamHandler('php://stderr', Logger::INFO)); - -$cacheFactory = new CacheFactory($logger); -$cache = $cacheFactory->create(); +$cache = $container['cache']; $cache->clear(); diff --git a/bin/cache-prune b/bin/cache-prune index 281c019d..37696e14 100755 --- a/bin/cache-prune +++ b/bin/cache-prune @@ -17,11 +17,8 @@ if (file_exists(__DIR__ . '/../config.ini.php')) { } Configuration::loadConfiguration($config, getenv()); -$logger = new SimpleLogger('rssbridge'); +$container = require __DIR__ . '/../lib/dependencies.php'; -$logger->addHandler(new StreamHandler('php://stderr', Logger::INFO)); - -$cacheFactory = new CacheFactory($logger); -$cache = $cacheFactory->create(); +$cache = $container['cache']; $cache->prune(); diff --git a/bin/test b/bin/test new file mode 100755 index 00000000..f3556fc1 --- /dev/null +++ b/bin/test @@ -0,0 +1,29 @@ +#!/usr/bin/env php +debug('This is a test debug message'); + +$logger->info('This is a test info message'); + +$logger->error('This is a test error message'); diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index 424cd6e3..c273aaca 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -99,7 +99,7 @@ EOD; $user = $data->user; if ($user->videos === null) { // twitch regularly does this for unknown reasons - //$this->logger->info('Twitch returned empty set of videos', ['data' => $data]); + $this->debug->info('Twitch returned empty set of videos', ['data' => $data]); return; } diff --git a/index.php b/index.php index 2a613363..7b441944 100644 --- a/index.php +++ b/index.php @@ -17,7 +17,9 @@ if (file_exists(__DIR__ . '/config.ini.php')) { } Configuration::loadConfiguration($config, getenv()); -$logger = new SimpleLogger('rssbridge'); +$container = require __DIR__ . '/lib/dependencies.php'; + +$logger = $container['logger']; set_exception_handler(function (\Throwable $e) use ($logger) { $response = new Response(render(__DIR__ . '/templates/exception.html.php', ['e' => $e]), 500); @@ -60,23 +62,6 @@ register_shutdown_function(function () use ($logger) { } }); -$cacheFactory = new CacheFactory($logger); - -// Uncomment this for info logging to fs -// $logger->addHandler(new StreamHandler('/tmp/rss-bridge.txt', Logger::INFO)); - -// Uncomment this for debug logging to fs -// $logger->addHandler(new StreamHandler('/tmp/rss-bridge-debug.txt', Logger::DEBUG)); - -if (Debug::isEnabled()) { - $logger->addHandler(new ErrorLogHandler(Logger::DEBUG)); - $cache = $cacheFactory->create('array'); -} else { - $logger->addHandler(new ErrorLogHandler(Logger::INFO)); - $cache = $cacheFactory->create(); -} -$httpClient = new CurlHttpClient(); - date_default_timezone_set(Configuration::getConfig('system', 'timezone')); $argv = $argv ?? null; @@ -88,7 +73,7 @@ if ($argv) { } try { - $rssBridge = new RssBridge($logger, $cache, $httpClient); + $rssBridge = new RssBridge($container); $response = $rssBridge->main($request); $response->send(); } catch (\Throwable $e) { diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index f270c1a3..855ddb93 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -2,10 +2,11 @@ final class BridgeCard { - public static function render(string $bridgeClassName, ?string $token): string - { - $bridgeFactory = new BridgeFactory(); - + public static function render( + BridgeFactory $bridgeFactory, + string $bridgeClassName, + ?string $token + ): string { $bridge = $bridgeFactory->create($bridgeClassName); $uri = $bridge->getURI(); diff --git a/lib/BridgeFactory.php b/lib/BridgeFactory.php index ad433287..c214e44b 100644 --- a/lib/BridgeFactory.php +++ b/lib/BridgeFactory.php @@ -8,10 +8,12 @@ final class BridgeFactory private array $enabledBridges = []; private array $missingEnabledBridges = []; - public function __construct() - { - $this->cache = RssBridge::getCache(); - $this->logger = RssBridge::getLogger(); + public function __construct( + CacheInterface $cache, + Logger $logger + ) { + $this->cache = $cache; + $this->logger = $logger; // Create all possible bridge class names from fs foreach (scandir(__DIR__ . '/../bridges/') as $file) { diff --git a/lib/CacheFactory.php b/lib/CacheFactory.php index 90aa21ba..47bbbf72 100644 --- a/lib/CacheFactory.php +++ b/lib/CacheFactory.php @@ -14,10 +14,6 @@ class CacheFactory public function create(string $name = null): CacheInterface { - $name ??= Configuration::getConfig('cache', 'type'); - if (!$name) { - throw new \Exception('No cache type configured'); - } $cacheNames = []; foreach (scandir(PATH_LIB_CACHES) as $file) { if (preg_match('/^([^.]+)Cache\.php$/U', $file, $m)) { diff --git a/lib/Container.php b/lib/Container.php new file mode 100644 index 00000000..6dd0b6d3 --- /dev/null +++ b/lib/Container.php @@ -0,0 +1,33 @@ +values[$offset] = $value; + } + + #[ReturnTypeWillChange] public function offsetGet($offset) + { + if (!isset($this->values[$offset])) { + throw new \Exception(sprintf('Unknown container key: "%s"', $offset)); + } + if (!isset($this->resolved[$offset])) { + $this->resolved[$offset] = $this->values[$offset]($this); + } + return $this->resolved[$offset]; + } + + #[ReturnTypeWillChange] public function offsetExists($offset) + { + } + + public function offsetUnset($offset): void + { + } +} diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 35318c5b..9c8f5767 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -2,18 +2,12 @@ final class RssBridge { - private static Logger $logger; - private static CacheInterface $cache; - private static HttpClient $httpClient; + private static Container $container; public function __construct( - Logger $logger, - CacheInterface $cache, - HttpClient $httpClient + Container $container ) { - self::$logger = $logger; - self::$cache = $cache; - self::$httpClient = $httpClient; + self::$container = $container; } public function main(Request $request): Response @@ -83,10 +77,9 @@ final class RssBridge return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Invalid action']), 400); } - $className = '\\' . $actionName; - $actionObject = new $className(); + $controller = self::$container[$actionName]; - $response = $actionObject($request); + $response = $controller($request); return $response; } @@ -94,16 +87,16 @@ final class RssBridge public static function getLogger(): Logger { // null logger is only for the tests not to fail - return self::$logger ?? new NullLogger(); + return self::$container['logger'] ?? new NullLogger(); } public static function getCache(): CacheInterface { - return self::$cache; + return self::$container['cache']; } public static function getHttpClient(): HttpClient { - return self::$httpClient; + return self::$container['http_client']; } } diff --git a/lib/dependencies.php b/lib/dependencies.php new file mode 100644 index 00000000..227a66f1 --- /dev/null +++ b/lib/dependencies.php @@ -0,0 +1,78 @@ +addHandler(new ErrorLogHandler(Logger::DEBUG)); + } else { + $logger->addHandler(new ErrorLogHandler(Logger::INFO)); + } + // Uncomment this for info logging to fs + // $logger->addHandler(new StreamHandler('/tmp/rss-bridge.txt', Logger::INFO)); + + // Uncomment this for debug logging to fs + //$logger->addHandler(new StreamHandler('/tmp/rss-bridge-debug.txt', Logger::DEBUG)); + return $logger; +}; + +$container['cache'] = function ($c) { + /** @var CacheFactory $cacheFactory */ + $cacheFactory = $c['cache_factory']; + $type = Configuration::getConfig('cache', 'type'); + if (!$type) { + throw new \Exception('No cache type configured'); + } + if (Debug::isEnabled()) { + $cache = $cacheFactory->create('array'); + } else { + $cache = $cacheFactory->create($type); + } + return $cache; +}; + +return $container; diff --git a/lib/logger.php b/lib/logger.php index 3ebe3b0a..74a0e713 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -175,8 +175,9 @@ final class ErrorLogHandler $context = Json::encode($record['context']); } } + // Intentionally omitting newline $text = sprintf( - "[%s] %s.%s %s %s\n", + '[%s] %s.%s %s %s', $record['created_at']->format('Y-m-d H:i:s'), $record['name'], $record['level_name'], From e7ae06dcf08f0c977a231bb1ce9cb0b6657b4cfd Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 29 Aug 2024 23:02:01 +0200 Subject: [PATCH 185/423] fix: bug in prior refactor (#4239) --- lib/Container.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/Container.php b/lib/Container.php index 6dd0b6d3..086bd1f6 100644 --- a/lib/Container.php +++ b/lib/Container.php @@ -12,7 +12,8 @@ class Container implements \ArrayAccess $this->values[$offset] = $value; } - #[ReturnTypeWillChange] public function offsetGet($offset) + #[\ReturnTypeWillChange] + public function offsetGet($offset) { if (!isset($this->values[$offset])) { throw new \Exception(sprintf('Unknown container key: "%s"', $offset)); @@ -23,7 +24,8 @@ class Container implements \ArrayAccess return $this->resolved[$offset]; } - #[ReturnTypeWillChange] public function offsetExists($offset) + #[\ReturnTypeWillChange] + public function offsetExists($offset) { } From 39952c2d95cf4806063abbc2c7508cf9ab4f93e5 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 30 Aug 2024 00:07:58 +0200 Subject: [PATCH 186/423] refactor: implement middleware chain (#4240) * refactor: implement middleware chain * refactor --- lib/RssBridge.php | 75 ++++--------------- lib/bootstrap.php | 1 + middlewares/BasicAuthMiddleware.php | 38 ++++++++++ middlewares/MaintenanceMiddleware.php | 17 +++++ middlewares/Middleware.php | 8 ++ middlewares/SecurityMiddleware.php | 21 ++++++ middlewares/TokenAuthenticationMiddleware.php | 29 +++++++ 7 files changed, 128 insertions(+), 61 deletions(-) create mode 100644 middlewares/BasicAuthMiddleware.php create mode 100644 middlewares/MaintenanceMiddleware.php create mode 100644 middlewares/Middleware.php create mode 100644 middlewares/SecurityMiddleware.php create mode 100644 middlewares/TokenAuthenticationMiddleware.php diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 9c8f5767..230488bf 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -12,63 +12,6 @@ final class RssBridge public function main(Request $request): Response { - foreach ($request->toArray() as $key => $value) { - if (!is_string($value)) { - return new Response(render(__DIR__ . '/../templates/error.html.php', [ - 'message' => "Query parameter \"$key\" is not a string.", - ]), 400); - } - } - - if (Configuration::getConfig('system', 'enable_maintenance_mode')) { - return new Response(render(__DIR__ . '/../templates/error.html.php', [ - 'title' => '503 Service Unavailable', - 'message' => 'RSS-Bridge is down for maintenance.', - ]), 503); - } - - // HTTP Basic auth check - if (Configuration::getConfig('authentication', 'enable')) { - if (Configuration::getConfig('authentication', 'password') === '') { - return new Response('The authentication password cannot be the empty string', 500); - } - $user = $request->server('PHP_AUTH_USER'); - $password = $request->server('PHP_AUTH_PW'); - if ($user === null || $password === null) { - $html = render(__DIR__ . '/../templates/error.html.php', [ - 'message' => 'Please authenticate in order to access this instance!', - ]); - return new Response($html, 401, ['WWW-Authenticate' => 'Basic realm="RSS-Bridge"']); - } - if ( - (Configuration::getConfig('authentication', 'username') !== $user) - || (! hash_equals(Configuration::getConfig('authentication', 'password'), $password)) - ) { - $html = render(__DIR__ . '/../templates/error.html.php', [ - 'message' => 'Please authenticate in order to access this instance!', - ]); - return new Response($html, 401, ['WWW-Authenticate' => 'Basic realm="RSS-Bridge"']); - } - // At this point the username and password was correct - } - - // Add token as attribute to request - $request = $request->withAttribute('token', $request->get('token')); - - // Token authentication check - if (Configuration::getConfig('authentication', 'token')) { - if (! $request->attribute('token')) { - return new Response(render(__DIR__ . '/../templates/token.html.php', [ - 'message' => '', - ]), 401); - } - if (! hash_equals(Configuration::getConfig('authentication', 'token'), $request->attribute('token'))) { - return new Response(render(__DIR__ . '/../templates/token.html.php', [ - 'message' => 'Invalid token', - ]), 401); - } - } - $action = $request->get('action', 'Frontpage'); $actionName = strtolower($action) . 'Action'; $actionName = implode(array_map('ucfirst', explode('-', $actionName))); @@ -77,11 +20,21 @@ final class RssBridge return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Invalid action']), 400); } - $controller = self::$container[$actionName]; + $handler = self::$container[$actionName]; - $response = $controller($request); - - return $response; + $middlewares = [ + new SecurityMiddleware(), + new MaintenanceMiddleware(), + new BasicAuthMiddleware(), + new TokenAuthenticationMiddleware(), + ]; + $action = function ($req) use ($handler) { + return $handler($req); + }; + foreach (array_reverse($middlewares) as $middleware) { + $action = fn ($req) => $middleware($req, $action); + } + return $action($request); } public static function getLogger(): Logger diff --git a/lib/bootstrap.php b/lib/bootstrap.php index 1d866067..36b13e19 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -37,6 +37,7 @@ spl_autoload_register(function ($className) { __DIR__ . '/../caches/', __DIR__ . '/../formats/', __DIR__ . '/../lib/', + __DIR__ . '/../middlewares/', ]; foreach ($folders as $folder) { $file = $folder . $className . '.php'; diff --git a/middlewares/BasicAuthMiddleware.php b/middlewares/BasicAuthMiddleware.php new file mode 100644 index 00000000..6b0803e2 --- /dev/null +++ b/middlewares/BasicAuthMiddleware.php @@ -0,0 +1,38 @@ +server('PHP_AUTH_USER'); + $password = $request->server('PHP_AUTH_PW'); + if ($user === null || $password === null) { + $html = render(__DIR__ . '/../templates/error.html.php', [ + 'message' => 'Please authenticate in order to access this instance!', + ]); + return new Response($html, 401, ['WWW-Authenticate' => 'Basic realm="RSS-Bridge"']); + } + if ( + (Configuration::getConfig('authentication', 'username') !== $user) + || (!hash_equals(Configuration::getConfig('authentication', 'password'), $password)) + ) { + $html = render(__DIR__ . '/../templates/error.html.php', [ + 'message' => 'Please authenticate in order to access this instance!', + ]); + return new Response($html, 401, ['WWW-Authenticate' => 'Basic realm="RSS-Bridge"']); + } + return $next($request); + } +} diff --git a/middlewares/MaintenanceMiddleware.php b/middlewares/MaintenanceMiddleware.php new file mode 100644 index 00000000..de8a1baf --- /dev/null +++ b/middlewares/MaintenanceMiddleware.php @@ -0,0 +1,17 @@ + '503 Service Unavailable', + 'message' => 'RSS-Bridge is down for maintenance.', + ]), 503); + } +} diff --git a/middlewares/Middleware.php b/middlewares/Middleware.php new file mode 100644 index 00000000..83d93a3b --- /dev/null +++ b/middlewares/Middleware.php @@ -0,0 +1,8 @@ +toArray() as $key => $value) { + if (!is_string($value)) { + return new Response(render(__DIR__ . '/../templates/error.html.php', [ + 'message' => "Query parameter \"$key\" is not a string.", + ]), 400); + } + } + return $next($request); + } +} diff --git a/middlewares/TokenAuthenticationMiddleware.php b/middlewares/TokenAuthenticationMiddleware.php new file mode 100644 index 00000000..f8234629 --- /dev/null +++ b/middlewares/TokenAuthenticationMiddleware.php @@ -0,0 +1,29 @@ +withAttribute('token', $request->get('token')); + + if (! $request->attribute('token')) { + return new Response(render(__DIR__ . '/../templates/token.html.php', [ + 'message' => 'Missing token', + ]), 401); + } + if (! hash_equals(Configuration::getConfig('authentication', 'token'), $request->attribute('token'))) { + return new Response(render(__DIR__ . '/../templates/token.html.php', [ + 'message' => 'Invalid token', + ]), 401); + } + + return $next($request); + } +} From 9f48370eb0fd5aba832b9db9eb9b1bc8915f5417 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 30 Aug 2024 00:22:11 +0200 Subject: [PATCH 187/423] fix: tweak caching logic (#4241) --- actions/DisplayAction.php | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index c00c0d5e..9749004f 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -125,17 +125,16 @@ class DisplayAction implements ActionInterface return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429); } if ($e instanceof HttpException) { - // Reproduce (and log) these responses regardless of error output and report limit - if ($e->getCode() === 429) { - $this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); - return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429); - } - if ($e->getCode() === 503) { - $this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); - return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 503); + if (in_array($e->getCode(), [429, 503])) { + // Log with debug, immediately reproduce and return + $this->logger->debug(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); + return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), $e->getCode()); } + // Some other status code which we let fail normally (but don't log it) + } else { + // Log error if it's not an HttpException + $this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]); } - $this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]); $errorOutput = Configuration::getConfig('error', 'output'); $reportLimit = Configuration::getConfig('error', 'report_limit'); $errorCount = 1; From 3e1a8b29d95fe7fc0120e813ab623720ae056b8b Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 30 Aug 2024 02:29:51 +0200 Subject: [PATCH 188/423] fix: extract duplicate config loading (#4242) Also fix a problem with bin/cache-prune and FileCache and its enable_purge option --- actions/DisplayAction.php | 4 +++- bin/cache-clear | 11 +--------- bin/cache-prune | 20 +++++++++---------- bin/test | 11 +--------- caches/FileCache.php | 2 ++ index.php | 13 ++---------- lib/Configuration.php | 42 ++++----------------------------------- lib/config.php | 13 ++++++++++++ 8 files changed, 36 insertions(+), 80 deletions(-) create mode 100644 lib/config.php diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 9749004f..bda45558 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -89,7 +89,9 @@ class DisplayAction implements ActionInterface $this->cache->set($cacheKey, $response, 60 * 15); } - if (rand(1, 100) === 2) { + // For 1% of requests, prune cache + if (rand(1, 100) === 1) { + // This might be resource intensive! $this->cache->prune(); } diff --git a/bin/cache-clear b/bin/cache-clear index 71466360..c8f53122 100755 --- a/bin/cache-clear +++ b/bin/cache-clear @@ -6,16 +6,7 @@ */ require __DIR__ . '/../lib/bootstrap.php'; - -$config = []; -if (file_exists(__DIR__ . '/../config.ini.php')) { - $config = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); - if (!$config) { - http_response_code(500); - exit("Error parsing config.ini.php\n"); - } -} -Configuration::loadConfiguration($config, getenv()); +require __DIR__ . '/../lib/config.php'; $container = require __DIR__ . '/../lib/dependencies.php'; diff --git a/bin/cache-prune b/bin/cache-prune index 37696e14..755ed8d5 100755 --- a/bin/cache-prune +++ b/bin/cache-prune @@ -6,19 +6,19 @@ */ require __DIR__ . '/../lib/bootstrap.php'; - -$config = []; -if (file_exists(__DIR__ . '/../config.ini.php')) { - $config = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); - if (!$config) { - http_response_code(500); - exit("Error parsing config.ini.php\n"); - } -} -Configuration::loadConfiguration($config, getenv()); +require __DIR__ . '/../lib/config.php'; $container = require __DIR__ . '/../lib/dependencies.php'; +/** @var CacheInterface $cache */ $cache = $container['cache']; +if ( + Configuration::getConfig('cache', 'type') === 'file' + && !Configuration::getConfig('FileCache', 'enable_purge') +) { + // Override enable_purge for this execution + Configuration::setConfig('FileCache', 'enable_purge', true); +} + $cache->prune(); diff --git a/bin/test b/bin/test index f3556fc1..74692410 100755 --- a/bin/test +++ b/bin/test @@ -6,16 +6,7 @@ */ require __DIR__ . '/../lib/bootstrap.php'; - -$config = []; -if (file_exists(__DIR__ . '/../config.ini.php')) { - $config = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); - if (!$config) { - http_response_code(500); - exit("Error parsing config.ini.php\n"); - } -} -Configuration::loadConfiguration($config, getenv()); +require __DIR__ . '/../lib/config.php'; $container = require __DIR__ . '/../lib/dependencies.php'; diff --git a/caches/FileCache.php b/caches/FileCache.php index 7a0eb81d..dfd295e8 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -97,8 +97,10 @@ class FileCache implements CacheInterface } $expiration = $item['expiration'] ?? time(); if ($expiration === 0 || $expiration > time()) { + // Cached forever, or not expired yet continue; } + // Expired, so delete file unlink($cacheFile); } } diff --git a/index.php b/index.php index 7b441944..e67b0c9f 100644 --- a/index.php +++ b/index.php @@ -5,17 +5,8 @@ if (version_compare(\PHP_VERSION, '7.4.0') === -1) { exit("RSS-Bridge requires minimum PHP version 7.4\n"); } -require_once __DIR__ . '/lib/bootstrap.php'; - -$config = []; -if (file_exists(__DIR__ . '/config.ini.php')) { - $config = parse_ini_file(__DIR__ . '/config.ini.php', true, INI_SCANNER_TYPED); - if (!$config) { - http_response_code(500); - exit("Error parsing config.ini.php\n"); - } -} -Configuration::loadConfiguration($config, getenv()); +require __DIR__ . '/lib/bootstrap.php'; +require __DIR__ . '/lib/config.php'; $container = require __DIR__ . '/lib/dependencies.php'; diff --git a/lib/Configuration.php b/lib/Configuration.php index b104a251..44fd3612 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -15,43 +15,6 @@ final class Configuration { } - public static function checkInstallation(): array - { - $errors = []; - - // OpenSSL: https://www.php.net/manual/en/book.openssl.php - if (!extension_loaded('openssl')) { - $errors[] = 'openssl extension not loaded'; - } - - // libxml: https://www.php.net/manual/en/book.libxml.php - if (!extension_loaded('libxml')) { - $errors[] = 'libxml extension not loaded'; - } - - // Multibyte String (mbstring): https://www.php.net/manual/en/book.mbstring.php - if (!extension_loaded('mbstring')) { - $errors[] = 'mbstring extension not loaded'; - } - - // SimpleXML: https://www.php.net/manual/en/book.simplexml.php - if (!extension_loaded('simplexml')) { - $errors[] = 'simplexml extension not loaded'; - } - - // Client URL Library (curl): https://www.php.net/manual/en/book.curl.php - // Allow RSS-Bridge to run without curl module in CLI mode without root certificates - if (!extension_loaded('curl') && !(php_sapi_name() === 'cli' && empty(ini_get('curl.cainfo')))) { - $errors[] = 'curl extension not loaded'; - } - - // JavaScript Object Notation (json): https://www.php.net/manual/en/book.json.php - if (!extension_loaded('json')) { - $errors[] = 'json extension not loaded'; - } - return $errors; - } - public static function loadConfiguration(array $customConfig = [], array $env = []) { if (!file_exists(__DIR__ . '/../config.default.ini.php')) { @@ -204,7 +167,10 @@ final class Configuration return self::$config[strtolower($section)][strtolower($key)] ?? $default; } - private static function setConfig(string $section, string $key, $value): void + /** + * @internal Please avoid usage + */ + public static function setConfig(string $section, string $key, $value): void { self::$config[strtolower($section)][strtolower($key)] = $value; } diff --git a/lib/config.php b/lib/config.php new file mode 100644 index 00000000..4ff72565 --- /dev/null +++ b/lib/config.php @@ -0,0 +1,13 @@ + Date: Fri, 30 Aug 2024 02:44:50 +0200 Subject: [PATCH 189/423] fix: bug in prior fix (#4243) Have to tweak the config BEFORE instantiating of course --- bin/cache-clear | 1 + bin/cache-prune | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/cache-clear b/bin/cache-clear index c8f53122..2ca84ce6 100755 --- a/bin/cache-clear +++ b/bin/cache-clear @@ -10,6 +10,7 @@ require __DIR__ . '/../lib/config.php'; $container = require __DIR__ . '/../lib/dependencies.php'; +/** @var CacheInterface $cache */ $cache = $container['cache']; $cache->clear(); diff --git a/bin/cache-prune b/bin/cache-prune index 755ed8d5..bb72c4ac 100755 --- a/bin/cache-prune +++ b/bin/cache-prune @@ -10,15 +10,15 @@ require __DIR__ . '/../lib/config.php'; $container = require __DIR__ . '/../lib/dependencies.php'; -/** @var CacheInterface $cache */ -$cache = $container['cache']; - if ( Configuration::getConfig('cache', 'type') === 'file' && !Configuration::getConfig('FileCache', 'enable_purge') ) { - // Override enable_purge for this execution + // Override enable_purge for this particular execution Configuration::setConfig('FileCache', 'enable_purge', true); } +/** @var CacheInterface $cache */ +$cache = $container['cache']; + $cache->prune(); From 6a24e53d6ca4fbfb3115a8cb30a51283684f0f20 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 30 Aug 2024 04:21:51 +0200 Subject: [PATCH 190/423] refactor (#4244) --- bridges/AO3Bridge.php | 32 +++++++++++--------- bridges/BMDSystemhausBlogBridge.php | 5 ++-- bridges/TwitchBridge.php | 22 +++++++------- docs/09_Technical_recommendations/index.md | 34 ++++++++++++---------- lib/FeedItem.php | 5 +++- lib/RssBridge.php | 22 ++------------ lib/contents.php | 15 ++++++++-- 7 files changed, 69 insertions(+), 66 deletions(-) diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php index 4c09c28c..970ed414 100644 --- a/bridges/AO3Bridge.php +++ b/bridges/AO3Bridge.php @@ -68,12 +68,13 @@ class AO3Bridge extends BridgeAbstract */ private function collectList($url) { - $httpClient = RssBridge::getHttpClient(); $version = 'v0.0.1'; - $agent = ['useragent' => "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"]; + $headers = [ + "useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)" + ]; + $response = getContents($url, $headers); - $response = $httpClient->request($url, $agent); - $html = \str_get_html($response->getBody()); + $html = \str_get_html($response); $html = defaultLinkTo($html, self::URI); // Get list title. Will include page range + count in some cases @@ -128,14 +129,15 @@ class AO3Bridge extends BridgeAbstract case ('last'): // only way to get this is using the navigate page unfortunately $url .= '/navigate'; - $response = $httpClient->request($url, $agent); - $html = \str_get_html($response->getBody()); + $response = getContents($url, $headers); + $html = \str_get_html($response); $html = defaultLinkTo($html, self::URI); $url = $html->find('ol.index.group > li > a', -1)->href; break; } - $response = $httpClient->request($url, $agent); - $html = \str_get_html($response->getBody()); + $response = getContents($url, $headers); + + $html = \str_get_html($response); $html = defaultLinkTo($html, self::URI); // remove duplicate fic summary if ($ficsum = $html->find('#workskin > .preface > .summary', 0)) { @@ -159,16 +161,18 @@ class AO3Bridge extends BridgeAbstract */ private function collectWork($url) { - $httpClient = RssBridge::getHttpClient(); $version = 'v0.0.1'; - $agent = ['useragent' => "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"]; + $headers = [ + "useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)" + ]; + $response = getContents($url . '/navigate', $headers); - $response = $httpClient->request($url . '/navigate', $agent); - $html = \str_get_html($response->getBody()); + $html = \str_get_html($response); $html = defaultLinkTo($html, self::URI); - $response = $httpClient->request($url . '?view_full_work=true', $agent); - $workhtml = \str_get_html($response->getBody()); + $response = getContents($url . '?view_full_work=true', $headers); + + $workhtml = \str_get_html($response); $workhtml = defaultLinkTo($workhtml, self::URI); $this->title = $html->find('h2 a', 0)->plaintext; diff --git a/bridges/BMDSystemhausBlogBridge.php b/bridges/BMDSystemhausBlogBridge.php index 12f3ca5e..98fb2d63 100644 --- a/bridges/BMDSystemhausBlogBridge.php +++ b/bridges/BMDSystemhausBlogBridge.php @@ -54,7 +54,7 @@ class BMDSystemhausBlogBridge extends BridgeAbstract public function collectData() { // get website content - $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('No contents received!'); + $html = getSimpleHTMLDOM($this->getURI()); // Convert relative links in HTML into absolute links $html = defaultLinkTo($html, self::URI); @@ -207,7 +207,8 @@ class BMDSystemhausBlogBridge extends BridgeAbstract //----------------------------------------------------- public function getURI() { - $lURI = $this->getURIbyCountry($this->getInput('country')); + $country = $this->getInput('country') ?? ''; + $lURI = $this->getURIbyCountry($country); return $lURI != '' ? $lURI : parent::getURI(); } diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index c273aaca..6605a973 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -196,23 +196,21 @@ EOD; // e.g. 01:53:27 private function formatTimestampTime($seconds) { - return sprintf( - '%02d:%02d:%02d', - floor($seconds / 3600), - ($seconds / 60) % 60, - $seconds % 60 - ); + $floor = floor($seconds / 3600); + $i = intval($seconds / 60) % 60; + $i1 = $seconds % 60; + + return sprintf('%02d:%02d:%02d', $floor, $i, $i1); } // e.g. 01h53m27s private function formatQueryTime($seconds) { - return sprintf( - '%02dh%02dm%02ds', - floor($seconds / 3600), - ($seconds / 60) % 60, - $seconds % 60 - ); + $floor = floor($seconds / 3600); + $i = intval($seconds / 60) % 60; + $i1 = $seconds % 60; + + return sprintf('%02dh%02dm%02ds', $floor, $i, $i1); } /** diff --git a/docs/09_Technical_recommendations/index.md b/docs/09_Technical_recommendations/index.md index a57f0bbd..c564418e 100644 --- a/docs/09_Technical_recommendations/index.md +++ b/docs/09_Technical_recommendations/index.md @@ -1,28 +1,32 @@ ## General recommendations -* Use [HTTPS](https://en.wikipedia.org/wiki/HTTPS) (`https://...`) over [HTTP](https://en.wikipedia.org/wiki/HTTPS) (`http://...`) whenever possible - ## Test a site before building a bridge -Some sites make use of anti-bot mechanisms (e.g.: by using JavaScript) in which case they work fine in regular browsers, but not in the PHP environment. To check if a site works with RSS-Bridge, create a new bridge using the [template](../05_Bridge_API/02_BridgeAbstract.md#template) and load a valid URL (not the base URL!). +Some sites make use of anti-bot mechanisms (e.g.: by using JavaScript) in which case they work fine in regular browsers, +but not in the PHP environment. + +To check if a site works with RSS-Bridge, create a new bridge using the +[template](../05_Bridge_API/02_BridgeAbstract.md#template) +and load a valid URL (not the base URL!). **Example (using github.com)** ```PHP logger = RssBridge::getLogger(); + global $container; + + // The default NullLogger is for when running the unit tests + $this->logger = $container['logger'] ?? new NullLogger(); } public function __set($name, $value) diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 230488bf..5e90fb13 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -2,12 +2,12 @@ final class RssBridge { - private static Container $container; + private Container $container; public function __construct( Container $container ) { - self::$container = $container; + $this->container = $container; } public function main(Request $request): Response @@ -20,7 +20,7 @@ final class RssBridge return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Invalid action']), 400); } - $handler = self::$container[$actionName]; + $handler = $this->container[$actionName]; $middlewares = [ new SecurityMiddleware(), @@ -36,20 +36,4 @@ final class RssBridge } return $action($request); } - - public static function getLogger(): Logger - { - // null logger is only for the tests not to fail - return self::$container['logger'] ?? new NullLogger(); - } - - public static function getCache(): CacheInterface - { - return self::$container['cache']; - } - - public static function getHttpClient(): HttpClient - { - return self::$container['http_client']; - } } diff --git a/lib/contents.php b/lib/contents.php index cc9542a9..56a3db20 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -14,8 +14,13 @@ function getContents( array $curlOptions = [], bool $returnFull = false ) { - $httpClient = RssBridge::getHttpClient(); - $cache = RssBridge::getCache(); + global $container; + + /** @var HttpClient $httpClient */ + $httpClient = $container['http_client']; + + /** @var CacheInterface $cache */ + $cache = $container['cache']; // TODO: consider url validation at this point @@ -212,7 +217,11 @@ function getSimpleHTMLDOMCached( $defaultBRText = DEFAULT_BR_TEXT, $defaultSpanText = DEFAULT_SPAN_TEXT ) { - $cache = RssBridge::getCache(); + global $container; + + /** @var CacheInterface $cache */ + $cache = $container['cache']; + $cacheKey = 'pages_' . $url; $content = $cache->get($cacheKey); if (!$content) { From 9cabf60144c843e4de21ec348cb88d8304604f13 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 30 Aug 2024 04:37:40 +0200 Subject: [PATCH 191/423] docs * refactor * docs --- docs/01_General/03_Requirements.md | 11 -- docs/01_General/05_FAQ.md | 29 ++--- docs/02_CLI/index.md | 22 ++-- docs/03_For_Hosts/01_Installation.md | 11 +- docs/03_For_Hosts/02_Updating.md | 38 ------- docs/03_For_Hosts/06_Authentication.md | 105 +----------------- docs/03_For_Hosts/07_Customizations.md | 15 ++- docs/03_For_Hosts/08_Custom_Configuration.md | 13 ++- docs/03_For_Hosts/index.md | 6 +- .../01_How_to_create_a_new_bridge.md | 3 +- docs/05_Bridge_API/02_BridgeAbstract.md | 7 +- 11 files changed, 64 insertions(+), 196 deletions(-) diff --git a/docs/01_General/03_Requirements.md b/docs/01_General/03_Requirements.md index 617cfadc..c9c91a52 100644 --- a/docs/01_General/03_Requirements.md +++ b/docs/01_General/03_Requirements.md @@ -1,14 +1,3 @@ - PHP 7.4 (or higher) - - [`openssl`](https://secure.php.net/manual/en/book.openssl.php) extension - - [`libxml`](https://secure.php.net/manual/en/book.libxml.php) extension (enabled by default, see [PHP Manual](http://php.net/manual/en/libxml.installation.php)) - - [`mbstring`](https://secure.php.net/manual/en/book.mbstring.php) extension - - [`simplexml`](https://secure.php.net/manual/en/book.simplexml.php) extension - - [`curl`](https://secure.php.net/manual/en/book.curl.php) extension - - [`json`](https://secure.php.net/manual/en/book.json.php) extension - - [`filter`](https://secure.php.net/manual/en/book.filter.php) extension - - [`zip`](https://secure.php.net/manual/en/book.zip.php) (for some bridges) - - [`sqlite3`](http://php.net/manual/en/book.sqlite3.php) extension (only when using SQLiteCache) - -Enable extensions by un-commenting the corresponding line in your PHP configuration (`php.ini`). diff --git a/docs/01_General/05_FAQ.md b/docs/01_General/05_FAQ.md index ade746d7..19cfae4b 100644 --- a/docs/01_General/05_FAQ.md +++ b/docs/01_General/05_FAQ.md @@ -1,30 +1,33 @@ -This page provides a collection of frequently asked questions and their answers. Please check this page before opening a new Issue :revolving_hearts: - -* [Why doesn't my bridge show new contents?](#why-doesnt-my-bridge-show-new-contents) -* [How can I make a bridge update more frequently?](#how-can-i-make-a-bridge-update-more-frequently) -* [Firefox doesn't show feeds anymore, what can I do?](#firefox-doesnt-show-feeds-anymore-what-can-i-do) - ## Why doesn't my bridge show new contents? -RSS-Bridge creates a cached version of your feed in order to reduce traffic and respond faster. The cached version is created on the first request and served for all subsequent requests. On every request RSS-Bridge checks if the cache timeout has elapsed. If the timeout has elapsed, it loads new contents and updates the cached version. +RSS-Bridge creates a cached version of your feed in order to reduce traffic and respond faster. +The cached version is created on the first request and served for all subsequent requests. +On every request RSS-Bridge checks if the cache timeout has elapsed. +If the timeout has elapsed, it loads new contents and updates the cached version. -_Notice_: RSS-Bridge only updates feeds if you actively request it, for example by pressing F5 in your browser or using a feed reader. +_Notice_: RSS-Bridge only updates feeds if you actively request it, +for example by pressing F5 in your browser or using a feed reader. -The cache duration is bridge specific and can last anywhere between five minutes and 24 hours. You can specify a custom cache timeout for each bridge if [this option](#how-can-i-make-a-bridge-update-more-frequently) has been enabled on the server. +The cache duration is bridge specific (usually `1h`) +You can specify a custom cache timeout for each bridge if +[this option](#how-can-i-make-a-bridge-update-more-frequently) has been enabled on the server. ## How can I make a bridge update more frequently? You can only do that if you are hosting the RSS-Bridge instance: +- Lower the bridge ttl: `CACHE_TIMEOUT` constant - Enable [`custom_timeout`](../03_For_Hosts/08_Custom_Configuration.md#customtimeout) -- Alternatively, change the default timeout for your bridge by modifying the `CACHE_TIMEOUT` constant in the relevant bridge file (e.g [here](https://github.com/RSS-Bridge/rss-bridge/blob/master/bridges/FilterBridge.php#L7) for the Filter Bridge). ## Firefox doesn't show feeds anymore, what can I do? -As of version 64, Firefox removed support for viewing Atom and RSS feeds in the browser. This results in the browser downloading the pages instead of showing contents. +As of version 64, Firefox removed support for viewing Atom and RSS feeds in the browser. +This results in the browser downloading the pages instead of showing contents. Further reading: - https://support.mozilla.org/en-US/kb/feed-reader-replacements-firefox - https://bugzilla.mozilla.org/show_bug.cgi?id=1477667 -To restore the original behavior in Firefox 64 or higher you can use following Add-on which attempts to recreate the original behavior (with some sugar on top): -- https://addons.mozilla.org/en-US/firefox/addon/rsspreview/ \ No newline at end of file +To restore the original behavior in Firefox 64 or higher you can use following Add-on +which attempts to recreate the original behavior (with some sugar on top): + +- https://addons.mozilla.org/en-US/firefox/addon/rsspreview/ diff --git a/docs/02_CLI/index.md b/docs/02_CLI/index.md index 9292746a..727e59cb 100644 --- a/docs/02_CLI/index.md +++ b/docs/02_CLI/index.md @@ -1,10 +1,12 @@ -RSS-Bridge supports calls via CLI. You can use the same parameters as you would normally use via the URI. Example: +RSS-Bridge supports calls via CLI. +You can use the same parameters as you would normally use via the URI. Example: `php index.php action=display bridge=DansTonChat format=Json` ## Required parameters -RSS-Bridge requires a few parameters that must be specified on every call. Omitting these parameters will result in error messages: +RSS-Bridge requires a few parameters that must be specified on every call. +Omitting these parameters will result in error messages: ### action @@ -17,20 +19,26 @@ Value | Description ### bridge -This parameter specifies the name of the bridge RSS-Bridge should return feeds from. The name of the bridge equals the class name of the bridges in the ./bridges/ folder without the 'Bridge' prefix. For example: DansTonChatBridge => DansTonChat. +This parameter specifies the name of the bridge RSS-Bridge should return feeds from. +The name of the bridge equals the class name of the bridges in the ./bridges/ folder without the 'Bridge' prefix. +For example: DansTonChatBridge => DansTonChat. ### format -This parameter specifies the format in which RSS-Bridge returns the contents. RSS-Bridge currently supports five formats: `Atom`, `Html`, `Json`, `Mrss`and `Plaintext`. +This parameter specifies the format in which RSS-Bridge returns the contents. ## Optional parameters -RSS-Bridge supports optional parameters. These parameters are only valid if the options have been enabled in the index.php script. +RSS-Bridge supports optional parameters. +These parameters are only valid if the options have been enabled in the index.php script. ### \_noproxy -This parameter is only available if a proxy server has been specified via `proxy.url` and `proxy.by_bridge` has been enabled. This is a Boolean parameter that can be set to `true` or `false`. +This parameter is only available if a proxy server has been specified via `proxy.url` and `proxy.by_bridge` +has been enabled. This is a Boolean parameter that can be set to `true` or `false`. ## Bridge parameters -Each bridge can specify its own set of parameters. As in the example above, some bridges don't specify any parameters or only optional parameters that can be neglected. For more details read the `PARAMETERS` definition for your bridge. \ No newline at end of file +Each bridge can specify its own set of parameters. +As in the example above, some bridges don't specify any parameters or only optional parameters that can be neglected. +For more details read the `PARAMETERS` definition for your bridge. diff --git a/docs/03_For_Hosts/01_Installation.md b/docs/03_For_Hosts/01_Installation.md index 729e6abb..3312230d 100644 --- a/docs/03_For_Hosts/01_Installation.md +++ b/docs/03_For_Hosts/01_Installation.md @@ -1,10 +1 @@ -In order to install RSS-Bridge on your own web server* do as follows: - -* Make sure your web server meets all [requirements](../01_General/03_Requirements.md) -* Download the ZIP file of the [last stable release](https://github.com/RSS-Bridge/rss-bridge/releases) -* Place all files on your web server - -For linux hosts: -* Grant read-write-access for `www-data` to the `./cache` directory (`chown -R www-data ./cache`) - -You have successfully installed RSS-Bridge. \ No newline at end of file +https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md diff --git a/docs/03_For_Hosts/02_Updating.md b/docs/03_For_Hosts/02_Updating.md index 3484c6dc..e69de29b 100644 --- a/docs/03_For_Hosts/02_Updating.md +++ b/docs/03_For_Hosts/02_Updating.md @@ -1,38 +0,0 @@ -Updating an existing installation is very simple, depending on your type of installation. - -## Release Build - -* Download latest version -* Extract all files -* Replace existing files - -This will update all core files to the latest version. Your custom configuration and bridges are left untouched. Keep in mind that changes to any core file of RSS-Bridge will be replaced. - -## Heroku - -### If you didn't fork the repo before - -Fork the repo by clicking the `Fork` button at the top right of this page (must be on desktop site). Then on your Heroku account, go to the application. Click on the `Deploy` tab and connect the repo named `yourusername/rss-bridge`. Do a manual deploy of the `master` branch. - -### If you forked the repo before - -[Click here to create a new pull request to your fork](https://github.com/RSS-Bridge/rss-bridge/pull/new/master). Select `compare across forks`, make the base repository `yourusername/rss-bridge` and ensure the branch is set to master. Put any title you want and create the pull request. On the page that comes after this, merge the pull request. - -You then want to go to your application in Heroku, connect your fork via the `Deploy` tab and deploy the `master` branch. - -You can turn on auto-deploy for the master branch if you don't want to go through the process of logging into Heroku and deploying the branch every time changes to the repo are made in the future. - -## Git - -To get the latest changes from the master branch - -``` -git pull -``` - -To use a specific tag - -``` -git fetch --all -git checkout tags/ -``` \ No newline at end of file diff --git a/docs/03_For_Hosts/06_Authentication.md b/docs/03_For_Hosts/06_Authentication.md index bb9c6656..f505f5a6 100644 --- a/docs/03_For_Hosts/06_Authentication.md +++ b/docs/03_For_Hosts/06_Authentication.md @@ -1,101 +1,6 @@ -Depending on your servers abilities you can choose between two types of authentication: -* [.htaccess](#htaccess) -* [RSS-Bridge Authentication](#rss-bridge-authentication) - -**General advice**: - -- Make sure to use a strong password, no matter which solution you choose! -- Enable HTTPS on your server to ensure your connection is encrypted and secure! - -## .htaccess - -.htaccess files are commonly used to restrict access to files on a web server. One of the features of .htaccess files is the ability to password protect specific (or all) directories. If setup correctly, a password is required to access the files. - -The usage of .htaccess files requires three basic steps: - -1) [Enable .htaccess](#enable-htaccess) -2) [Create a .htpasswd file](#create-a-htpasswd-file) -3) [Create a .htaccess file](#create-a-htaccess-file) - -### Enable .htaccess - -This process depends on the server you are using. Some providers may require you to change some settings, or place/change some file. Here are some helpful links for your server (please add your own if missing :sparkling_heart:) - -- Apache: http://ask.xmodulo.com/enable-htaccess-apache.html - -### Create a .htpasswd file - -The `.htpasswd` file contains the user name and password used for login to your web server. Please notice that the password is stored in encrypted form, which requires you to encrypt your password before creating the `.htpasswd` file! - -Here are three ways of creating your own `.htpasswd` file: - -**1) Example file** - -Example `.htpasswd` file (user name: "test", password: "test"): - -```.htpasswd -test:$apr1$a52u9ILP$XTNG8qMJiEXSm1zD0lQcR0 -``` - -Just copy and paste the contents to your `.htpasswd` file. - -**2) Online generator (read warning!)** - -You can create your own `.htpasswd` file online using a `.htpasswd` generator like this: https://www.htaccesstools.com/htpasswd-generator/ - -**WARNING!** -- Never insert real passwords to an online generator! - -**3) Generate your own password** - -Another way to create your own `.htpasswd` file is to run this script on your server (it'll output the data for you, you just have to paste it int a `.htpasswd` file): - -```PHP - -``` - ->source: https://www.htaccesstools.com/articles/create-password-for-htpasswd-file-using-php/ - -### Create a .htaccess file - -The `.htaccess` file is used to specify which directories are password protected. For that purpose you should place the file in whatever directory you want to restrict access. If you want to restrict access to RSS-Bridge in general, you should place the file in the root directory (where `index.php` is located). - -Two parameters must be specified in the `.htaccess` file: - -* AuthName -* AuthUserFile - -`AuthName` specifies the name of the authentication (i.e. "RSS-Bridge"). `AuthUserFile` defines the **absolute** path to a `.htpasswd` file. - -Here are two ways of creating your own `.htaccess` file: - -**1) Example file** - -```.htaccess -AuthType Basic -AuthName "My Protected Area" -AuthUserFile /path/to/.htpasswd -Require valid-user -``` - -Notice: You must change the `AuthUserFile` location to fit your own server (i.e. `/var/www/html/rss-bridge/.htpasswd`) - -**2) Online generator** - -You can use an online generator to create the file for you and copy-paste it to your `.htaccess` file: https://www.htaccesstools.com/htaccess-authentication/ - -## RSS-Bridge Authentication - -RSS-Bridge ships with an authentication module designed for single user environments. You can enable authentication and specify the username & password in the [configuration file](../03_For_Hosts/08_Custom_Configuration.md#authentication). - -Please notice that the password is stored in plain text and thus is readable to anyone who can access the file. Make sure to restrict access to the file, so that it cannot be read remotely! \ No newline at end of file +* http basic auth +* token +* Access control via webserver (see nginx/caddy/apache docs) + +https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md diff --git a/docs/03_For_Hosts/07_Customizations.md b/docs/03_For_Hosts/07_Customizations.md index be4c7f85..380f5f3a 100644 --- a/docs/03_For_Hosts/07_Customizations.md +++ b/docs/03_For_Hosts/07_Customizations.md @@ -1,9 +1,14 @@ -RSS-Bridge ships a few options the host may or may not activate. All options are listed in the [config.default.ini.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/config.default.ini.php) file, see [Custom Configuration](08_Custom_Configuration.md) section for more information. +RSS-Bridge ships a few options the host may or may not activate. +All options are listed in the [config.default.ini.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/config.default.ini.php) file, +see [Custom Configuration](08_Custom_Configuration.md) section for more information. ## Customizable cache timeout -Sometimes it is necessary to specify custom timeouts to update contents more frequently than the bridge maintainer intended. In these cases the client may specify a custom cache timeout to prevent loading contents from cache earlier (or later). +Sometimes it is necessary to specify custom timeouts to update contents more frequently +than the bridge maintainer intended. +In these cases the client may specify a custom cache timeout to prevent loading contents +from cache earlier (or later). -This option can be activated by setting the [`cache.custom_timeout`](08_Custom_Configuration.md#custom_timeout) option to `true`. When enabled each bridge receives an additional parameter `Cache timeout in seconds` that can be set to any value between 1 and 86400 (24 hours). If the value is not within the limits the default settings apply (as specified by the bridge maintainer). - -The cache timeout is send to RSS-Bridge using the `_cache_timeout` parameter. RSS-Bridge will return an error message if the parameter is received and the option is disabled. +This option can be activated by setting the [`cache.custom_timeout`](08_Custom_Configuration.md#custom_timeout) option to `true`. +When enabled each bridge receives an additional parameter `Cache timeout in seconds` +that can be set to any value. diff --git a/docs/03_For_Hosts/08_Custom_Configuration.md b/docs/03_For_Hosts/08_Custom_Configuration.md index 9a1f78f2..6e22f7ee 100644 --- a/docs/03_For_Hosts/08_Custom_Configuration.md +++ b/docs/03_For_Hosts/08_Custom_Configuration.md @@ -1,16 +1,21 @@ RSS-Bridge supports custom configurations for common parameters on the server side! -A default configuration file (`config.default.ini.php`) is shipped with RSS-Bridge. Please do not edit this file, as it gets replaced when upgrading RSS-Bridge! +A default configuration file (`config.default.ini.php`) is shipped with RSS-Bridge. +Please do not edit this file, as it gets replaced when upgrading RSS-Bridge! -You should, however, use this file as template to create your own configuration (or leave it as is, to keep the default settings). In order to create your own configuration perform following actions: +You should, however, use this file as template to create your own configuration +(or leave it as is, to keep the default settings). +In order to create your own configuration perform following actions: * Create the file `config.ini.php` in the RSS-Bridge root folder (next to `config.default.ini.php`) * Copy the contents from `config.default.ini.php` to your configuration file * Change the parameters to satisfy your requirements -RSS-Bridge will automatically detect the `config.ini.php` and use it. If the file doesn't exist it will default to `config.default.ini.php` automatically. +RSS-Bridge will automatically detect the `config.ini.php` and use it. +If the file doesn't exist it will default to `config.default.ini.php` automatically. -__Notice__: If a parameter is not specified in your `config.ini.php` RSS-Bridge will automatically use the default settings from `config.default.ini.php`. +__Notice__: If a parameter is not specified in your `config.ini.php` RSS-Bridge will +automatically use the default settings from `config.default.ini.php`. # Available parameters diff --git a/docs/03_For_Hosts/index.md b/docs/03_For_Hosts/index.md index b89f321a..a4e55d69 100644 --- a/docs/03_For_Hosts/index.md +++ b/docs/03_For_Hosts/index.md @@ -1,8 +1,5 @@ This section is directed at **hosts** and **server administrators**. -To install RSS-Bridge, please follow the [installation instructions](../03_For_Hosts/01_Installation.md). -You must have access to a web server with a working PHP environment! - RSS-Bridge comes with a large amount of bridges. Some bridges could be implemented more efficiently by actually using proprietary APIs, @@ -11,4 +8,5 @@ but there are reasons against it: - RSS-Bridge exists in the first place to NOT use APIs. - See [the rant](https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md#Rant) -- APIs require private keys that could be stored on servers running RSS-Bridge,which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution. \ No newline at end of file +- APIs require private keys that could be stored on servers running RSS-Bridge, +- which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution. diff --git a/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md b/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md index 391d179f..02287962 100644 --- a/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md +++ b/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md @@ -27,4 +27,5 @@ The file must start with the PHP tags and end with an empty line. The closing ta // This line is empty (just imagine it!) ``` -The next step is to extend one of the base classes. Refer to one of an base classes listed on the [Bridge API](../05_Bridge_API/index.md) page. \ No newline at end of file +The next step is to extend one of the base classes. +Refer to one of an base classes listed on the [Bridge API](../05_Bridge_API/index.md) page. diff --git a/docs/05_Bridge_API/02_BridgeAbstract.md b/docs/05_Bridge_API/02_BridgeAbstract.md index b6813a16..9cb16050 100644 --- a/docs/05_Bridge_API/02_BridgeAbstract.md +++ b/docs/05_Bridge_API/02_BridgeAbstract.md @@ -50,10 +50,10 @@ For example: `MyBridge.php` => `MyBridge` ```PHP @@ -76,7 +76,8 @@ const CACHE_TIMEOUT // (optional) Defines the maximum duration for the cache in ```PHP Date: Sun, 1 Sep 2024 00:27:45 +1000 Subject: [PATCH 192/423] [ABCNewsBridge] Fix broken due to site redesign (#4247) --- bridges/ABCNewsBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/ABCNewsBridge.php b/bridges/ABCNewsBridge.php index c00fed1c..154eb489 100644 --- a/bridges/ABCNewsBridge.php +++ b/bridges/ABCNewsBridge.php @@ -31,17 +31,17 @@ class ABCNewsBridge extends BridgeAbstract { $url = sprintf('https://www.abc.net.au/news/%s', $this->getInput('topic')); $dom = getSimpleHTMLDOM($url); - $dom = $dom->find('div[data-component="CardList"]', 0); + $dom = $dom->find('div[data-component="PaginationList"]', 0); if (!$dom) { throw new \Exception(sprintf('Unable to find css selector on `%s`', $url)); } $dom = defaultLinkTo($dom, $this->getURI()); - foreach ($dom->find('div[data-component="GenericCard"]') as $article) { + foreach ($dom->find('article[data-component="DetailCard"]') as $article) { $a = $article->find('a', 0); $this->items[] = [ 'title' => $a->plaintext, 'uri' => $a->href, - 'content' => $article->find('[data-component="CardDescription"]', 0)->plaintext, + 'content' => $article->find('p', 0)->plaintext, 'timestamp' => strtotime($article->find('time', 0)->datetime), ]; } From a6bdc322b01de91b7a9e27243b30a65caf2d3c5a Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 1 Sep 2024 21:48:14 +0200 Subject: [PATCH 193/423] refactor: extract exception and cache middleware (#4248) --- actions/DisplayAction.php | 36 +----------------- index.php | 15 +++----- lib/Configuration.php | 4 ++ lib/RssBridge.php | 4 +- lib/dependencies.php | 12 +----- middlewares/CacheMiddleware.php | 59 +++++++++++++++++++++++++++++ middlewares/ExceptionMiddleware.php | 24 ++++++++++++ templates/html-format.html.php | 1 - 8 files changed, 99 insertions(+), 56 deletions(-) create mode 100644 middlewares/CacheMiddleware.php create mode 100644 middlewares/ExceptionMiddleware.php diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index bda45558..845bfd84 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -22,25 +22,6 @@ class DisplayAction implements ActionInterface $format = $request->get('format'); $noproxy = $request->get('_noproxy'); - $cacheKey = 'http_' . json_encode($request->toArray()); - /** @var Response $cachedResponse */ - $cachedResponse = $this->cache->get($cacheKey); - if ($cachedResponse) { - $ifModifiedSince = $request->server('HTTP_IF_MODIFIED_SINCE'); - $lastModified = $cachedResponse->getHeader('last-modified'); - if ($ifModifiedSince && $lastModified) { - $lastModified = new \DateTimeImmutable($lastModified); - $lastModifiedTimestamp = $lastModified->getTimestamp(); - $modifiedSince = strtotime($ifModifiedSince); - // TODO: \DateTimeImmutable can be compared directly - if ($lastModifiedTimestamp <= $modifiedSince) { - $modificationTimeGMT = gmdate('D, d M Y H:i:s ', $lastModifiedTimestamp); - return new Response('', 304, ['last-modified' => $modificationTimeGMT . 'GMT']); - } - } - return $cachedResponse->withHeader('rss-bridge', 'This is a cached response'); - } - if (!$bridgeName) { return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge parameter']), 400); } @@ -66,6 +47,8 @@ class DisplayAction implements ActionInterface define('NOPROXY', true); } + $cacheKey = 'http_' . json_encode($request->toArray()); + $bridge = $this->bridgeFactory->create($bridgeClassName); $response = $this->createResponse($request, $bridge, $format); @@ -80,21 +63,6 @@ class DisplayAction implements ActionInterface $this->cache->set($cacheKey, $response, $ttl); } - if (in_array($response->getCode(), [403, 429, 503])) { - // Cache these responses for about ~20 mins on average - $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); - } - - if ($response->getCode() === 500) { - $this->cache->set($cacheKey, $response, 60 * 15); - } - - // For 1% of requests, prune cache - if (rand(1, 100) === 1) { - // This might be resource intensive! - $this->cache->prune(); - } - return $response; } diff --git a/index.php b/index.php index e67b0c9f..ec7490d6 100644 --- a/index.php +++ b/index.php @@ -63,13 +63,8 @@ if ($argv) { $request = Request::fromGlobals(); } -try { - $rssBridge = new RssBridge($container); - $response = $rssBridge->main($request); - $response->send(); -} catch (\Throwable $e) { - // Probably an exception inside an action - $logger->error('Exception in RssBridge::main()', ['e' => $e]); - $response = new Response(render(__DIR__ . '/templates/exception.html.php', ['e' => $e]), 500); - $response->send(); -} +$rssBridge = new RssBridge($container); + +$response = $rssBridge->main($request); + +$response->send(); \ No newline at end of file diff --git a/lib/Configuration.php b/lib/Configuration.php index 44fd3612..187848fb 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -82,6 +82,10 @@ final class Configuration } } + if (Debug::isEnabled()) { + self::setConfig('cache', 'type', 'array'); + } + if (!is_array(self::getConfig('system', 'enabled_bridges'))) { self::throwConfigError('system', 'enabled_bridges', 'Is not an array'); } diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 5e90fb13..c7b132d6 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -23,6 +23,8 @@ final class RssBridge $handler = $this->container[$actionName]; $middlewares = [ + new CacheMiddleware($this->container['cache']), + new ExceptionMiddleware($this->container['logger']), new SecurityMiddleware(), new MaintenanceMiddleware(), new BasicAuthMiddleware(), @@ -34,6 +36,6 @@ final class RssBridge foreach (array_reverse($middlewares) as $middleware) { $action = fn ($req) => $middleware($req, $action); } - return $action($request); + return $action($request->withAttribute('action', $actionName)); } } diff --git a/lib/dependencies.php b/lib/dependencies.php index 227a66f1..45ae5d61 100644 --- a/lib/dependencies.php +++ b/lib/dependencies.php @@ -56,22 +56,14 @@ $container['logger'] = function () { // $logger->addHandler(new StreamHandler('/tmp/rss-bridge.txt', Logger::INFO)); // Uncomment this for debug logging to fs - //$logger->addHandler(new StreamHandler('/tmp/rss-bridge-debug.txt', Logger::DEBUG)); + // $logger->addHandler(new StreamHandler('/tmp/rss-bridge-debug.txt', Logger::DEBUG)); return $logger; }; $container['cache'] = function ($c) { /** @var CacheFactory $cacheFactory */ $cacheFactory = $c['cache_factory']; - $type = Configuration::getConfig('cache', 'type'); - if (!$type) { - throw new \Exception('No cache type configured'); - } - if (Debug::isEnabled()) { - $cache = $cacheFactory->create('array'); - } else { - $cache = $cacheFactory->create($type); - } + $cache = $cacheFactory->create(Configuration::getConfig('cache', 'type')); return $cache; }; diff --git a/middlewares/CacheMiddleware.php b/middlewares/CacheMiddleware.php new file mode 100644 index 00000000..ae0d0d33 --- /dev/null +++ b/middlewares/CacheMiddleware.php @@ -0,0 +1,59 @@ +cache = $cache; + } + + public function __invoke(Request $request, $next): Response + { + $action = $request->attribute('action'); + + if ($action !== 'DisplayAction') { + // We only cache DisplayAction (for now) + return $next($request); + } + + // TODO: might want to remove som params from query + $cacheKey = 'http_' . json_encode($request->toArray()); + $cachedResponse = $this->cache->get($cacheKey); + + if ($cachedResponse) { + $ifModifiedSince = $request->server('HTTP_IF_MODIFIED_SINCE'); + $lastModified = $cachedResponse->getHeader('last-modified'); + if ($ifModifiedSince && $lastModified) { + $lastModified = new \DateTimeImmutable($lastModified); + $lastModifiedTimestamp = $lastModified->getTimestamp(); + $modifiedSince = strtotime($ifModifiedSince); + // TODO: \DateTimeImmutable can be compared directly + if ($lastModifiedTimestamp <= $modifiedSince) { + $modificationTimeGMT = gmdate('D, d M Y H:i:s ', $lastModifiedTimestamp); + return new Response('', 304, ['last-modified' => $modificationTimeGMT . 'GMT']); + } + } + return $cachedResponse; + } + + /** @var Response $response */ + $response = $next($request); + + if (in_array($response->getCode(), [403, 429, 500, 503])) { + // Cache these responses for about ~20 mins on average + $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); + } + + // For 1% of requests, prune cache + if (rand(1, 100) === 1) { + // This might be resource intensive! + $this->cache->prune(); + } + + return $response; + } +} \ No newline at end of file diff --git a/middlewares/ExceptionMiddleware.php b/middlewares/ExceptionMiddleware.php new file mode 100644 index 00000000..8bb74713 --- /dev/null +++ b/middlewares/ExceptionMiddleware.php @@ -0,0 +1,24 @@ +logger = $logger; + } + + public function __invoke(Request $request, $next): Response + { + try { + return $next($request); + } catch (\Throwable $e) { + $this->logger->error('Exception in ExceptionMiddleware', ['e' => $e]); + + return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 500); + } + } +} \ No newline at end of file diff --git a/templates/html-format.html.php b/templates/html-format.html.php index 8e8abfcf..a62bb16a 100644 --- a/templates/html-format.html.php +++ b/templates/html-format.html.php @@ -22,7 +22,6 @@ -

    From 486191b419d545fcc2feec54378917b634812730 Mon Sep 17 00:00:00 2001 From: Dag Date: Mon, 2 Sep 2024 21:43:40 +0200 Subject: [PATCH 194/423] fix(cve_details) (#4251) --- bridges/CVEDetailsBridge.php | 42 +++++++++--------------------------- 1 file changed, 10 insertions(+), 32 deletions(-) diff --git a/bridges/CVEDetailsBridge.php b/bridges/CVEDetailsBridge.php index b52d290e..5f4f9daa 100644 --- a/bridges/CVEDetailsBridge.php +++ b/bridges/CVEDetailsBridge.php @@ -42,45 +42,23 @@ class CVEDetailsBridge extends BridgeAbstract $this->fetchContent(); } - foreach ($this->html->find('#searchresults > .row') as $i => $tr) { - // There are some optional vulnerability types, which will be - // added to the categories as well as the CWE number -- which is - // always given. - $categories = [$this->vendor]; - $enclosures = []; - - $detailLink = $tr->find('h3 > a', 0); - $detailHtml = getSimpleHTMLDOM($detailLink->href); - - // The CVE number itself + $var = $this->html->find('#searchresults > div > div.row'); + foreach ($var as $i => $tr) { + $uri = $tr->find('h3 > a', 0)->href ?? null; $title = $tr->find('h3 > a', 0)->innertext; - $content = $tr->find('.cvesummarylong', 0)->innertext; - $cweList = $detailHtml->find('h2', 2)->next_sibling(); - foreach ($cweList->find('li') as $li) { - $cweWithDescription = $li->find('a', 0)->innertext ?? ''; - - if (preg_match('/CWE-(\d+)/', $cweWithDescription, $cwe)) { - $categories[] = 'CWE-' . $cwe[1]; - $enclosures[] = 'https://cwe.mitre.org/data/definitions/' . $cwe[1] . '.html'; - } - } - - if ($this->product != '') { - $categories[] = $this->product; - } + $content = $tr->find('.cvesummarylong', 0)->innertext ?? ''; + $timestamp = $tr->find('[data-tsvfield="publishDate"]', 0)->innertext ?? 0; $this->items[] = [ - 'uri' => 'https://cvedetails.com/' . $detailHtml->find('h1 > a', 0)->href, + 'uri' => $uri, 'title' => $title, - 'timestamp' => $tr->find('[data-tsvfield="publishDate"]', 0)->innertext, + 'timestamp' => $timestamp, 'content' => $content, - 'categories' => $categories, - 'enclosures' => $enclosures, + 'categories' => [$this->vendor], + 'enclosures' => [], 'uid' => $title, ]; - - // We only want to fetch the latest 10 CVEs - if (count($this->items) >= 10) { + if (count($this->items) >= 30) { break; } } From 3dc8b65a0bb0296bdf0609c83841339014b1a663 Mon Sep 17 00:00:00 2001 From: July Date: Mon, 2 Sep 2024 15:49:49 -0400 Subject: [PATCH 195/423] [GovTrackBridge] Add feed for GovTrack events and blog (#4231) * [GovTrackBridge] Add feed for GovTrack events and blog * [GovTrackBridge] add missing default value * [GovTrackBridge] leaner items array and limit implementation --- bridges/GovTrackBridge.php | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 bridges/GovTrackBridge.php diff --git a/bridges/GovTrackBridge.php b/bridges/GovTrackBridge.php new file mode 100644 index 00000000..4674668a --- /dev/null +++ b/bridges/GovTrackBridge.php @@ -0,0 +1,125 @@ + [ + 'name' => 'Feed to track', + 'type' => 'list', + 'defaultValue' => 'posts', + 'values' => [ + 'All Legislative Activity' => 'bill-activity', + 'Bill Summaries' => 'bill-summaries', + 'Legislation Coming Up' => 'coming-up', + 'Major Legislative Activity' => 'major-bill-activity', + 'New Bills and Resolutions' => 'introduced-bills', + 'New Laws' => 'enacted-bills', + 'Posts from Us' => 'posts' + ] + ], + 'limit' => self::LIMIT + ]]; + + public function collectData() + { + $html = getSimpleHTMLDOMCached($this->getURI()); + if ($this->getInput('feed') != 'posts') { + $this->collectEvent($html); + return; + } + + $html = defaultLinkTo($html, parent::getURI()); + $limit = $this->getInput('limit') ?? 10; + foreach ($html->find('section') as $element) { + if (--$limit == 0) { + break; + } + + $info = explode(' ', $element->find('p', 0)->innertext); + $item = [ + 'categories' => [implode(' ', array_slice($info, 4))], + 'timestamp' => strtotime(implode(' ', array_slice($info, 0, 3))), + 'title' => $element->find('a', 0)->innertext, + 'uri' => $element->find('a', 0)->href, + ]; + + $html = getSimpleHTMLDOMCached($item['uri']); + $html = defaultLinkTo($html, parent::getURI()); + + $content = $html->find('#content .col-md', 1); + $info = explode(' by ', $content->find('p', 0)->plaintext); + $content->removeChild($content->firstChild()); + + $item['author'] = implode(' ', array_slice($info, 1)); + $item['content'] = $content->innertext; + + $this->items[] = $item; + } + } + + private function collectEvent($html) + { + $opt = []; + preg_match('/"csrfmiddlewaretoken" value="(.*)"/', $html, $opt); + $header = [ + "cookie: csrftoken=$opt[1]", + "x-csrftoken: $opt[1]", + 'referer: ' . parent::getURI(), + ]; + preg_match('/var selected_feed = "(.*)";/', $html, $opt); + $post = [ + 'count' => $this->getInput('limit') ?? 20, + 'feed' => $opt[1] + ]; + $opt = [ CURLOPT_POSTFIELDS => $post ]; + + $html = getContents(parent::getURI() . 'events/_load_events', $header, $opt); + $html = defaultLinkTo(str_get_html($html), parent::getURI()); + + foreach ($html->find('.tracked_event') as $event) { + $bill = $event->find('.event_title a, .event_body a', 0); + $date = explode(' ', $event->find('.event_date', 0)->plaintext); + preg_match('/Sponsor:(.*)\n/', $event->plaintext, $opt); + + $item = [ + 'author' => $opt[1] ?? '', + 'content' => $event->find('td', 1)->innertext, + 'enclosures' => [$event->find('img', 0)->src], + 'timestamp' => strtotime(implode(' ', array_slice($date, 2))), + 'title' => explode(': ', $bill->innertext)[0], + 'uri' => $bill->href, + ]; + + foreach ($event->find('.event_title, .event_type span') as $tag) { + if (!$tag->find('a', 0)) { + $item['categories'][] = $tag->plaintext; + } + } + + $this->items[] = $item; + } + } + + public function getName() + { + $name = parent::getName(); + if ($this->getInput('feed') != null) { + $name .= ' - ' . $this->getKey('feed'); + } + return $name; + } + + public function getURI() + { + if ($this->getInput('feed') != 'posts') { + $url = parent::getURI() . 'events/' . $this->getInput('feed'); + } else { + $url = parent::getURI() . $this->getInput('feed'); + } + return $url; + } +} From 293d04f296c33f7c524b84b7a0ae0d2625ca2729 Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 3 Sep 2024 07:02:37 +0200 Subject: [PATCH 196/423] fix(spotify): detect rate limiting (#4253) --- bridges/SpotifyBridge.php | 68 ++++++++++++++++++++++----------- middlewares/CacheMiddleware.php | 4 +- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/bridges/SpotifyBridge.php b/bridges/SpotifyBridge.php index e03d43a1..ab093a61 100644 --- a/bridges/SpotifyBridge.php +++ b/bridges/SpotifyBridge.php @@ -37,7 +37,10 @@ class SpotifyBridge extends BridgeAbstract 'name' => 'Spotify URIs', 'type' => 'text', 'required' => true, - 'exampleValue' => 'spotify:artist:4lianjyuR1tqf6oUX8kjrZ [,spotify:playlist:37i9dQZF1DXcBWIGoYBM5M,spotify:show:6ShFMYxeDNMo15COLObDvC]', + + // spotify:playlist:37i9dQZF1DXcBWIGoYBM5M + // spotify:show:6ShFMYxeDNMo15COLObDvC + 'exampleValue' => 'spotify:artist:4lianjyuR1tqf6oUX8kjrZ', ], 'albumtype' => [ 'name' => 'Album type', @@ -93,6 +96,25 @@ class SpotifyBridge extends BridgeAbstract private $token = ''; public function collectData() + { + /** + * https://developer.spotify.com/documentation/web-api/concepts/rate-limits + */ + $cacheKey = 'spotify_rate_limit'; + + try { + $this->collectDataInternal(); + } catch (HttpException $e) { + if ($e->getCode() === 429) { + $retryAfter = $e->response->getHeader('Retry-After') ?? (60 * 5); + $this->cache->set($cacheKey, true, $retryAfter); + throw new RateLimitException(sprintf('Rate limited by spotify, try again in %s seconds', $retryAfter)); + } + throw $e; + } + } + + private function collectDataInternal() { $this->fetchAccessToken(); @@ -125,6 +147,27 @@ class SpotifyBridge extends BridgeAbstract } } + private function fetchAccessToken() + { + $cacheKey = sprintf('SpotifyBridge:%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); + + $token = $this->cache->get($cacheKey); + if ($token) { + $this->token = $token; + } else { + $basicAuth = base64_encode(sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret'))); + $json = getContents('https://accounts.spotify.com/api/token', [ + "Authorization: Basic $basicAuth", + ], [ + CURLOPT_POSTFIELDS => 'grant_type=client_credentials', + ]); + $data = Json::decode($json); + $this->token = $data['access_token']; + + $this->cache->set($cacheKey, $this->token, 3600); + } + } + private function getEntriesFromQuery() { $entries = []; @@ -276,27 +319,6 @@ class SpotifyBridge extends BridgeAbstract return DateTime::createFromFormat('Y-m-d', $date)->getTimestamp(); } - private function fetchAccessToken() - { - $cacheKey = sprintf('SpotifyBridge:%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); - - $token = $this->cache->get($cacheKey); - if ($token) { - $this->token = $token; - } else { - $basicAuth = base64_encode(sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret'))); - $json = getContents('https://accounts.spotify.com/api/token', [ - "Authorization: Basic $basicAuth", - ], [ - CURLOPT_POSTFIELDS => 'grant_type=client_credentials', - ]); - $data = Json::decode($json); - $this->token = $data['access_token']; - - $this->cache->set($cacheKey, $this->token, 3600); - } - } - public function getURI() { if (empty($this->uri)) { @@ -346,4 +368,4 @@ class SpotifyBridge extends BridgeAbstract { return 'https://www.scdn.co/i/_global/favicon.png'; } -} +} \ No newline at end of file diff --git a/middlewares/CacheMiddleware.php b/middlewares/CacheMiddleware.php index ae0d0d33..bffde4af 100644 --- a/middlewares/CacheMiddleware.php +++ b/middlewares/CacheMiddleware.php @@ -44,8 +44,8 @@ class CacheMiddleware implements Middleware $response = $next($request); if (in_array($response->getCode(), [403, 429, 500, 503])) { - // Cache these responses for about ~20 mins on average - $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); + // Cache these responses for about ~10 mins on average + $this->cache->set($cacheKey, $response, 60 * 5 + rand(1, 60 * 10)); } // For 1% of requests, prune cache From 358bebbb890fce7a9636acfe9f21412dd1790fdb Mon Sep 17 00:00:00 2001 From: Pavel Korytov Date: Sat, 7 Sep 2024 08:02:27 +0500 Subject: [PATCH 197/423] [EconomistWorldInBriefBridge] Fix bridge (#4258) --- bridges/EconomistWorldInBriefBridge.php | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/bridges/EconomistWorldInBriefBridge.php b/bridges/EconomistWorldInBriefBridge.php index 4e65b15f..72b66198 100644 --- a/bridges/EconomistWorldInBriefBridge.php +++ b/bridges/EconomistWorldInBriefBridge.php @@ -60,20 +60,21 @@ class EconomistWorldInBriefBridge extends BridgeAbstract ]; } $html = getSimpleHTMLDOM(self::URI, $headers); - $gobbets = $html->find('._gobbets', 0); + $gobbets = $html->find('p[data-component="the-world-in-brief-paragraph"]'); if ($this->getInput('splitGobbets') == 1) { $this->splitGobbets($gobbets); } else { $this->mergeGobbets($gobbets); }; if ($this->getInput('agenda') == 1) { - $articles = $html->find('._articles', 0); + $articles = $html->find('div[data-test-id="chunks"] > div > div', 0); + if ($articles != null) { $this->collectArticles($articles); } } if ($this->getInput('quote') == 1) { - $quote = $html->find('._quote-container', 0); + $quote = $html->find('blockquote[data-test-id="inspirational-quote"]', 0); $this->addQuote($quote); } } @@ -83,7 +84,7 @@ class EconomistWorldInBriefBridge extends BridgeAbstract $today = new Datetime(); $today->setTime(0, 0, 0, 0); $limit = $this->getInput('limit'); - foreach ($gobbets->find('._gobbet') as $gobbet) { + foreach ($gobbets as $gobbet) { $title = $gobbet->plaintext; $match = preg_match('/[\.,]/', $title, $matches, PREG_OFFSET_CAPTURE); if ($match > 0) { @@ -109,7 +110,7 @@ class EconomistWorldInBriefBridge extends BridgeAbstract $today = new Datetime(); $today->setTime(0, 0, 0, 0); $contents = ''; - foreach ($gobbets->find('._gobbet') as $gobbet) { + foreach ($gobbets as $gobbet) { $contents .= "

    {$gobbet->innertext}"; } $this->items[] = [ @@ -126,10 +127,14 @@ class EconomistWorldInBriefBridge extends BridgeAbstract $i = 0; $today = new Datetime(); $today->setTime(0, 0, 0, 0); - foreach ($articles->find('._article') as $article) { - $title = $article->find('._headline', 0)->plaintext; - $image = $article->find('._main-image', 0); - $content = $article->find('._content', 0); + foreach ($articles->children() as $element) { + if ($element->tag != 'div') { + continue; + } + $image = $element->find('figure', 0); + $title = $element->find('h3', 0)->plaintext; + $content = $element->find('h3', 0)->parent(); + $content->find('h3', 0)->outertext = ''; $res_content = ''; if ($image != null && $this->getInput('agendaPictures') == 1) { From 40041dd65f9ea81abda0092dff2e160b61a5e682 Mon Sep 17 00:00:00 2001 From: tillcash Date: Mon, 9 Sep 2024 22:36:08 +0530 Subject: [PATCH 198/423] [DailythanthiBridge] fix url (#4261) --- bridges/DailythanthiBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/DailythanthiBridge.php b/bridges/DailythanthiBridge.php index 114f42d8..4e891e1c 100644 --- a/bridges/DailythanthiBridge.php +++ b/bridges/DailythanthiBridge.php @@ -3,7 +3,7 @@ class DailythanthiBridge extends BridgeAbstract { const NAME = 'Dailythanthi'; - const URI = 'https://www.dailythanthi.com/'; + const URI = 'https://www.dailythanthi.com'; const DESCRIPTION = 'Retrieve news from dailythanthi.com'; const MAINTAINER = 'tillcash'; const PARAMETERS = [ @@ -46,7 +46,7 @@ class DailythanthiBridge extends BridgeAbstract public function collectData() { - $dom = getSimpleHTMLDOM(self::URI . $this->getInput('topic')); + $dom = getSimpleHTMLDOM(self::URI . '/' . $this->getInput('topic')); foreach ($dom->find('div.ListingNewsWithMEDImage') as $element) { $slug = $element->find('a', 1); From d9316cdc606ff09a3623035e0766cd3ef2bc6904 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Wed, 11 Sep 2024 15:14:19 +0200 Subject: [PATCH 199/423] [PicukiBridge] Try to fix the bridge (#4262) This is a try to fix the bridge HTML parsing --- bridges/PicukiBridge.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bridges/PicukiBridge.php b/bridges/PicukiBridge.php index 5f1096b8..8e818bb0 100644 --- a/bridges/PicukiBridge.php +++ b/bridges/PicukiBridge.php @@ -58,26 +58,26 @@ class PicukiBridge extends BridgeAbstract } $count = 0; - foreach ($html->find('.box-photos .box-photo') as $element) { + foreach ($html->find('div[class=.box-photo][data-s=media]') as $element) { // skip ad items if (in_array('adv', explode(' ', $element->class))) { continue; } - $url = urljoin(self::URI, $element->find('a', 0)->href); - $html = getSimpleHTMLDOMCached($url); + $url = $element->find('a', 0)->href; + $html_single = getSimpleHTMLDOMCached($url); $sourceUrl = null; - if (preg_match($re, $html, $matches) > 0) { + if (preg_match($re, $html_single, $matches) > 0) { $sourceUrl = 'https://instagram.com/p/' . $matches[1]; } - $author = trim($element->find('.user-nickname', 0)->plaintext); + //$author = trim($element->find('.single-photo-nickname', 0)->plaintext); $date = date_create(); $relativeDate = str_replace(' ago', '', $element->find('.time', 0)->plaintext); date_sub($date, date_interval_create_from_date_string($relativeDate)); - $description = trim($element->find('.photo-description', 0)->plaintext); + $description = trim($element->find('.photo-action-description', 0)->plaintext); $isVideo = (bool) $element->find('.video-icon', 0); $videoNote = $isVideo ? '

    (video)

    ' : ''; @@ -91,7 +91,7 @@ class PicukiBridge extends BridgeAbstract $this->items[] = [ 'uri' => $url, - 'author' => $author, + /*'author' => $author,*/ 'timestamp' => date_format($date, 'r'), 'title' => strlen($description) > 60 ? mb_substr($description, 0, 57) . '...' : $description, 'thumbnail' => $imageUrl, From 80c43f10d83dcf4c0b9ff2707c6fe08fff8869ed Mon Sep 17 00:00:00 2001 From: osvfj Date: Thu, 12 Sep 2024 09:07:22 +0000 Subject: [PATCH 200/423] [TCBScansBridge] Add bridge (#4263) --- bridges/TCBScansBridge.php | 102 +++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 bridges/TCBScansBridge.php diff --git a/bridges/TCBScansBridge.php b/bridges/TCBScansBridge.php new file mode 100644 index 00000000..a07c3f5b --- /dev/null +++ b/bridges/TCBScansBridge.php @@ -0,0 +1,102 @@ + [ + 'name' => 'Manga', + 'title' => 'Select your prefered manga', + 'exampleValue' => 'One Piece', + 'type' => 'list', + 'values' => [ + 'Ace Novel - Manga Adaptation' => 'mangas/1/ace-novel-manga-adaptation', + 'Attack on Titan' => 'mangas/8/attack-on-titan', + 'Black Clover' => 'mangas/3/black-clover', + 'Black Clover Gaiden: Quartet Knights' => 'mangas/24/black-clover-gaiden-quartet-knights', + 'Bleach' => 'mangas/2/bleach', + 'Build King' => 'mangas/9/build-king', + 'Chainsaw Man' => 'mangas/13/chainsaw-man', + 'Demon Slayer: Kimetsu no Yaiba ' => 'mangas/19/demon-slayer-kimetsu-no-yaiba', + 'Haikyuu!! (New Special!)' => 'mangas/11/haikyu-special', + 'Hunter X Hunter' => 'mangas/15/hunter-x-hunter', + 'Jujutsu Kaisen' => 'mangas/4/jujutsu-kaisen', + 'My Hero Academia' => 'mangas/6/my-hero-academia', + "My Hero Academia One-Shot: You're Next!!" => 'mangas/25/my-hero-academia-one-shot-you-re-next', + 'One Piece ' => 'mangas/5/one-piece', + 'One Piece - Nami vs Kalifa by Boichi' => 'mangas/12/one-piece-nami-vs-kalifa-by-boichi', + 'One-Punch Man' => 'mangas/10/one-punch-man', + 'Spy X Family' => 'mangas/23/spy-x-family', + ], + ], + 'full_chapter' => [ + 'name' => 'Load images in the item', + 'type' => 'checkbox', + 'title' => 'Activate to always load the full chapter', + 'defaultValue' => 'checked' + ], + 'hide_title' => [ + 'name' => 'Hide title of the chapter', + 'type' => 'checkbox', + 'title' => 'Activate to hide the title of the chapter and just show the number' + ] + ]]; + const CACHE_TIMEOUT = 60 * 15; + + public function collectData() + { + $manga = $this->getInput('manga'); + $html = getSimpleHTMLDOMCached($this->getURI() . $manga); + $html = defaultLinkTo($html, $this->getURI()); + $full_chapter = $this->getInput('full_chapter'); + + $chapter = $html->find('a.block.border.border-border.bg-card.mb-3.p-3.rounded', 0); + + $item = []; + $item['title'] = $this->getInput('hide_title') ? $chapter->find('div.text-lg.font-bold', 0)->plaintext : $chapter->find('div.text-gray-500', 0)->plaintext; + $item['uri'] = $chapter->href; + $item['uid'] = $chapter->href; + + + if ($full_chapter) { + $item['content'] = $this->getFullChapter($item['uri']); + } else { + $item['content'] = <<Read chapter + EOD; + ; + } + $this->items[] = $item; + } + + private function getFullChapter($uri) + { + $html = getSimpleHTMLDOMCached($uri); + $pictures = $html->find('picture.fixed-ratio'); + $img_html = ''; + + foreach ($pictures as $picture) { + $img_html .= << + EOD; + } + return $img_html; + } + + public function getName() + { + if (!is_null($this->getKey('manga'))) { + return $this->getKey('manga') . ' | ' . self::NAME; + } + + return self::NAME; + } + + public function getIcon() + { + return $this->getURI() . '/files/favicon-32x32.png'; + } +} \ No newline at end of file From af26d845d907cf810bd4028a5ea587fb902a9225 Mon Sep 17 00:00:00 2001 From: Mynacol Date: Mon, 7 Oct 2024 20:26:48 +0200 Subject: [PATCH 201/423] Include all bridges in tarballs Currently, two "demo" and "example" bridges are excluded from GitHub's autogenerated tarballs. As I argued, those files can still be helpful for integration tests, as they are run in NixOS and don't need internet access or depend on the availability of external services [1]. Additionally, the official docker image builds from the checkout so it includes those bridges when users use containers or a git checkout compared to tarballs. This commit therefore unifies the list of available bridges between deployment methods. [1] https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/web-apps/rss-bridge.nix#L20 --- .gitattributes | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 5981fb34..28053256 100644 --- a/.gitattributes +++ b/.gitattributes @@ -47,8 +47,6 @@ phpcs.xml export-ignore phpcompatibility.xml export-ignore tests/ export-ignore cache/.gitkeep export-ignore -bridges/DemoBridge.php export-ignore -bridges/FeedExpanderExampleBridge.php export-ignore ## Composer # From 63c16e470daf6ee4fd2123295bd05ae0a15c7c0c Mon Sep 17 00:00:00 2001 From: Bocki Date: Wed, 16 Oct 2024 15:36:57 +0200 Subject: [PATCH 202/423] [prtester] Rework test storage (#4292) * Update prtester.py * Update prhtmlgenerator.yml --- .github/prtester.py | 14 ++++---- .github/workflows/prhtmlgenerator.yml | 50 ++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/.github/prtester.py b/.github/prtester.py index 3d7dae99..53d77725 100644 --- a/.github/prtester.py +++ b/.github/prtester.py @@ -4,7 +4,7 @@ import re from bs4 import BeautifulSoup from datetime import datetime from typing import Iterable -import os.path +import os import urllib # This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge @@ -39,6 +39,8 @@ def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool) -> Iterable: instance_suffix = '' + prid = os.getenv("PR") + tester_url = f'https://rss-bridge.github.io/rss-bridge-tests/prs/{prid}' if instance.name: instance_suffix = f' ({instance.name})' table_rows = [] @@ -140,10 +142,10 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w if status_is_ok: status = '✔️' if with_upload and (not with_reduced_upload or not status_is_ok): - termpad = requests.post(url="https://termpad.com/", data=page_text) - termpad_url = termpad.text.strip() - termpad_url = termpad_url.replace('termpad.com/','termpad.com/raw/') - table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({termpad_url}) | {status} |') + filename = f'{os.getcwd()}/{instance.name}_{form_number}.html' + with open(file=filename, mode='wb') as file: + file.write(page_text) + table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({tester_url}/{instance.name}_{form_number}.html) | {status} |') form_number += 1 return table_rows @@ -187,4 +189,4 @@ if __name__ == '__main__': with_reduced_upload=args.reduced_upload and not args.no_upload, title=args.title, output_file=args.output_file - ); \ No newline at end of file + ); diff --git a/.github/workflows/prhtmlgenerator.yml b/.github/workflows/prhtmlgenerator.yml index 7985250a..90343973 100644 --- a/.github/workflows/prhtmlgenerator.yml +++ b/.github/workflows/prhtmlgenerator.yml @@ -13,7 +13,7 @@ jobs: # Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989 steps: - name: Check out self - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} @@ -33,7 +33,7 @@ jobs: docker build -t prbuild .; docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3001:80 prbuild - name: Setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.7' cache: 'pip' @@ -51,9 +51,17 @@ jobs: body="${body//$'\n'/'%0A'}"; body="${body//$'\r'/'%0D'}"; echo "bodylength=${#body}" >> $GITHUB_OUTPUT + env: + PR: ${{ github.event.number }} + - name: Upload generated tests + uses: actions/upload-artifact@v4 + id: upload-generated-tests + with: + name: tests + path: '*.html' - name: Find Comment if: ${{ steps.testrun.outputs.bodylength > 130 }} - uses: peter-evans/find-comment@v2 + uses: peter-evans/find-comment@v3 id: fc with: issue-number: ${{ github.event.pull_request.number }} @@ -61,9 +69,43 @@ jobs: body-includes: Pull request artifacts - name: Create or update comment if: ${{ steps.testrun.outputs.bodylength > 130 }} - uses: peter-evans/create-or-update-comment@v2 + uses: peter-evans/create-or-update-comment@v4 with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.pull_request.number }} body-file: comment.txt edit-mode: replace + upload_tests: + name: Upload tests + runs-on: ubuntu-latest + needs: test-pr + steps: + - uses: actions/checkout@v4 + with: + repository: 'RSS-Bridge/rss-bridge-tests' + ref: 'main' + token: ${{ secrets.RSSTESTER_ACTION }} + + - name: Setup git config + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "<>" + + - name: Download tests + uses: actions/download-artifact@v4 + with: + name: tests + + - name: Move tests + run: | + cd prs + mkdir -p ${{github.event.number}} + cd ${{github.event.number}} + mv -f $GITHUB_WORKSPACE/*.html . + + - name: Commit and push generated tests + run: | + export COMMIT_MESSAGE="Added tests for PR ${{github.event.number}}" + git add . + git commit -m "$COMMIT_MESSAGE" + git push From 6aba9fdf541c7fecea56bc6fc6496372f1679314 Mon Sep 17 00:00:00 2001 From: tillcash Date: Wed, 16 Oct 2024 22:05:06 +0530 Subject: [PATCH 203/423] [MaalaimalarBridge] fix url (#4295) --- bridges/MaalaimalarBridge.php | 90 +++++++++++++++++------------------ 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/bridges/MaalaimalarBridge.php b/bridges/MaalaimalarBridge.php index 59c76b01..e83eafe2 100644 --- a/bridges/MaalaimalarBridge.php +++ b/bridges/MaalaimalarBridge.php @@ -3,7 +3,7 @@ class MaalaimalarBridge extends BridgeAbstract { const NAME = 'Maalaimalar'; - const URI = 'https://www.maalaimalar.com/'; + const URI = 'https://www.maalaimalar.com'; const DESCRIPTION = 'Retrieve news from maalaimalar.com'; const MAINTAINER = 'tillcash'; const PARAMETERS = [ @@ -13,54 +13,54 @@ class MaalaimalarBridge extends BridgeAbstract 'type' => 'list', 'values' => [ 'news' => [ - 'tamilnadu' => 'news/state', - 'puducherry' => 'puducherry', - 'india' => 'news/national', - 'world' => 'news/world', + 'tamilnadu' => '/news/state', + 'puducherry' => '/news/puducherry', + 'india' => '/news/national', + 'world' => '/news/world', ], 'district' => [ - 'chennai' => 'chennai', - 'ariyalur' => 'ariyalur', - 'chengalpattu' => 'chengalpattu', - 'coimbatore' => 'coimbatore', - 'cuddalore' => 'cuddalore', - 'dharmapuri' => 'dharmapuri', - 'dindugal' => 'dindugal', - 'erode' => 'erode', - 'kaanchepuram' => 'kaanchepuram', - 'kallakurichi' => 'kallakurichi', - 'kanyakumari' => 'kanyakumari', - 'karur' => 'karur', - 'krishnagiri' => 'krishnagiri', - 'madurai' => 'madurai', - 'mayiladuthurai' => 'mayiladuthurai', - 'nagapattinam' => 'nagapattinam', - 'namakal' => 'namakal', - 'nilgiris' => 'nilgiris', - 'perambalur' => 'perambalur', - 'pudukottai' => 'pudukottai', - 'ramanathapuram' => 'ramanathapuram', - 'ranipettai' => 'ranipettai', - 'salem' => 'salem', - 'sivagangai' => 'sivagangai', - 'tanjore' => 'tanjore', - 'theni' => 'theni', - 'thenkasi' => 'thenkasi', - 'thiruchirapalli' => 'thiruchirapalli', - 'thirunelveli' => 'thirunelveli', - 'thirupathur' => 'thirupathur', - 'thiruvarur' => 'thiruvarur', - 'thoothukudi' => 'thoothukudi', - 'tirupur' => 'tirupur', - 'tiruvallur' => 'tiruvallur', - 'tiruvannamalai' => 'tiruvannamalai', - 'vellore' => 'vellore', - 'villupuram' => 'villupuram', - 'virudhunagar' => 'virudhunagar', + 'chennai' => '/news/district/chennai', + 'ariyalur' => '/news/district/ariyalur', + 'chengalpattu' => '/news/district/chengalpattu', + 'coimbatore' => '/news/district/coimbatore', + 'cuddalore' => '/news/district/cuddalore', + 'dharmapuri' => '/news/district/dharmapuri', + 'dindugal' => '/news/district/dindugal', + 'erode' => '/news/district/erode', + 'kaanchepuram' => '/news/district/kaanchepuram', + 'kallakurichi' => '/news/district/kallakurichi', + 'kanyakumari' => '/news/district/kanyakumari', + 'karur' => '/news/district/karur', + 'krishnagiri' => '/news/district/krishnagiri', + 'madurai' => '/news/district/madurai', + 'mayiladuthurai' => '/news/district/mayiladuthurai', + 'nagapattinam' => '/news/district/nagapattinam', + 'namakal' => '/news/district/namakal', + 'nilgiris' => '/news/district/nilgiris', + 'perambalur' => '/news/district/perambalur', + 'pudukottai' => '/news/district/pudukottai', + 'ramanathapuram' => '/news/district/ramanathapuram', + 'ranipettai' => '/news/district/ranipettai', + 'salem' => '/news/district/salem', + 'sivagangai' => '/news/district/sivagangai', + 'tanjore' => '/news/district/tanjore', + 'theni' => '/news/district/theni', + 'thenkasi' => '/news/district/thenkasi', + 'thiruchirapalli' => '/news/district/thiruchirapalli', + 'thirunelveli' => '/news/district/thirunelveli', + 'thirupathur' => '/news/district/thirupathur', + 'thiruvarur' => '/news/district/thiruvarur', + 'thoothukudi' => '/news/district/thoothukudi', + 'tirupur' => '/news/district/tirupur', + 'tiruvallur' => '/news/district/tiruvallur', + 'tiruvannamalai' => '/news/district/tiruvannamalai', + 'vellore' => '/news/district/vellore', + 'villupuram' => '/news/district/villupuram', + 'virudhunagar' => '/news/district/virudhunagar', ], 'cinema' => [ - 'news' => 'cinema/cinemanews', - 'gossip' => 'cinema/gossip', + 'news' => '/cinema/cinemanews', + 'gossip' => '/cinema/gossip', ], ], ], From eb21e97d014decf37bdaef5e7c6254607db12c71 Mon Sep 17 00:00:00 2001 From: Tostiman <18124323+t0stiman@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:37:30 +0200 Subject: [PATCH 204/423] [OvertakeBridge] Renamed RaceDepartmentBridge to OvertakeBridge (#4294) --- .../{RaceDepartmentBridge.php => OvertakeBridge.php} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename bridges/{RaceDepartmentBridge.php => OvertakeBridge.php} (78%) diff --git a/bridges/RaceDepartmentBridge.php b/bridges/OvertakeBridge.php similarity index 78% rename from bridges/RaceDepartmentBridge.php rename to bridges/OvertakeBridge.php index a601b4d6..6de15276 100644 --- a/bridges/RaceDepartmentBridge.php +++ b/bridges/OvertakeBridge.php @@ -1,16 +1,16 @@ collectExpandableDatas('https://www.racedepartment.com/ams/index.rss', 10); + $this->collectExpandableDatas('https://www.overtake.gg/ams/index.rss', 10); } protected function parseItem(array $item) From 3a5de759fa74252d8ea7c76e84d7a552bd5af8fb Mon Sep 17 00:00:00 2001 From: Tostiman <18124323+t0stiman@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:57:44 +0200 Subject: [PATCH 205/423] [CarThrottleBridge] update for new layout (#4285) --- bridges/CarThrottleBridge.php | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/bridges/CarThrottleBridge.php b/bridges/CarThrottleBridge.php index 0a05ee60..2b8ca2b2 100644 --- a/bridges/CarThrottleBridge.php +++ b/bridges/CarThrottleBridge.php @@ -80,31 +80,22 @@ class CarThrottleBridge extends BridgeAbstract $item['author'] = $this->parseAuthor($articlePage); - $articleElement = $articlePage->find('article')[0]; + $articleImage = $articlePage->find('div.block-layout-field-image')[0]; + $article = $articlePage->find('div.block-layout-body')[1]; //remove ads - foreach ($articleElement->find('aside') as $ad) { + foreach ($article->find('aside') as $ad) { $ad->outertext = ''; } - $summary = $articleElement->find('div.summary')[0]; - - //remove header so we are left with the article content - foreach ($articleElement->find('header') as $found) { - $found->outertext = ''; - } - - //remove comments (registering on carthrottle.com is impossible so the comment sections are empty anyway) - foreach ($articleElement->find('#lbs-comments') as $found) { - $found->outertext = ''; - } + $summary = $articlePage->find('div.summary')[0]; //these are supposed to be hidden - foreach ($articleElement->find('.visually-hidden') as $found) { + foreach ($article->find('.visually-hidden') as $found) { $found->outertext = ''; } - $item['content'] = $summary . $articleElement; + $item['content'] = $summary . $articleImage . $article; array_push($this->items, $item); } @@ -117,7 +108,7 @@ class CarThrottleBridge extends BridgeAbstract return ''; } - $a = $authorDivs[0]->find('a'); + $a = $authorDivs[0]->find('a')[0]; if ($a) { return $a->innertext; } From cdf21d48e5dbdd09f41528cc9e522507d67adf71 Mon Sep 17 00:00:00 2001 From: Eugene Molotov Date: Wed, 16 Oct 2024 21:58:18 +0500 Subject: [PATCH 206/423] [RutubeBridge] Multiple fixes (#4284) --- bridges/RutubeBridge.php | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/bridges/RutubeBridge.php b/bridges/RutubeBridge.php index f8e106c8..6e48c27a 100644 --- a/bridges/RutubeBridge.php +++ b/bridges/RutubeBridge.php @@ -66,7 +66,15 @@ class RutubeBridge extends BridgeAbstract { $jsonDataRegex = '/window.reduxState = (.*);/'; preg_match($jsonDataRegex, $html, $matches) or returnServerError('Could not find reduxState'); - return json_decode(str_replace('\x', '\\\x', $matches[1])); + $map = [ + '\x26' => '&', + '\x3c' => '<', + '\x3d' => '=', + '\x3e' => '>', + '\x3f' => '?', + ]; + $jsonString = str_replace(array_keys($map), array_values($map), $matches[1]); + return json_decode($jsonString, false); } private function getVideosFromReduxState() @@ -77,8 +85,10 @@ class RutubeBridge extends BridgeAbstract $reduxState = $this->getJSONData($html); $videos = []; if ($this->getInput('c')) { - $videos = $reduxState->userChannel->videos->results; - $this->title = $reduxState->userChannel->info->name; + $videosMethod = 'videos(' . $this->getInput('c') . ')'; + $channelInfoMethod = 'channelInfo({"userChannelId":' . $this->getInput('c') . '})'; + $videos = $reduxState->api->queries->$videosMethod->data->results; + $this->title = $reduxState->api->queries->$channelInfoMethod->data->name; } elseif ($this->getInput('p')) { $playListVideosMethod = 'getPlaylistVideos(' . $this->getInput('p') . ')'; $videos = $reduxState->api->queries->$playListVideosMethod->data->results; From f89c75b4b8d7bf054fccd2905b7de6eb97f73c91 Mon Sep 17 00:00:00 2001 From: Pavel Korytov Date: Wed, 16 Oct 2024 19:59:36 +0300 Subject: [PATCH 207/423] [ArsTechnicaBridge] Fix the bridge after redesign (#4282) --- bridges/ArsTechnicaBridge.php | 79 +++++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index fcb1bd4f..ac722dc9 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -37,39 +37,82 @@ class ArsTechnicaBridge extends FeedExpander { $item_html = getSimpleHTMLDOMCached($item['uri']); $item_html = defaultLinkTo($item_html, self::URI); - $item['content'] = $item_html->find('.article-content', 0); + + $content = ''; + $header = $item_html->find('article header', 0); + $leading = $header->find('p[class*=leading]', 0); + if ($leading != null) { + $content .= '

    ' . $leading->innertext . '

    '; + } + $intro_image = $header->find('img.intro-image', 0); + if ($intro_image != null) { + $content .= '
    ' . $intro_image; + + $image_caption = $header->find('.caption .caption-content', 0); + if ($image_caption != null) { + $content .= '
    ' . $image_caption->innertext . '
    '; + } + $content .= '
    '; + } + + foreach ($item_html->find('.post-content') as $content_tag) { + $content .= $content_tag->innertext; + } + + $item['content'] = str_get_html($content); $parsely = $item_html->find('[name="parsely-page"]', 0); $parsely_json = json_decode(html_entity_decode($parsely->content), true); $item['categories'] = $parsely_json['tags']; - $pages = $item_html->find('nav.page-numbers > .numbers > a', -2); - if (null !== $pages) { - for ($i = 2; $i <= $pages->innertext; $i++) { - $page_url = $item['uri'] . '&page=' . $i; - $page_html = getSimpleHTMLDOMCached($page_url); - $page_html = defaultLinkTo($page_html, self::URI); - $item['content'] .= $page_html->find('.article-content', 0); + // Some lightboxes are nested in figures. I'd guess that's a + // bug in the website + foreach ($item['content']->find('figure div div.ars-lightbox') as $weird_lightbox) { + $weird_lightbox->parent->parent->outertext = $weird_lightbox; + } + + // It's easier to reconstruct the whole thing than remove + // duplicate reactive tags + foreach ($item['content']->find('.ars-lightbox') as $lightbox) { + $lightbox_content = ''; + foreach ($lightbox->find('.ars-lightbox-item') as $lightbox_item) { + $img = $lightbox_item->find('img', 0); + if ($img != null) { + $lightbox_content .= '
    ' . $img; + $caption = $lightbox_item->find('div.pswp-caption-content', 0); + if ($caption != null) { + $credit = $lightbox_item->find('div.ars-gallery-caption-credit', 0); + if ($credit != null) { + $credit->innertext = 'Credit: ' . $credit->innertext; + } + $lightbox_content .= '
    ' . $caption->innertext . '
    '; + } + $lightbox_content .= '
    '; + } } - $item['content'] = str_get_html($item['content']); + $lightbox->innertext = $lightbox_content; } // remove various ars advertising - $item['content']->find('#social-left', 0)->remove(); - foreach ($item['content']->find('.ars-component-buy-box') as $ad) { + foreach ($item['content']->find('.ars-interlude-container') as $ad) { $ad->remove(); } - foreach ($item['content']->find('.ad_wrapper') as $ad) { - $ad->remove(); + foreach ($item['content']->find('.toc-container') as $toc) { + $toc->remove(); } - foreach ($item['content']->find('.sidebar') as $ad) { - $ad->remove(); + + // Mostly YouTube videos + $iframes = $item['content']->find('iframe'); + foreach ($iframes as $iframe) { + $iframe->outertext = '' . $iframe->src . ''; + } + // This fixed padding around the former iframes and actual inline videos + foreach ($item['content']->find('div[style*=aspect-ratio]') as $styled) { + $styled->removeAttribute('style'); } $item['content'] = backgroundToImg($item['content']); - - $item['uid'] = explode('=', $item['uri'])[1]; - + $item['uid'] = strval($parsely_json['post_id']); return $item; } } From bd823100cdf1b05a52a11ead7267a05a297095e0 Mon Sep 17 00:00:00 2001 From: vlnst Date: Wed, 16 Oct 2024 20:04:26 +0300 Subject: [PATCH 208/423] [maint] Update instance location (#4279) --- docs/01_General/06_Public_Hosts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index fa8d5fdd..7d921d20 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -24,7 +24,7 @@ | ![](https://iplookup.flagfox.net/images/h16/UA.png) | https://rss.noleron.com | ![](https://img.shields.io/website/https/rss.noleron.com) | [@ihor](https://noleron.com/about) | Hosted with Hosting Ukraine, Ukraine | ![](https://iplookup.flagfox.net/images/h16/IN.png) | https://rssbridge.projectsegfau.lt | ![](https://img.shields.io/website/https/rssbridge.projectsegfau.lt) | [@gi-yt](https://aryak.me) | Self-Hosted at Mumbai, India with Airtel (ISP) | | ![](https://iplookup.flagfox.net/images/h16/US.png) | https://rb.vern.cc | ![](https://img.shields.io/website/https/rb.vern.cc.svg) | [@vern.cc](https://vern.cc/en/admin) | Hosted with Hetzner, US | -| ![](https://iplookup.flagfox.net/images/h16/RO.png) | https://rss.bloat.cat | ![](https://img.shields.io/website/https/rss.bloat.cat) | [@vlnst](https://bloat.cat/contact) | Hosted with Kyun, Romania | +| ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss.bloat.cat | ![](https://img.shields.io/website/https/rss.bloat.cat) | [@vlnst](https://bloat.cat/contact) | Hosted with Datalix, Germany | ## Inactive instances From 5d8305067306c95297fcb1a9f504e13ac5a45452 Mon Sep 17 00:00:00 2001 From: tillcash Date: Wed, 16 Oct 2024 22:43:00 +0530 Subject: [PATCH 209/423] [ForensicArchitectureBridge] Add Bridge (#4280) --- bridges/ForensicArchitectureBridge.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 bridges/ForensicArchitectureBridge.php diff --git a/bridges/ForensicArchitectureBridge.php b/bridges/ForensicArchitectureBridge.php new file mode 100644 index 00000000..5f024f3a --- /dev/null +++ b/bridges/ForensicArchitectureBridge.php @@ -0,0 +1,25 @@ +investigations as $investigation) { + $this->items[] = [ + 'content' => $investigation->abstract, + 'timestamp' => $investigation->publication_date, + 'title' => $investigation->title, + 'uid' => $investigation->id, + 'uri' => self::URI . 'investigation/' . $investigation->slug, + ]; + } + } +} From 0c96a47e8cd1c9567e1638b820be95f875d0716b Mon Sep 17 00:00:00 2001 From: somini Date: Wed, 16 Oct 2024 18:14:06 +0100 Subject: [PATCH 210/423] Remove PanacheDigitalGamesBridge (#4277) The Blog has a feed now: https://panachedigitalgames.com/en/feed/ --- bridges/PanacheDigitalGamesBridge.php | 50 --------------------------- 1 file changed, 50 deletions(-) delete mode 100644 bridges/PanacheDigitalGamesBridge.php diff --git a/bridges/PanacheDigitalGamesBridge.php b/bridges/PanacheDigitalGamesBridge.php deleted file mode 100644 index bb0c00b9..00000000 --- a/bridges/PanacheDigitalGamesBridge.php +++ /dev/null @@ -1,50 +0,0 @@ -getURI(); - $html = getSimpleHTMLDOMCached($articles); - - foreach ($html->find('.news-item') as $element) { - $item = []; - - $title = $element->find('.news-item-texts-title', 0); - $link = $element->find('.news-item-texts a', 0); - $timestamp = $element->find('.news-item-texts-date', 0); - - $item['title'] = $title->plaintext; - $item['uri'] = self::URI . $link->href; - $item['timestamp'] = strtotime($timestamp->plaintext); - - $image_html = $element->find('.news-item-thumbnail-image', 0); - if ($image_html) { - $image_strings = explode('\'', $image_html); - - if (count($image_strings) == 4) { - $item['content'] = ''; - } - } - - $this->items[] = $item; - } - } -} From e7d6f89887f49b4ff86a34f2095017f4d8495569 Mon Sep 17 00:00:00 2001 From: Bocki Date: Wed, 16 Oct 2024 19:21:24 +0200 Subject: [PATCH 211/423] [ForensicArchitecture] Remove for bugfixing (#4297) --- bridges/ForensicArchitectureBridge.php | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 bridges/ForensicArchitectureBridge.php diff --git a/bridges/ForensicArchitectureBridge.php b/bridges/ForensicArchitectureBridge.php deleted file mode 100644 index 5f024f3a..00000000 --- a/bridges/ForensicArchitectureBridge.php +++ /dev/null @@ -1,25 +0,0 @@ -investigations as $investigation) { - $this->items[] = [ - 'content' => $investigation->abstract, - 'timestamp' => $investigation->publication_date, - 'title' => $investigation->title, - 'uid' => $investigation->id, - 'uri' => self::URI . 'investigation/' . $investigation->slug, - ]; - } - } -} From e5e2059ed777dde67bb5e8adb8303134d48c6f9e Mon Sep 17 00:00:00 2001 From: Bocki Date: Wed, 16 Oct 2024 19:46:56 +0200 Subject: [PATCH 212/423] [maint] Update all workflow action versions (#4298) --- .github/workflows/dockerbuild.yml | 12 ++++++------ .github/workflows/documentation.yml | 2 +- .github/workflows/lint.yml | 6 +++--- .github/workflows/tests.yml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dockerbuild.yml b/.github/workflows/dockerbuild.yml index 82d8611a..39645558 100644 --- a/.github/workflows/dockerbuild.yml +++ b/.github/workflows/dockerbuild.yml @@ -21,7 +21,7 @@ jobs: - name: Docker meta id: docker_meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: | ${{ env.DOCKERHUB_SLUG }} @@ -33,26 +33,26 @@ jobs: type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/20') }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/bake-action@v2 + uses: docker/bake-action@v5 with: files: | ./docker-bake.hcl diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b00c898a..e0201022 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -9,7 +9,7 @@ jobs: documentation: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: persist-credentials: false - name: Setup PHP diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 83911ab6..206b53de 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,7 +13,7 @@ jobs: matrix: php-versions: ['7.4'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php-versions }} @@ -26,7 +26,7 @@ jobs: matrix: php-versions: ['7.4'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php-versions }} @@ -38,7 +38,7 @@ jobs: executable_php_files_check: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: | if find -name "*.php" -executable -type f -print -exec false {} + then diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e7684e6b..af7dc29c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: matrix: php-versions: ['7.4', '8.0', '8.1'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php-versions }} From 776e27218ad03cd9299319be2100d4065bba566b Mon Sep 17 00:00:00 2001 From: Bocki Date: Thu, 17 Oct 2024 00:00:52 +0200 Subject: [PATCH 213/423] [maint] fix phpunit test (#4300) --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index af7dc29c..93f07b0f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,5 +17,7 @@ jobs: - uses: shivammathur/setup-php@v2 with: php-version: ${{ matrix.php-versions }} + env: + update: true - run: composer install - run: composer test From 70cf917f090bea8b98e62f9e10aa99d946bd9106 Mon Sep 17 00:00:00 2001 From: Bocki Date: Thu, 17 Oct 2024 00:09:35 +0200 Subject: [PATCH 214/423] [ForensicArchitecture] Create ForensicArchitectureBridge.php (#4301) --- bridges/ForensicArchitectureBridge.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 bridges/ForensicArchitectureBridge.php diff --git a/bridges/ForensicArchitectureBridge.php b/bridges/ForensicArchitectureBridge.php new file mode 100644 index 00000000..5f024f3a --- /dev/null +++ b/bridges/ForensicArchitectureBridge.php @@ -0,0 +1,25 @@ +investigations as $investigation) { + $this->items[] = [ + 'content' => $investigation->abstract, + 'timestamp' => $investigation->publication_date, + 'title' => $investigation->title, + 'uid' => $investigation->id, + 'uri' => self::URI . 'investigation/' . $investigation->slug, + ]; + } + } +} From 664436c5f4c9520b039ffcc7d23cb9c788b1e343 Mon Sep 17 00:00:00 2001 From: Bocki Date: Thu, 17 Oct 2024 01:25:07 +0200 Subject: [PATCH 215/423] [prtester] Optimize tester workflow (#4303) --- .github/workflows/prhtmlgenerator.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/prhtmlgenerator.yml b/.github/workflows/prhtmlgenerator.yml index 90343973..c0105e29 100644 --- a/.github/workflows/prhtmlgenerator.yml +++ b/.github/workflows/prhtmlgenerator.yml @@ -5,9 +5,24 @@ on: branches: [ master ] jobs: + check-bridges: + name: Check if bridges were changed + runs-on: ubuntu-latest + outputs: + BRIDGES: ${{ steps.check1.outputs.BRIDGES }} + steps: + - name: Check number of bridges + id: check1 + run: | + PR=${{github.event.number}}; + wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch; + bridgeamount=$(cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq | wc -l); + echo "BRIDGES=$bridgeamount" >> "$GITHUB_OUTPUT" test-pr: name: Generate HTML runs-on: ubuntu-latest + needs: check-bridges + if: needs.check-bridges.outputs.BRIDGES > 0 env: PYTHONUNBUFFERED: 1 # Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989 From 56994b3b5c732089548c7fba49286916550e569a Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Thu, 17 Oct 2024 08:47:44 +0200 Subject: [PATCH 216/423] [ZeitBridge] Remove content from original feed (#4260) The original feed contains a small version of the header image and the summary or a literal "None". The header image is already added, but the original content was kept. This removes the original content and adds the summary if it exists. --- bridges/ZeitBridge.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index ae8a1a66..d4d66a1c 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -117,6 +117,14 @@ class ZeitBridge extends FeedExpander }, $authors)); } + $item['content'] = ''; + + // summary + $summary = $article->find('.summary'); + if ($summary) { + $item['content'] .= implode('', $summary); + } + // header image $headerimg = $article->find('*[data-ct-row="headerimage"]', 0) ?? $article->find('.article-header', 0) ?? $article->find('header', 0); if ($headerimg) { From bd88bc27d3113717caf28023bbeb39a02f58901f Mon Sep 17 00:00:00 2001 From: Tostiman <18124323+t0stiman@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:14:51 +0200 Subject: [PATCH 217/423] [TheDrive] New bridge (#4304) --- bridges/TheDriveBridge.php | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 bridges/TheDriveBridge.php diff --git a/bridges/TheDriveBridge.php b/bridges/TheDriveBridge.php new file mode 100644 index 00000000..f164dccd --- /dev/null +++ b/bridges/TheDriveBridge.php @@ -0,0 +1,45 @@ +collectExpandableDatas('https://www.thedrive.com/feed', 20); + } + + protected function parseItem($feedItem) + { + $item = parent::parseItem($feedItem); + + //remove warzone articles + if (str_contains($item['uri'], 'the-war-zone')) { + return null; + } + + //the first image in the article is an attachment for some reason + foreach ($item['enclosures'] as $attachment) { + $item['content'] = '' . $item['content']; + } + $item['enclosures'] = []; + + //make youtube videos clickable + $html = str_get_html($item['content']); + + foreach ($html->find('div.lazied-youtube-frame') as $youtubeVideoDiv) { + $videoID = $youtubeVideoDiv->getAttribute('data-video-id'); + + //place around the
    + $youtubeVideoDiv->outertext = '' . $youtubeVideoDiv->outertext . ''; + } + + $item['content'] = $html; + + return $item; + } +} From 51cdb66f9c65c41fd1c5b0564001ec5724be1575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tu=C4=9Fhan=20Belbek?= Date: Thu, 17 Oct 2024 14:17:48 +0200 Subject: [PATCH 218/423] [HarvardBusinessReviewBridge] Add bridge (#4293) --- bridges/HarvardBusinessReviewBridge.php | 88 +++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 bridges/HarvardBusinessReviewBridge.php diff --git a/bridges/HarvardBusinessReviewBridge.php b/bridges/HarvardBusinessReviewBridge.php new file mode 100644 index 00000000..cd99a1ba --- /dev/null +++ b/bridges/HarvardBusinessReviewBridge.php @@ -0,0 +1,88 @@ + [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => true, + 'title' => 'Maximum number of items to return', + 'defaultValue' => 6, //More requires clicking button "Load more" + ], + ]]; + + public function collectData() + { + $url = self::URI . '/the-latest'; + $html = getSimpleHTMLDOM($url); + + foreach ($html->find('li.stream-entry') as $data) { + // Skip if $data is null + if ($data === null) { + continue; + } + + try { + // Skip entries containing the text 'stream-ad-container' + if ($data->innertext !== null && strpos($data->innertext, 'stream-ad-container') !== false) { + continue; + } + + // Skip entries with class 'sponsored' + if ($data->hasClass('sponsored')) { + continue; + } + + $item = []; + $linkElement = $data->find('a', 0); + $titleElement = $data->find('h3.hed a', 0); + $authorElement = $data->find('ul.byline-list li', 0); + $timestampElement = $data->find('li.pubdate time', 0); + $contentElement = $data->find('div.dek', 0); + + if ($linkElement) { + $item['uri'] = self::URI . $linkElement->getAttribute('href'); + } else { + continue; // Skip this entry if no link is found + } + if ($titleElement) { + $item['title'] = trim($titleElement->plaintext); + } else { + continue; // Skip this entry if no title is found + } + if ($authorElement) { + $item['author'] = trim($authorElement->plaintext); + } else { + $item['author'] = 'Unknown'; // Default value if author is missing + } + if ($timestampElement) { + $item['timestamp'] = strtotime($timestampElement->plaintext); + } else { + $item['timestamp'] = time(); // Default to current time if timestamp is missing + } + if ($contentElement) { + $item['content'] = trim($contentElement->plaintext); + } else { + $item['content'] = ''; // Default to empty string if content is missing + } + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + + if (count($this->items) >= $this->getInput('postcount')) { + break; + } + } catch (Exception $e) { + // Log the error if necessary + continue; // Skip to the next iteration on error + } + } + } +} \ No newline at end of file From f9a51b676896126ea70ee73ff1027cbd43531d8f Mon Sep 17 00:00:00 2001 From: Jonas Taedcke <1809673+jonastaedcke@users.noreply.github.com> Date: Fri, 18 Oct 2024 08:29:07 +0200 Subject: [PATCH 219/423] [AppleMusicBridge] Further data request to receive artist information. (#4271) --- bridges/AppleMusicBridge.php | 101 +++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 23 deletions(-) diff --git a/bridges/AppleMusicBridge.php b/bridges/AppleMusicBridge.php index 900a7009..fb1164ec 100644 --- a/bridges/AppleMusicBridge.php +++ b/bridges/AppleMusicBridge.php @@ -18,9 +18,42 @@ class AppleMusicBridge extends BridgeAbstract 'required' => true, ], ]]; - const CACHE_TIMEOUT = 21600; // 6 hours + const CACHE_TIMEOUT = 60 * 60 * 6; // 6 hours + + private $title; public function collectData() + { + $items = $this->getJson(); + $artist = $this->getArtist($items); + + $this->title = $artist->artistName; + + foreach ($items as $item) { + if ($item->wrapperType === 'collection') { + $copyright = $item->copyright ?? ''; + $artworkUrl500 = str_replace('/100x100', '/500x500', $item->artworkUrl100); + $artworkUrl2000 = str_replace('/100x100', '/2000x2000', $item->artworkUrl100); + $escapedCollectionName = htmlspecialchars($item->collectionName); + + $this->items[] = [ + 'title' => $item->collectionName, + 'uri' => $item->collectionViewUrl, + 'timestamp' => $item->releaseDate, + 'enclosures' => $artworkUrl500, + 'author' => $item->artistName, + 'content' => "
    + artworkUrl60 60w, $item->artworkUrl100 100w, $artworkUrl500 500w, $artworkUrl2000 2000w\" sizes=\"100%\" src=\"$artworkUrl2000\" alt=\"Cover of $escapedCollectionName\" style=\"display: block; margin: 0 auto;\" /> +
    + from artistLinkUrl\">$item->artistName
    $copyright +
    +
    ", + ]; + } + } + } + + private function getJson() { # Limit the amount of releases to 50 if ($this->getInput('limit') > 50) { @@ -29,31 +62,53 @@ class AppleMusicBridge extends BridgeAbstract $limit = $this->getInput('limit'); } - $url = 'https://itunes.apple.com/lookup?id=' - . $this->getInput('artist') - . '&entity=album&limit=' - . $limit . - '&sort=recent'; + $url = 'https://itunes.apple.com/lookup?id=' . $this->getInput('artist') . '&entity=album&limit=' . $limit . '&sort=recent'; $html = getSimpleHTMLDOM($url); - $json = json_decode($html); + $result = $json->results; - foreach ($json->results as $obj) { - if ($obj->wrapperType === 'collection') { - $copyright = $obj->copyright ?? ''; - - $this->items[] = [ - 'title' => $obj->artistName . ' - ' . $obj->collectionName, - 'uri' => $obj->collectionViewUrl, - 'timestamp' => $obj->releaseDate, - 'enclosures' => $obj->artworkUrl100, - 'content' => '

    ' - . $obj->artistName . ' - ' . $obj->collectionName - . '
    ' - . $copyright, - ]; - } + if (!is_array($result) || count($result) == 0) { + returnServerError('There is no artist with id "' . $this->getInput('artist') . '".'); } + + return $result; + } + + private function getArtist($json) + { + $nameArray = array_filter($json, function ($obj) { + return $obj->wrapperType == 'artist'; + }); + + if (count($nameArray) === 1) { + return $nameArray[0]; + } + + return parent::getName(); + } + + public function getName() + { + if (isset($this->title)) { + return $this->title; + } + + return parent::getName(); + } + + public function getIcon() + { + if (empty($this->getInput('artist'))) { + return parent::getIcon(); + } + + // it isn't necessary to set the correct artist name into the url + $url = 'https://music.apple.com/us/artist/jon-bellion/' . $this->getInput('artist'); + $html = getSimpleHTMLDOMCached($url); + $image = $html->find('meta[property="og:image"]', 0)->content; + + $imageUpdatedSize = preg_replace('/\/\d*x\d*cw/i', '/144x144-999', $image); + + return $imageUpdatedSize; } } From b9eb3c887af0a82d71cead3dcdab2a86659260a7 Mon Sep 17 00:00:00 2001 From: somini Date: Fri, 18 Oct 2024 07:31:08 +0100 Subject: [PATCH 220/423] [PCGWNewsBridge] Remove bridge (#4305) Fix #4291 --- bridges/PCGWNewsBridge.php | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 bridges/PCGWNewsBridge.php diff --git a/bridges/PCGWNewsBridge.php b/bridges/PCGWNewsBridge.php deleted file mode 100644 index 4b3a7c76..00000000 --- a/bridges/PCGWNewsBridge.php +++ /dev/null @@ -1,38 +0,0 @@ -getURI()); - - $now = strtotime('now'); - - foreach ($html->find('.mw-parser-output .news_li') as $element) { - $item = []; - - $date_string = $element->find('b', 0)->innertext; - $date = strtotime($date_string); - if ($date > $now) { - $date = strtotime($date_string . ' - 1 year'); - } - $item['title'] = self::NAME . ' for ' . date('Y-m-d', $date); - $item['content'] = $element; - $item['uri'] = $this->getURI(); - $item['timestamp'] = $date; - - $this->items[] = $item; - } - } -} From 668f3a9d7e244e7969d3dd5fdac3f87c6fc88a66 Mon Sep 17 00:00:00 2001 From: Arnav Jain Date: Sat, 19 Oct 2024 20:19:24 +0200 Subject: [PATCH 221/423] [AppleMusicBridge] fix linting error (#4308) --- bridges/AppleMusicBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/AppleMusicBridge.php b/bridges/AppleMusicBridge.php index fb1164ec..b633c69f 100644 --- a/bridges/AppleMusicBridge.php +++ b/bridges/AppleMusicBridge.php @@ -43,7 +43,10 @@ class AppleMusicBridge extends BridgeAbstract 'enclosures' => $artworkUrl500, 'author' => $item->artistName, 'content' => "
    - artworkUrl60 60w, $item->artworkUrl100 100w, $artworkUrl500 500w, $artworkUrl2000 2000w\" sizes=\"100%\" src=\"$artworkUrl2000\" alt=\"Cover of $escapedCollectionName\" style=\"display: block; margin: 0 auto;\" /> + artworkUrl60 60w, $item->artworkUrl100 100w, $artworkUrl500 500w, $artworkUrl2000 2000w\" + sizes=\"100%\" src=\"$artworkUrl2000\" + alt=\"Cover of $escapedCollectionName\" + style=\"display: block; margin: 0 auto;\" />
    from artistLinkUrl\">$item->artistName
    $copyright
    From 6c88f2c21e10cdca5ab296d69ba141b263fe9704 Mon Sep 17 00:00:00 2001 From: User123698745 Date: Sun, 20 Oct 2024 00:18:52 +0200 Subject: [PATCH 222/423] [prtester] fix prtester no longer supporting multiple bridges being changed, because the filenames are not unique (#4310) --- .github/.gitignore | 1 + .github/prtester.py | 36 ++++++++++++++++++++++++++---------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/.gitignore b/.github/.gitignore index 6310b3dd..7ebb4030 100644 --- a/.github/.gitignore +++ b/.github/.gitignore @@ -4,3 +4,4 @@ # Generated files comment*.md comment*.txt +*.html diff --git a/.github/prtester.py b/.github/prtester.py index 53d77725..c5c5be22 100644 --- a/.github/prtester.py +++ b/.github/prtester.py @@ -5,6 +5,7 @@ from bs4 import BeautifulSoup from datetime import datetime from typing import Iterable import os +import glob import urllib # This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge @@ -14,18 +15,33 @@ import urllib # It also add a tag with the url of em's public instance, so viewing # the HTML file locally will actually work as designed. +ARTIFACT_FILE_EXTENSION = '.html' + class Instance: name = '' url = '' def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: bool, title: str, output_file: str): start_date = datetime.now() + + prid = os.getenv('PR') + artifact_base_url = f'https://rss-bridge.github.io/rss-bridge-tests/prs/{prid}' + artifact_directory = os.getcwd() + for file in glob.glob(f'*{ARTIFACT_FILE_EXTENSION}', root_dir=artifact_directory): + os.remove(file) + table_rows = [] for instance in instances: page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page - table_rows += testBridges(instance, bridge_cards, with_upload, with_reduced_upload) # run the main scraping code with the list of bridges + table_rows += testBridges( + instance=instance, + bridge_cards=bridge_cards, + with_upload=with_upload, + with_reduced_upload=with_reduced_upload, + artifact_directory=artifact_directory, + artifact_base_url=artifact_base_url) # run the main scraping code with the list of bridges with open(file=output_file, mode='w+', encoding='utf-8') as file: table_rows_value = '\n'.join(sorted(table_rows)) file.write(f''' @@ -37,10 +53,8 @@ def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: *last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}* '''.strip()) -def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool) -> Iterable: +def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool, artifact_directory: str, artifact_base_url: str) -> Iterable: instance_suffix = '' - prid = os.getenv("PR") - tester_url = f'https://rss-bridge.github.io/rss-bridge-tests/prs/{prid}' if instance.name: instance_suffix = f' ({instance.name})' table_rows = [] @@ -105,13 +119,13 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w selectionvalue = selectionentry.get('value') break context_parameters[listname] = selectionvalue - termpad_url = 'about:blank' + artifact_url = 'about:blank' if error_messages: status = '
    '.join(map(lambda m: f'❌ `{m}`', error_messages)) else: # if all example/default values are present, form the full request url, run the request, add a tag with # the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and - # then upload it to termpad.com, a pastebin-like-site. + # then save it to a html file. context_parameters.update({ 'action': 'display', 'bridge': bridgeid, @@ -142,16 +156,18 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, w if status_is_ok: status = '✔️' if with_upload and (not with_reduced_upload or not status_is_ok): - filename = f'{os.getcwd()}/{instance.name}_{form_number}.html' - with open(file=filename, mode='wb') as file: + filename = f'{bridge_name} {form_number}{instance_suffix}{ARTIFACT_FILE_EXTENSION}' + filename = re.sub(r'[^a-z0-9 \_\-\.]', '', filename, flags=re.I).replace(' ', '_') + with open(file=f'{artifact_directory}/{filename}', mode='wb') as file: file.write(page_text) - table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({tester_url}/{instance.name}_{form_number}.html) | {status} |') + artifact_url = f'{artifact_base_url}/{filename}' + table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({artifact_url}) | {status} |') form_number += 1 return table_rows def getFirstLine(value: str) -> str: # trim whitespace and remove text that can break the table or is simply unnecessary - clean_value = re.sub('^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip()) + clean_value = re.sub(r'^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip()) first_line = next(iter(clean_value.splitlines()), '') max_length = 250 if (len(first_line) > max_length): From c3dc46a307b73b30dc1ded7301209d8b74d4b6c1 Mon Sep 17 00:00:00 2001 From: Matthieu Rakotojaona Date: Sun, 20 Oct 2024 17:16:29 +0000 Subject: [PATCH 223/423] [prtester] Update python dependency (#4311) This is necessary for glob.glob() with the root_dir argument --- .github/workflows/prhtmlgenerator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prhtmlgenerator.yml b/.github/workflows/prhtmlgenerator.yml index c0105e29..163d51e3 100644 --- a/.github/workflows/prhtmlgenerator.yml +++ b/.github/workflows/prhtmlgenerator.yml @@ -50,7 +50,7 @@ jobs: - name: Setup python uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.13' cache: 'pip' - name: Install requirements run: | From bc536f392877623ce48140a72a0025415458061d Mon Sep 17 00:00:00 2001 From: Arnav Jain Date: Sun, 3 Nov 2024 18:20:48 +0100 Subject: [PATCH 224/423] =?UTF-8?q?[D=C3=A4cksnackBridge]=20New=20Bridge?= =?UTF-8?q?=20(#4309)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [DäcksnackBridge] new bridge * [DäcksnackBridge] move preamble before figure --- bridges/DacksnackBridge.php | 104 ++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 bridges/DacksnackBridge.php diff --git a/bridges/DacksnackBridge.php b/bridges/DacksnackBridge.php new file mode 100644 index 00000000..7aab48d1 --- /dev/null +++ b/bridges/DacksnackBridge.php @@ -0,0 +1,104 @@ + '01', + 'februari' => '02', + 'mars' => '03', + 'april' => '04', + 'maj' => '05', + 'juni' => '06', + 'juli' => '07', + 'augusti' => '08', + 'september' => '09', + 'oktober' => '10', + 'november' => '11', + 'december' => '12' + ]; + + // Split the date string into parts + list($day, $monthName, $year) = explode(' ', $dateString); + + // Convert month name to month number + $month = $monthNames[$monthName]; + + // Format to a string recognizable by DateTime + $formattedDate = sprintf('%04d-%02d-%02d', $year, $month, $day); + + // Create a DateTime object + $dateValue = new DateTime($formattedDate); + + if ($dateValue) { + $dateValue->setTime(0, 0); // Set time to 00:00 + return $dateValue->getTimestamp(); + } + + return $dateValue ? $dateValue->getTimestamp() : false; + } + + public function collectData() + { + $NEWSURL = self::URI; + $html = getSimpleHTMLDOMCached($NEWSURL, 18000) or + returnServerError('Could not request: ' . $NEWSURL); + + foreach ($html->find('a.main-news-item') as $element) { + // Debug::log($element); + + $title = trim($element->find('h2', 0)->plaintext); + $category = trim($element->find('.category-tag', 0)->plaintext); + $url = self::URI . $element->getAttribute('href'); + $published = $this->parseSwedishDates(trim($element->find('.published', 0)->plaintext)); + + $article_html = getSimpleHTMLDOMCached($url, 18000) or + returnServerError('Could not request: ' . $url); + $article_content = $article_html->find('#ctl00_ContentPlaceHolder1_NewsArticleVeiw_pnlArticle', 0); + + $figure = self::URI . $article_content->find('img.news-image', 0)->getAttribute('src'); + $figure_caption = $article_content->find('.image-description', 0)->plaintext; + $author = $article_content->find('span.main-article-author', 0)->plaintext; + $preamble = $article_content->find('h4.main-article-ingress', 0)->plaintext; + + $article_text = ''; + foreach ($article_content->find('div') as $div) { + if (!$div->hasAttribute('class')) { + $article_text = $div; + } + } + + // Use a regular expression to extract the name + if (preg_match('/Text:\s*(.*?)\s*Foto:/', $author, $matches)) { + $author = $matches[1]; // This will contain 'Jonna Jansson' + } + + $content = ' [' . $category . '] ' . $preamble . '

    '; + $content .= '
    '; + $content .= ''; + $content .= '
    ' . $figure_caption . '
    '; + $content .= '
    '; + $content .= $article_text; + + $this->items[] = [ + 'uri' => $url, + 'title' => $title, + 'author' => $author, + 'timestamp' => $published, + 'content' => trim($content), + ]; + } + } +} From 082542dabcfd0ac1b68923a438c218023353d4f9 Mon Sep 17 00:00:00 2001 From: Arnav Jain Date: Sun, 3 Nov 2024 18:22:44 +0100 Subject: [PATCH 225/423] [TestFaktaBridge] new bridge (#4307) * [TestFaktaBridge] new bridge * [TestFaktaBridge] fix linting errors --- bridges/TestFaktaBridge.php | 100 ++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 bridges/TestFaktaBridge.php diff --git a/bridges/TestFaktaBridge.php b/bridges/TestFaktaBridge.php new file mode 100644 index 00000000..b9a65138 --- /dev/null +++ b/bridges/TestFaktaBridge.php @@ -0,0 +1,100 @@ + 'Jan', + 'Feb' => 'Feb', + 'Mar' => 'Mar', + 'Apr' => 'Apr', + 'Maj' => 'May', + 'Jun' => 'Jun', + 'Jul' => 'Jul', + 'Aug' => 'Aug', + 'Sep' => 'Sep', + 'Okt' => 'Oct', + 'Nov' => 'Nov', + 'Dec' => 'Dec' + ]; + + // Replace Swedish month names with English + $dateString = preg_replace_callback( + '/\b(' . implode('|', array_keys($months)) . ')\b/', + function ($matches) use ($months) { + return $months[$matches[0]]; + }, + $dateString + ); + + // Create DateTime object + $dateValue = DateTime::createFromFormat( + 'd M, Y', + trim($dateString), + new DateTimeZone('Europe/Stockholm') + ); + if ($dateValue) { + $dateValue->setTime(0, 0); // Set time to 00:00 + return $dateValue->getTimestamp(); + } + + return $dateValue ? $dateValue->getTimestamp() : false; + } + + public function collectData() + { + $NEWSURL = self::URI . '/sv'; + $html = getSimpleHTMLDOMCached($NEWSURL, 18000) or + returnServerError('Could not request: ' . $NEWSURL); + + foreach ($html->find('.row-container') as $element) { + // Debug::log($element); + + $title = $element->find('h2', 0)->plaintext; + $category = trim($element->find('.red-label', 0)->plaintext); + $url = self::URI . $element->find('a', 0)->getAttribute('href'); + $figure = $element->find('img', 0); + $preamble = trim($element->find('.text', 0)->plaintext); + + $article_html = getSimpleHTMLDOMCached($url, 18000) or + returnServerError('Could not request: ' . $url); + $article_content = $article_html->find('div.content', 0); + $article_text = $article_html->find('article', 0); + + $requestor = $article_html->find('div.uppdrag', 0)->plaintext; + $author = trim($article_html->find('span.name', 0)->plaintext); + $published = $this->parseSwedishDates( + str_replace( + 'Publicerad: ', + '', + trim($article_html->find('span.created', 0)->plaintext) + ) + ); + + $content = $figure . '
    '; + $content .= '' . strtoupper($category) . ' ' . $requestor . '

    '; + $content .= '' . $preamble . '

    '; + $content .= $article_text; + + $this->items[] = [ + 'uri' => $url, + 'title' => $title, + 'author' => $author, + 'timestamp' => $published, + 'content' => trim($content), + ]; + } + } +} From 29d984cbe78bddd31fa97f810f1eebd0a46cbcb7 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 3 Nov 2024 18:25:51 +0100 Subject: [PATCH 226/423] [TagesspiegelBridge] Add bridge for tagesspiegel.de (#4270) * [TagesspiegelBridge] Add bridge for tagesspiegel.de * [TagesspiegelBridge] Raise timtout to 60min --- bridges/TagesspiegelBridge.php | 221 +++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 bridges/TagesspiegelBridge.php diff --git a/bridges/TagesspiegelBridge.php b/bridges/TagesspiegelBridge.php new file mode 100644 index 00000000..b3a3fc64 --- /dev/null +++ b/bridges/TagesspiegelBridge.php @@ -0,0 +1,221 @@ + [ + 'name' => 'Category', + 'type' => 'list', + 'values' => [ + 'Startseite' + => 'https://tagesspiegel.de/contentexport/feed', + 'Plus' + => 'https://tagesspiegel.de/contentexport/feed/plus/', + 'Politik' + => 'https://tagesspiegel.de/contentexport/feed/politik/', + 'Internationales' + => 'https://tagesspiegel.de/contentexport/feed/internationales/', + 'Berlin' + => 'https://tagesspiegel.de/contentexport/feed/berlin/', + 'Berlin - Bezirke' + => 'https://tagesspiegel.de/contentexport/feed/berlin/bezirke/', + 'Berlin - Berliner Wirtschaft' + => 'https://tagesspiegel.de/contentexport/feed/berlin/berliner-wirtschaft/', + 'Berlin - Berliner Sport' + => 'https://tagesspiegel.de/contentexport/feed/berlin/berliner_sport/', + 'Berlin - Polizei & Justiz' + => 'https://tagesspiegel.de/contentexport/feed/berlin/polizei-justiz/', + 'Berlin - Stadtleben' + => 'https://tagesspiegel.de/contentexport/feed/berlin/stadtleben/', + 'Berlin - Schule' + => 'https://tagesspiegel.de/contentexport/feed/berlin/schule/', + 'Gesellschaft' + => 'https://tagesspiegel.de/contentexport/feed/gesellschaft/', + 'Gesellschaft - Liebe & Partnerschaft' + => 'https://tagesspiegel.de/contentexport/feed/gesellschaft/liebe-partnerschaft/', + 'Gesellschaft - Queer' + => 'https://tagesspiegel.de/contentexport/feed/gesellschaft/queerspiegel/', + 'Gesellschaft - Panorama' + => 'https://tagesspiegel.de/contentexport/feed/gesellschaft/panorama/', + 'Gesellschaft - Medien' + => 'https://tagesspiegel.de/contentexport/feed/gesellschaft/medien/', + 'Gesellschaft - Geschichte' + => 'https://tagesspiegel.de/contentexport/feed/gesellschaft/geschichte/', + 'Gesellschaft - Reise' + => 'https://tagesspiegel.de/contentexport/feed/gesellschaft/reise/', + 'Wirtschaft' + => 'https://tagesspiegel.de/contentexport/feed/wirtschaft/', + 'Wirtschaft - Immobilien' + => 'https://tagesspiegel.de/contentexport/feed/wirtschaft/immobilien/', + 'Wirtschaft - Jobs & Karriere' + => 'https://tagesspiegel.de/contentexport/feed/wirtschaft/karriere/', + 'Wirtschaft - Finanzen' + => 'https://tagesspiegel.de/contentexport/feed/wirtschaft/finanzen/', + 'Wirtschaft - Mobilität' + => 'https://tagesspiegel.de/contentexport/feed/wirtschaft/mobilitaet/', + 'Kultur' + => 'https://tagesspiegel.de/contentexport/feed/kultur/', + 'Kultur - Literatur' + => 'https://tagesspiegel.de/contentexport/feed/kultur/literatur/', + 'Kultur - Comics' + => 'https://tagesspiegel.de/contentexport/feed/kultur/comics/', + 'Kultur - Kino' + => 'https://tagesspiegel.de/contentexport/feed/kultur/kino/', + 'Kultur - Pop' + => 'https://tagesspiegel.de/contentexport/feed/kultur/pop/', + 'Kultur - Ausstellungen' + => 'https://tagesspiegel.de/contentexport/feed/kultur/ausstellungen/', + 'Kultur - Bühne' + => 'https://tagesspiegel.de/contentexport/feed/kultur/buehne/', + 'Wissen' + => 'https://tagesspiegel.de/contentexport/feed/wissen/', + 'Gesundheit' + => 'https://tagesspiegel.de/contentexport/feed/gesundheit/', + 'Sport' + => 'https://tagesspiegel.de/contentexport/feed/sport/', + 'Meinung' + => 'https://tagesspiegel.de/contentexport/feed/meinung/', + 'Meinung - Kolumnen' + => 'https://tagesspiegel.de/contentexport/feed/meinung/kolumnen/', + 'Meinung - Lesermeinung' + => 'https://tagesspiegel.de/contentexport/feed/meinung/lesermeinung/', + 'Potsdam' + => 'https://tagesspiegel.de/contentexport/feed/potsdam/', + 'Potsdam - Landeshauptstadt' + => 'https://tagesspiegel.de/contentexport/feed/potsdam/landeshauptstadt/', + 'Potsdam - Potsdam-Mittelmark' + => 'https://tagesspiegel.de/contentexport/feed/potsdam/potsdam-mittelmark/', + 'Potsdam - Brandenburg' + => 'https://tagesspiegel.de/contentexport/feed/potsdam/brandenburg/', + 'Potsdam - Kultur' + => 'https://tagesspiegel.de/contentexport/feed/potsdam/potsdam-kultur/', + 'Podcasts' + => 'https://tagesspiegel.de/contentexport/feed/podcasts/', + ] + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify number of full articles to return', + 'defaultValue' => 5 + ] + ]]; + + public function collectData() + { + $url = $this->getInput('category'); + $limit = $this->getInput('limit') ?: 5; + + $this->collectExpandableDatas($url, $limit); + } + + protected function parseItem(array $item) + { + $item['enclosures'] = []; + + $article = getSimpleHTMLDOM($item['uri']); + $item = $this->parseArticle($item, $article); + + return $item; + } + + private function parseArticle($item, $article) + { + $item['categories'] = []; + + // Add tag for articles only available with "Tagesspiegel Plus" + $plusicon = $article->find('span[data-ob="plus"]', 0); + if ($plusicon) { + $item['categories'][] = 'Tagesspiegel Plus'; + } + + // Add section from breadcrumbs as tags + $breadcrumbs = $article->find('ol[property="breadcrumb"]', 0); + $names = $breadcrumbs->find('span[property="name"]'); + $names = array_slice($names, 1, -1); + foreach ($names as $name) { + $item['categories'][] = trim($name->plaintext); + } + + // Get categories from article + $home_link = $article->find('a[data-gtm-class="article-home-link"]', 0); + if ($home_link) { + $tag_container = $home_link->parent->nextSibling(); + if ($tag_container) { + $tags = $tag_container->find('li'); + + if ($tags) { + foreach ($tags as $tag) { + $item['categories'][] = trim($tag->plaintext); + } + } + } + } + + $article = $article->find('article', 0); + + // Remove known bad elements + foreach ( + $article->find( + 'script, aside, nav, dl.debug-piano, .link--external svg, time, a[data-gtm-class="article-home-link"]' + ) as $bad + ) { + $bad->remove(); + } + + // Remove references to external content (requires javascript for consent) + foreach ($article->find('p') as $par) { + if ($par->plaintext == 'Empfohlener redaktioneller Inhalt') { + $par->parent->parent->parent->parent->remove(); + } + } + + // Reload html, as remove() is buggy + $article = str_get_html($article->outertext); + + + // Clean article content + $elements = $article->find('h3, p, figure, blockquote'); + foreach ($elements as $i => $element) { + foreach ($element->find('img, picture source') as $img) { + // Add URI to src + if ($img->hasAttribute('src')) { + if (str_starts_with($img->attr['src'], '/')) { + $img->attr['src'] = urljoin(self::URI, $img->attr['src']); + } + } + + // Add URI to srcset + if ($img->hasAttribute('srcset')) { + $srcsets = explode(',', $img->attr['srcset']); + foreach ($srcsets as &$srcset) { + $parts = explode(' ', trim($srcset)); + if (count($parts) > 0) { + if (str_starts_with($parts[0], '/')) { + $parts[0] = urljoin(self::URI, $parts[0]); + } + } + $srcset = implode(' ', $parts); + } + $img->attr['srcset'] = implode(', ', $srcsets); + } + } + + // Remove paragraphs that are already included in other elements + if ($element->tag == 'p') { + if ($element->parent->tag == 'blockquote' || $element->parent->tag == 'figure') { + unset($elements[$i]); + } + } + } + $item['content'] = implode('', $elements); + + return $item; + } +} From bd0fb1da9901d0c91b1a68a8a50046a434440caf Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Sun, 3 Nov 2024 18:28:32 +0100 Subject: [PATCH 227/423] [IdealoBridge] Fix (#4316) When a product was available before as used product in the past, and now it's not available used anymore, a price update article was generated on every feed loading, because the old used price was still stored in the cache, and therefore different as "no price". The issue was also present in the cas of a New product price that becomes unavailable. Now, when either there is no New or Used price available, the previous price is delete from the cache. --- bridges/IdealoBridge.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php index f426a45c..7432f78a 100644 --- a/bridges/IdealoBridge.php +++ b/bridges/IdealoBridge.php @@ -152,14 +152,21 @@ class IdealoBridge extends BridgeAbstract $PriceNew = $FirstButton->find('strong', 0)->plaintext; // Save current price $this->saveCacheValue($KeyNEW, $PriceNew); + } else if ($FirstButton === null) { + // In case there is no actual New Price delete the previous value in the cache + $this->cache->delete($this->getShortName() . '_' . $KeyNEW); } + // Second Button contains the used product price $SecondButton = $html->find('.oopStage-conditionButton-wrapper-text', 1); if ($SecondButton) { $PriceUsed = $SecondButton->find('strong', 0)->plaintext; // Save current price $this->saveCacheValue($KeyUSED, $PriceUsed); + } else if ($SecondButton === null) { + // In case there is no actual Used Price delete the previous value in the cache + $this->cache->delete($this->getShortName() . '_' . $KeyUSED); } // Only continue if a price has changed From 8d6d0fa10c64c2ae6617d72793f45f7865c7c010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tu=C4=9Fhan=20Belbek?= Date: Sun, 3 Nov 2024 20:30:28 +0300 Subject: [PATCH 228/423] [DuvarOrgBridge] Add Duvar.org bridge for scraping news articles (#4315) * Add Duvar.org bridge for scraping news articles * PR Fixes * Update DuvarOrgBridge.php to set a default value for the URL suffix --------- Co-authored-by: Tughan Belbek --- bridges/DuvarOrgBridge.php | 86 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 bridges/DuvarOrgBridge.php diff --git a/bridges/DuvarOrgBridge.php b/bridges/DuvarOrgBridge.php new file mode 100644 index 00000000..f5f01063 --- /dev/null +++ b/bridges/DuvarOrgBridge.php @@ -0,0 +1,86 @@ + [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => true, + 'title' => 'Maximum number of items to return', + 'defaultValue' => 20, + ], + 'urlsuffix' => [ + 'name' => 'URL Suffix', + 'type' => 'list', + 'title' => 'Suffix for the URL to scrape a specific section', + 'defaultValue' => 'Main', + 'values' => [ + 'Main' => '', + 'Balanced' => '/uyumlu', + 'Protest' => '/muhalif', + 'Center' => '/merkez', + 'Alternative' => '/alternatif', + 'Global' => '/global', + ], + ], + ]]; + + public function collectData() + { + $postCount = $this->getInput('postcount'); + $urlSuffix = $this->getInput('urlsuffix'); + $url = self::URI . $urlSuffix; + $html = getSimpleHTMLDOM($url); + + foreach ($html->find('article.news-item') as $data) { + if ($data === null) { + continue; + } + + try { + $item = []; + $linkElement = $data->find('h2.news-title a', 0); + $titleElement = $data->find('h2.news-title a', 0); + $timestampElement = $data->find('time.meta-tag.date-tag', 0); + $contentElement = $data->find('div.news-description', 0); + + if ($linkElement) { + $item['uri'] = $linkElement->getAttribute('href'); + } else { + continue; + } + if ($titleElement) { + $item['title'] = trim($titleElement->plaintext); + } else { + continue; + } + if ($timestampElement) { + $item['timestamp'] = strtotime($timestampElement->plaintext); + } else { + $item['timestamp'] = time(); + } + if ($contentElement) { + $item['content'] = trim($contentElement->plaintext); + } else { + $item['content'] = ''; + } + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + + if (count($this->items) >= $postCount) { + break; + } + } catch (Exception $e) { + continue; + } + } + } +} \ No newline at end of file From 1cd5b072f37a6cab9b183f732338e623eeadb612 Mon Sep 17 00:00:00 2001 From: Rose Liverman Date: Sun, 3 Nov 2024 10:33:05 -0700 Subject: [PATCH 229/423] Formatting fix "For Hosts" documentation (#4317) --- docs/03_For_Hosts/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/03_For_Hosts/index.md b/docs/03_For_Hosts/index.md index a4e55d69..5ecbd4b7 100644 --- a/docs/03_For_Hosts/index.md +++ b/docs/03_For_Hosts/index.md @@ -6,7 +6,7 @@ Some bridges could be implemented more efficiently by actually using proprietary but there are reasons against it: - RSS-Bridge exists in the first place to NOT use APIs. -- See [the rant](https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md#Rant) + See [the rant](https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md#Rant). - APIs require private keys that could be stored on servers running RSS-Bridge, -- which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution. + which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution. From dd165ea9d1d2306f96188943fc7441d55f4cc1e4 Mon Sep 17 00:00:00 2001 From: Dennis Date: Mon, 4 Nov 2024 15:16:58 +0100 Subject: [PATCH 230/423] [HuntShowdownNewsBridge] Fetches the latest articles from Hunt Showdown (#4318) * feat: add Hunt Showdown News Bridge for fetching latest news articles * chore: clean up formatting and remove unnecessary whitespace in HuntShowdownNewsBridge.php --- bridges/HuntShowdownNewsBridge.php | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 bridges/HuntShowdownNewsBridge.php diff --git a/bridges/HuntShowdownNewsBridge.php b/bridges/HuntShowdownNewsBridge.php new file mode 100644 index 00000000..6aca88f7 --- /dev/null +++ b/bridges/HuntShowdownNewsBridge.php @@ -0,0 +1,40 @@ +find('.col'); + + // Removing first element because it's a "load more" button + array_shift($articles); + foreach ($articles as $article) { + $item = []; + + $article_title = $article->find('h3', 0)->plaintext; + $article_content = $article->find('p', 0)->plaintext; + $article_cover = $article->find('img', 0)->src; + + // If there is a cover, add it to the content + if (!empty($article_cover)) { + $article_cover = '' . $article_title . '

    '; + $article_content = $article_cover . $article_content; + } + + $item['uri'] = $article->find('a', 0)->href; + $item['title'] = $article_title; + $item['content'] = $article_content; + $item['enclosures'] = [$article_cover]; + $item['timestamp'] = $article->find('span', 0)->plaintext; + + $this->items[] = $item; + } + } +} \ No newline at end of file From 6c86e2c1f73897180d56d0a0d10e37e8eb63dbb9 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 8 Nov 2024 08:11:18 +0100 Subject: [PATCH 231/423] [IdealoBridge] Really fix Logic and enhance Feed Content (#4321) - Fix Feed Title generation (wrong usage of loadCacheValue) - Use a more reliable way to get New and Used Price - If no new Price and no Used Price are present in the page, then don't delete previous New Price and previous Used Price - If there is no New Price and no Used Price, then return no Feed Item - Fix the "now" date format - Make the Feed Item Title more readable - Use the Product Link as the Feed URL --- bridges/IdealoBridge.php | 48 ++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php index 7432f78a..92bb30d0 100644 --- a/bridges/IdealoBridge.php +++ b/bridges/IdealoBridge.php @@ -49,7 +49,7 @@ class IdealoBridge extends BridgeAbstract $cacheDuration = 604800; $link = $this->getInput('Link'); $keyTITLE = $link . 'TITLE'; - $product = $this->loadCacheValue($keyTITLE, $cacheDuration); + $product = $this->loadCacheValue($keyTITLE); // The cache does not contain the title of the bridge, we must get it and save it in the cache if ($product === null) { @@ -147,30 +147,32 @@ class IdealoBridge extends BridgeAbstract $OldPriceUsed = $this->loadCacheValue($KeyUSED); // First button contains the new price. Found at oopStage-conditionButton-wrapper-text class (.) - $FirstButton = $html->find('.oopStage-conditionButton-wrapper-text', 0); - if ($FirstButton) { - $PriceNew = $FirstButton->find('strong', 0)->plaintext; + $ActualNewPrice = $html->find('div[id=oopStage-conditionButton-new]', 0); + // Second Button contains the used product price + $ActualUsedPrice = $html->find('div[id=oopStage-conditionButton-used]', 0); + + if ($ActualNewPrice) { + $PriceNew = $ActualNewPrice->find('strong', 0)->plaintext; // Save current price $this->saveCacheValue($KeyNEW, $PriceNew); - } else if ($FirstButton === null) { - // In case there is no actual New Price delete the previous value in the cache + } else if ($ActualNewPrice === null && $ActualUsedPrice !== null) { + // In case there is no actual New Price and a Ured Price exists, then delete the previous value in the cache $this->cache->delete($this->getShortName() . '_' . $KeyNEW); } // Second Button contains the used product price - $SecondButton = $html->find('.oopStage-conditionButton-wrapper-text', 1); - if ($SecondButton) { - $PriceUsed = $SecondButton->find('strong', 0)->plaintext; + if ($ActualUsedPrice) { + $PriceUsed = $ActualUsedPrice->find('strong', 0)->plaintext; // Save current price $this->saveCacheValue($KeyUSED, $PriceUsed); - } else if ($SecondButton === null) { - // In case there is no actual Used Price delete the previous value in the cache + } else if ($ActualUsedPrice === null && $ActualNewPrice !== null) { + // In case there is no actual Used Price and a New Price exists, then delete the previous value in the cache $this->cache->delete($this->getShortName() . '_' . $KeyUSED); } - // Only continue if a price has changed - if ($PriceNew != $OldPriceNew || $PriceUsed != $OldPriceUsed) { + // Only continue if a price has changed and there exists a New or Used price (sometimes no new Price _and_ Used Price are shown) + if (!($ActualNewPrice === null && $ActualUsedPrice === null ) && ($PriceNew != $OldPriceNew || $PriceUsed != $OldPriceUsed)) { // Get Product Image $image = $html->find('.datasheet-cover-image', 0)->src; @@ -198,9 +200,9 @@ class IdealoBridge extends BridgeAbstract $content .= ""; - $now = date('d.m.j H:m'); + $now = date('d/m/Y H:i'); - $Pricealarm = 'Pricealarm %s: %s %s %s'; + $Pricealarm = 'Pricealarm %s: %s %s - %s'; // Currently under Max new price if ($this->getInput('MaxPriceNew') != '') { @@ -247,7 +249,7 @@ class IdealoBridge extends BridgeAbstract $title .= 'USED' . $this->getPriceTrend($PriceUsed, $OldPriceUsed) . ' '; } $title .= $Productname; - $title .= ' '; + $title .= ' - '; $title .= $now; $item = [ @@ -275,4 +277,18 @@ class IdealoBridge extends BridgeAbstract return parent::getName(); } } + + /** + * Returns the RSS Feed URL according to the parameters + * @return string the RSS feed URL + */ + public function getURI() + { + switch ($this->queriedContext) { + case '0': + return $this->getInput('Link'); + default: + return parent::getURI(); + } + } } From 2294dac3f15c6eca47a47dad5f5c47750dec3647 Mon Sep 17 00:00:00 2001 From: July Date: Sat, 23 Nov 2024 12:47:08 -0500 Subject: [PATCH 232/423] [AO3Bridge] Add fetch limit to reduce requests (#4328) --- bridges/AO3Bridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php index 970ed414..7e18b657 100644 --- a/bridges/AO3Bridge.php +++ b/bridges/AO3Bridge.php @@ -27,6 +27,7 @@ class AO3Bridge extends BridgeAbstract 'Entire work' => 'all', ], ], + 'limit' => self::LIMIT, ], 'Bookmarks' => [ 'user' => [ @@ -84,6 +85,8 @@ class AO3Bridge extends BridgeAbstract } $this->title = $heading->plaintext; + $limit = $this->getInput('limit') ?? 3; + $count = 0; foreach ($html->find('.index.group > li') as $element) { $item = []; @@ -118,7 +121,7 @@ class AO3Bridge extends BridgeAbstract $item['uid'] = $item['uri'] . "/$strdate/$chapters"; // Fetch workskin of desired chapter(s) in list - if ($this->getInput('range')) { + if ($this->getInput('range') && ($limit == 0 || $count++ < $limit)) { $url = $item['uri']; switch ($this->getInput('range')) { case ('all'): From 9457e075f6a15c702bed94cf84b62ce949815dc9 Mon Sep 17 00:00:00 2001 From: July Date: Sat, 23 Nov 2024 12:50:40 -0500 Subject: [PATCH 233/423] [PriviblurBridge] Fix invalid favicon, use either Tumblr or blog icon (#4327) --- bridges/PriviblurBridge.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bridges/PriviblurBridge.php b/bridges/PriviblurBridge.php index 198c38bc..6b442e75 100644 --- a/bridges/PriviblurBridge.php +++ b/bridges/PriviblurBridge.php @@ -17,6 +17,7 @@ class PriviblurBridge extends BridgeAbstract ]; private $title; + private $favicon = 'https://www.tumblr.com/favicon.ico'; public function collectData() { @@ -25,6 +26,11 @@ class PriviblurBridge extends BridgeAbstract $html = defaultLinkTo($html, $url); $this->title = $html->find('head title', 0)->innertext; + if ($html->find('#blog-header img.avatar', 0)) { + $icon = $html->find('#blog-header img.avatar', 0)->src; + $this->favicon = str_replace('pnj', 'png', $icon); + } + $elements = $html->find('.post'); foreach ($elements as $element) { $item = []; @@ -64,6 +70,11 @@ class PriviblurBridge extends BridgeAbstract public function getURI() { - return $this->getInput('url') ? $this->getInput('url') : parent::getURI(); + return $this->getInput('url') ?? parent::getURI(); + } + + public function getIcon() + { + return $this->favicon; } } From a6e8760726c468d48ea45a3bb94577b7b94c08a8 Mon Sep 17 00:00:00 2001 From: Sebastian Wolf <117176763+swofl@users.noreply.github.com> Date: Sat, 23 Nov 2024 18:54:21 +0100 Subject: [PATCH 234/423] [FragDenStaatBridge] add new bridge (#4330) --- bridges/FragDenStaatBridge.php | 78 ++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 bridges/FragDenStaatBridge.php diff --git a/bridges/FragDenStaatBridge.php b/bridges/FragDenStaatBridge.php new file mode 100644 index 00000000..aee1885c --- /dev/null +++ b/bridges/FragDenStaatBridge.php @@ -0,0 +1,78 @@ + [ + 'name' => 'Query Limit', + 'title' => 'Amount of articles to query', + 'type' => 'number', + 'defaultValue' => 5, + ], + ] ]; + + protected function parseTeaser($teaser) + { + $result = []; + + $header = $teaser->find('h3 > a', 0); + $result['title'] = $header->plaintext; + $result['uri'] = static::URI . $header->href; + $result['enclosures'] = []; + $result['enclosures'][] = $teaser->find('img', 0)->src; + $result['uid'] = hash('sha256', $result['title']); + $result['timestamp'] = strtotime($teaser->find('time', 0)->getAttribute('datetime')); + + return $result; + } + + public function collectData() + { + $html = getSimpleHTMLDOM(self::URI . '/artikel/exklusiv/'); + + $queryLimit = (int) $this->getInput('qLimit'); + if ($queryLimit > 12) { + $queryLimit = 12; + } + + $teasers = []; + + $teaserElements = $html->find('article'); + + for ($i = 0; $i < $queryLimit; $i++) { + array_push($teasers, $this->parseTeaser($teaserElements[$i])); + } + + foreach ($teasers as $article) { + $articleHtml = getSimpleHTMLDOMCached($article['uri'], static::CACHE_TIMEOUT * 6); + $articleCore = $articleHtml->find('article.blog-article', 0); + + $content = ''; + + $lead = $articleCore->find('div.lead > p', 0)->innertext; + + $content .= '

    ' . $lead . '

    '; + + foreach ($articleCore->find('div.blog-content > p, div.blog-content > h3') as $paragraph) { + $content .= $paragraph->outertext; + } + + $article['content'] = '' . $content; + + $article['author'] = ''; + + foreach ($articleCore->find('a[rel="author"]') as $author) { + $article['author'] .= $author->innertext . ', '; + } + + $article['author'] = rtrim($article['author'], ', '); + + $this->items[] = $article; + } + } +} From 2ee615e5882d5f68c500d863a675cfeb82320366 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Sat, 23 Nov 2024 19:11:36 +0100 Subject: [PATCH 235/423] [PepperBridgeAbstract, DealabsBridge, HotUKDealsBridge, MydealsBridge] Streamlining Group Management (#4336) * [PepperBridgeAbstract, DealabsBridge, HotUKDealsBridge, MydealsBridge] Streamlining Group Management Since groups can change URLs, be created, or removed at the discretion of website administrators, maintaining a valid and functional list of groups is impractical. Users can now enter the part of the URL that defines the group in a text field, rather than searching through a lengthy, likely outdated list. The way the RSS feed title is retrieved had to be adjusted accordingly. Titles are now cached for 15 days to avoid unnecessary website access and to prevent potential bot blocking. Existing feeds will continue to work, as their parameters remain unchanged; only the method for inputting them has been modified. * [PepperBridgeAbstract, DealabsBridge, HotUKDealsBridge, MydealsBridge] Streamlining Group Management Coding policy fixes * [PepperBridgeAbstract, DealabsBridge, HotUKDealsBridge, MydealsBridge] Streamlining Group Management Fix wrong comment * [PepperBridgeAbstract, DealabsBridge, HotUKDealsBridge, MydealsBridge] Streamlining Group Management Add Example values for Group context --- bridges/DealabsBridge.php | 1841 +---------------- bridges/HotUKDealsBridge.php | 3205 +----------------------------- bridges/MydealsBridge.php | 1956 +----------------- bridges/PepperBridgeAbstract.php | 44 +- 4 files changed, 69 insertions(+), 6977 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index 3ee1c6f5..a3d1aaa5 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -40,1837 +40,14 @@ class DealabsBridge extends PepperBridgeAbstract 'Deals par groupe' => [ 'group' => [ 'name' => 'Groupe', - 'type' => 'list', - 'title' => 'Groupe dont il faut afficher les deals', - 'values' => [ - 'Abattants WC' => 'abattants-wc', - 'Abonnement PlayStation Plus' => 'playstation-plus', - 'Abonnements cinéma' => 'abonnements-cinema', - 'Abonnements de train' => 'abonnements-de-train', - 'Abonnements internet' => 'abonnements-internet', - 'Abonnements presse' => 'abonnements-presse', - 'Accessoires aquarium' => 'accessoires-aquarium', - 'Accessoires auto' => 'auto', - 'Accessoires électroniques' => 'accessoires-gadgets', - 'Accessoires gamers PC' => 'accessoires-gamers-pc', - 'Accessoires gaming' => 'accessoires-gaming', - 'Accessoires iPhone' => 'accessoires-iphone', - 'Accessoires mode' => 'accessoires-mode', - 'Accessoires moto' => 'moto', - 'Accessoires Nintendo' => 'accessoires-nintendo', - 'Accessoires PC portables' => 'accessoires-pc-portables', - 'Accessoires photo' => 'accessoires-photo', - 'Accessoires PlayStation' => 'accessoires-playstation', - 'Accessoires pour barbecue' => 'accessoires-barbecue', - 'Accessoires studio photo' => 'accessoires-studio-photo', - 'Accessoires téléphonie' => 'accessoires-telephonie', - 'Accessoires TV' => 'accessoires-tv', - 'Accessoires vélo' => 'accessoires-velo', - 'Accessoires Xbox' => 'accessoires-xbox', - 'Acer' => 'acer', - 'Acer Predator' => 'acer-predator', - 'Achats / Ventes' => 'achats-ventes-echanges-estimations-dons', - 'Achats à l'étranger' => 'limport-sites-avis-questions-langues', - 'Adaptateurs' => 'adaptateurs', - 'Adhérents Fnac' => 'adherents-fnac', - 'Adhésions & Souscriptions' => 'adhesions-souscriptions-abonnements', - 'adidas' => 'adidas', - 'Adidas Gazelle' => 'adidas-gazelle', - 'adidas Stan Smith' => 'adidas-stan-smith', - 'adidas Superstar' => 'adidas-superstar', - 'adidas Ultraboost' => 'adidas-ultraboost', - 'adidas Yung-1' => 'adidas-yung-1', - 'adidas ZX Flux' => 'adidas-zx-flux', - 'Adoucissant' => 'adoucissant', - 'Agendas' => 'agendas', - 'Age of Empires' => 'age-of-empires', - 'Age of Empires: Definitive Edition' => 'age-of-empires-definitive-edition', - 'Alarmes' => 'alarmes', - 'Albums photo' => 'albums-photo', - 'Alcools' => 'alcools', - 'Alcools forts' => 'alcools-forts', - 'Alimentation' => 'epicerie', - 'Alimentation bébés' => 'alimentation-bebes', - 'Alimentation PC' => 'alimentation-pc', - 'Alimentation sportifs' => 'alimentation-sportifs', - 'Amazfit Bip' => 'xiaomi-amazfit-bip', - 'Amazon Echo' => 'amazon-echo', - 'Amazon Echo Dot' => 'amazon-echo-dot', - 'Amazon Echo Plus' => 'amazon-echo-plus', - 'Amazon Echo Show' => 'amazon-echo-show', - 'Amazon Echo Show 5' => 'amazon-echo-show-5', - 'Amazon Echo Spot' => 'amazon-echo-spot', - 'Amazon Fire TV' => 'amazon-fire-tv', - 'Amazon Kindle' => 'amazon-kindle', - 'Amazon Prime' => 'amazon-prime', - 'AMD Radeon' => 'amd-radeon', - 'AMD Ryzen' => 'amd-ryzen', - 'AMD Ryzen 5 5600X' => 'amd-ryzen-5-5600x', - 'AMD Ryzen 7 5800X' => 'amd-ryzen-7-5800x', - 'AMD Ryzen 9 5900X' => 'amd-ryzen-9-5900x', - 'AMD Ryzen 9 5950X' => 'amd-ryzen-9-5950x', - 'AMD Vega' => 'amd-vega', - 'amiibo' => 'amiibo', - 'Amplis (guitare/basse)' => 'amplis-guitare-basse', - 'Amplis audio' => 'amplis', - 'Ampoules' => 'ampoules', - 'Ampoules à LED' => 'ampoules-a-led', - 'Angleterre' => 'angleterre', - 'Animal Crossing' => 'animal-crossing', - 'Animal Crossing: New Horizons' => 'animal-crossing-new-horizons', - 'Animaux' => 'animaux', - 'Anker' => 'anker', - 'Anno 1800' => 'anno-1800', - 'Annonces officielles' => 'annonces-officielles', - 'Anthem' => 'anthem', - 'Anti-nuisibles' => 'anti-nuisibles', - 'Anti-puces' => 'anti-puces', - 'Antivirus' => 'antivirus', - 'Antivols' => 'antivols', - 'Apex Legends' => 'apex-legends', - 'Appareils à raclette' => 'appareils-raclette', - 'Appareils de musculation' => 'appareils-de-musculation', - 'Appareils photo' => 'appareils-photo', - 'Appareils photo Canon' => 'appareils-photo-canon', - 'Appareils photo compacts' => 'appareils-photo-compacts', - 'Appareils photo instantanés' => 'appareils-photo-instantanes', - 'Appareils photo Nikon' => 'appareils-photo-nikon', - 'Appareils photo Olympus' => 'appareils-photo-olympus', - 'Appareils photo Panasonic' => 'appareils-photo-panasonic', - 'Appareils photo Sony' => 'appareils-photo-sony', - 'Apple' => 'apple', - 'Apple AirPods' => 'apple-airpods', - 'Apple AirPods 2' => 'apple-airpods-2', - 'Apple AirPods Max' => 'apple-airpods-max', - 'Apple AirPods Pro' => 'apple-airpods-pro', - 'Apple HomePod' => 'apple-homepod', - 'Apple HomePod Mini' => 'apple-homepod-mini', - 'Apple TV' => 'apple-tv', - 'Apple TV+' => 'apple-tv-plus', - 'Apple Watch' => 'apple-watch', - 'Apple Watch 3' => 'apple-watch-3', - 'Apple Watch 4' => 'apple-watch-4', - 'Apple Watch 5' => 'apple-watch-5', - 'Apple Watch 6' => 'apple-watch-6', - 'Apple Watch SE' => 'apple-watch-se', - 'Applications' => 'applications', - 'Applications Android' => 'applications-android', - 'Applications iOS' => 'applications-ios', - 'Appliques murales' => 'appliques-murales', - 'Applis & logiciels' => 'applis-logiciels', - 'Après-shampooings' => 'apres-shampooings', - 'Aquariums' => 'aquariums', - 'Arbres à chat' => 'arbres-a-chat', - 'Arduino' => 'arduino', - 'Armoires & placards' => 'armoires-et-placards', - 'Articles de cuisine et d'entretien' => 'articles-de-cuisine', - 'Arts culinaires' => 'arts-culinaires', - 'Arts de la table' => 'arts-de-la-table', - 'ASICS' => 'asics', - 'Asmodée' => 'asmodee', - 'Aspirateurs' => 'aspirateurs', - 'Aspirateurs balais' => 'aspirateurs-balais', - 'Aspirateurs Dreame' => 'aspirateurs-xiaomi', - 'Aspirateurs Dyson' => 'aspirateurs-dyson', - 'Aspirateurs robot' => 'aspirateurs-robot', - 'Aspirateurs Rowenta' => 'apsirateurs-rowenta', - 'Aspirateurs sans sac' => 'aspirateurs-sans-sac', - 'Assassin's Creed' => 'assassin-s-creed', - 'Assassin's Creed: Unity' => 'assassins-creed-unity', - 'Assassin's Creed: Valhalla' => 'assassin-s-creed-valhalla', - 'Assassin's Creed Odyssey' => 'assassin-s-creed-odyssey', - 'Assassin's Creed Origins' => 'assassin-s-creed-origins', - 'Assurances' => 'assurances', - 'Astuces pour économiser' => 'vos-astuces-pour-faire-des-economies', - 'Asus' => 'asus', - 'Asus ROG' => 'asus-rog', - 'Asus ROG Phone' => 'asus-rog-phone', - 'Asus ROG Phone 2' => 'asus-rog-phone-2', - 'ASUS Transformer' => 'asus-transformer', - 'Asus VivoBook' => 'asus-vivobook', - 'Asus ZenBook' => 'asus-zenbook', - 'Asus ZenFone 2' => 'asus-zenfone-2', - 'Asus ZenFone 3' => 'asus-zenfone-3', - 'Asus ZenFone 4' => 'asus-zenfone-4', - 'Asus ZenFone 6' => 'asus-zenfone-6', - 'Asus ZenFone GO' => 'asus-zenfone-go', - 'Asus ZenFone Zoom' => 'asus-zenfone-zoom', - 'Audio & Hi-fi' => 'audio-et-hi-fi', - 'Aukey' => 'aukey', - 'Auto-Moto' => 'auto-moto', - 'Autoradios' => 'autoradios', - 'Azzaro Wanted' => 'azzaro-wanted', - 'Baby foot' => 'baby-foot', - 'BabyLiss' => 'babyliss', - 'Babyphones' => 'babyphones', - 'Badminton' => 'badminton', - 'Bagagerie' => 'bagagerie', - 'Baignoires pour bébé' => 'baignoires-pour-bebe', - 'Bains de bouche' => 'bains-de-bouche', - 'Balais & serpillères' => 'balais-et-serpilleres', - 'Balances connectées' => 'balances-connectees', - 'Balançoires' => 'balancoires', - 'Ballet & danse' => 'ballet-et-danse', - 'Ballons de football' => 'ballons-de-football', - 'Bandes dessinées' => 'bandes-dessinees', - 'Banques' => 'banques', - 'Barbecue' => 'barbecue', - 'Barbecue électrique' => 'barbecue-electrique', - 'Barbecue Weber' => 'barbecue-weber', - 'Barbie' => 'barbie', - 'Barres de son' => 'barres-de-son', - 'Barres de son Yamaha' => 'barres-de-son-yamaha', - 'Batman Arkham' => 'batman-arkham', - 'Batteries externes' => 'batteries-externes', - 'Batteries voiture' => 'batteries-voiture', - 'Batteurs' => 'batteurs-electriques', - 'Battlefield' => 'battlefield', - 'Battlefield 1' => 'battlefield-1', - 'Battlefield V' => 'battlefield-5', - 'Béaba' => 'beaba', - 'Beats by Dre' => 'beats-by-dre', - 'Beats Solo 3' => 'beats-solo-3', - 'Beats Studio 3' => 'beats-studio-3', - 'Beauté' => 'beaute', - 'Bébés' => 'bebes-nouveaux-nes', - 'BenQ' => 'benq', - 'Be quiet!' => 'be-quiet', - 'Beyerdynamic MMX 300' => 'beyerdynamic-mmx-300', - 'Biberons' => 'biberons', - 'Bien-être & santé' => 'bien-etre-et-massages', - 'Bières' => 'bieres', - 'Bijoux' => 'bijoux', - 'Bikinis' => 'bikinis', - 'Bilans de santé & dépistages' => 'bilans-de-sante-et-depistages', - 'Billets de bus' => 'billets-de-bus', - 'Billets de train' => 'billets-de-train', - 'BioShock' => 'bioshock', - 'BioShock Infinite' => 'bioshock-infinite', - 'Bitdefender' => 'bitdefender', - 'Blabla' => 'blabla-parlez-de-tout-et-de-rien', - 'Black & Decker' => 'black-decker', - 'Blackberry' => 'blackberry', - 'Black Desert Online' => 'black-desert-online', - 'Blédina' => 'bledina', - 'Blenders' => 'blenders', - 'Bleu de Chanel' => 'bleu-de-chanel', - 'Blousons de moto' => 'blousons-de-moto', - 'Blu-Ray' => 'blu-ray', - 'Bodys pour bébé' => 'bodys-pour-bebe', - 'Boissons' => 'boissons', - 'Boîtes à outils' => 'boites-a-outils', - 'Boîtiers PC' => 'boitiers-pc', - 'Boîtiers TV' => 'boitiers-tv', - 'Bonbons' => 'bonbons', - 'Bonnets' => 'bonnets', - 'Bonnets de bain' => 'bonnets-de-bain', - 'Borderlands' => 'borderlands', - 'Borderlands 3' => 'borderlands-3', - 'Bosch' => 'bosch', - 'Bose' => 'bose', - 'Bose Headphones 700' => 'bose-headphones-700', - 'Bose Home Speaker 500' => 'bose-home-speaker-500', - 'Bose QuietComfort' => 'bose-quietcomfort', - 'Bose QuietComfort 35 II' => 'bose-quietcomfort-35ii', - 'Bose SoundLink' => 'bose-soundlink', - 'Bose SoundTouch' => 'bose-soundtouch', - 'Bottes' => 'bottes', - 'Bottes de moto' => 'bottes-de-moto', - 'Bottes de neige' => 'bottes-neige', - 'Bottes de pluie' => 'bottes-pluie', - 'Bottes femme' => 'bottes-femme', - 'Bottes homme' => 'bottes-homme', - 'Bougies & bougeoirs' => 'bougies-et-bougeoirs', - 'Box beauté' => 'box-beaute', - 'Bracelet fitness' => 'bracelet-fitness', - 'Brandt' => 'brandt', - 'Braun Series 3' => 'braun-series-3', - 'Braun Series 5' => 'braun-series-5', - 'Braun Series 7' => 'braun-series-7', - 'Braun Series 9' => 'braun-series-9', - 'Braun Silk Épil' => 'braun-silk-epil', - 'Brita' => 'brita', - 'Brosses à dents' => 'brosses-a-dents', - 'Brosses à dents électriques' => 'brosses-a-dents-electriques', - 'Brosses à dents électriques Oral-B' => 'brosses-a-dents-electriques-oral-b', - 'Brosses pour animaux' => 'brosses-pour-animaux', - 'Cable management' => 'cable-management', - 'Câbles' => 'cables', - 'Câbles Ethernet' => 'cables-ethernet', - 'Câbles HDMI' => 'cables-hdmi', - 'Câbles Jack' => 'cables-jack', - 'Câbles USB' => 'cables-usb', - 'Cadeaux' => 'cadeaux', - 'Cadres' => 'cadres', - 'Cadres de vélo' => 'cadres-de-velo', - 'Café' => 'cafe', - 'Café en dosettes' => 'cafe-en-dosettes', - 'Café en grain' => 'cafe-en-grain', - 'Cafetières' => 'cafetieres', - 'Cafetières expresso' => 'cafetieres-expresso', - 'Cafetières filtre' => 'cafetieres-filtre', - 'Cafetières italiennes' => 'cafetieres-italiennes', - 'Cahiers' => 'cahiers', - 'Caissons de basses' => 'caissons-de-basses', - 'Calendrier de l'Avent Lego' => 'calendriers-avent-lego', - 'Calendriers' => 'calendriers', - 'Calendriers de l'Avent' => 'calendriers-avent', - 'Call of Duty' => 'call-of-duty', - 'Call of Duty: Black Ops Cold War' => 'call-of-duty-black-ops-cold-war', - 'Call of Duty: Black Ops III' => 'call-of-duty-black-ops-3', - 'Call of Duty: Black Ops IIII' => 'call-of-duty-black-ops-4', - 'Call of Duty: Infinite Warfare' => 'call-of-duty-infinite-warfare', - 'Call of Duty: Modern Warfare' => 'call-of-duty-modern-warfare', - 'Call of Duty: WW2' => 'call-of-duty-ww2', - 'Calor' => 'calor', - 'Caméras' => 'cameras', - 'Caméras IP' => 'cameras-ip', - 'Caméras sportives' => 'cameras-sportives', - 'Camping' => 'camping', - 'Canapés' => 'canape', - 'Canon' => 'canon', - 'Captain Toad: Treasure Tracker' => 'captain-toad-treasure-tracker', - 'Caravanes' => 'caravanes', - 'Carburant' => 'carburant', - 'Cartables' => 'cartables', - 'Cartes & programmes de fidélité' => 'cartes-et-programmes-de-fidelite', - 'Cartes bancaires' => 'cartes-bancaires', - 'Cartes de développement' => 'cartes-developpement', - 'Cartes graphiques' => 'cartes-graphiques', - 'Cartes mémoire' => 'cartes-memoire', - 'Cartes mères' => 'cartes-meres', - 'Cartes postales' => 'cartes-postales', - 'Cartes prépayées Playstation Store' => 'playstation-store', - 'Cartes SD' => 'cartes-sd', - 'Cartes son' => 'cartes-son', - 'Casio' => 'casio', - 'Casque sans fil Xbox' => 'casque-sans-fil-xbox', - 'Casques Apple' => 'casques-apple', - 'Casques à réduction de bruit' => 'casque-reduction-active-bruit', - 'Casques audio' => 'casques-audio', - 'Casques Bose' => 'casques-bose', - 'Casques de moto' => 'casques-de-moto', - 'Casques de vélo' => 'casques-de-velo', - 'Casques Jabra' => 'casques-jabra', - 'Casques Samsung' => 'casques-samsung', - 'Casques sans fil' => 'casques-sans-fil', - 'Casques Sennheiser' => 'casques-sennheiser', - 'Casques Sony' => 'casques-sony', - 'Casques VR' => 'vr', - 'Casquettes' => 'casquettes', - 'Casseroles' => 'casseroles', - 'Catit' => 'catit', - 'Caves à vin' => 'caves-a-vin', - 'CD & vinyles' => 'cd-vinyles', - 'CDAV' => 'cdav', - 'Ceintures' => 'ceintures', - 'Centrales vapeur' => 'centrales-vapeur', - 'Chaînes hi-fi' => 'chaines-hi-fi', - 'Chaises' => 'chaises', - 'Chaises hautes' => 'chaises-hautes', - 'Chambre' => 'chambre', - 'Champagne' => 'champagne', - 'Chapeaux' => 'chapeaux', - 'Chapeaux & casquettes' => 'chapeaux-casquettes', - 'Chargeurs' => 'chargeurs', - 'Chargeurs allume-cigare' => 'chargeurs-allume-cigare', - 'Chargeurs de piles' => 'chargeurs-de-piles', - 'Chargeurs sans fil' => 'chargeurs-sans-fil', - 'Chasse' => 'chasse', - 'Chatières' => 'chatieres', - 'Chats' => 'chats', - 'Chauffage' => 'chauffage', - 'Chaussettes & collants' => 'chaussettes-et-collants', - 'Chaussons' => 'chaussons', - 'Chaussures' => 'chaussures', - 'Chaussures adidas' => 'chaussures-adidas', - 'Chaussures de football' => 'chaussures-de-football', - 'Chaussures de randonnée' => 'chaussures-de-randonnee', - 'Chaussures de ski' => 'chaussures-de-ski', - 'Chaussures de ville' => 'chaussures-de-ville', - 'Chaussures New Balance' => 'chaussures-new-balance', - 'Chaussures Nike' => 'chaussures-nike', - 'Chaussures pour enfants' => 'chaussures-enfants', - 'Chaussures pour femme' => 'chaussures-femme', - 'Chaussures pour homme' => 'chaussures-homme', - 'Chaussures Puma' => 'chaussures-puma', - 'Chaussures Reebok' => 'chaussures-reebok', - 'Chaussures running' => 'chaussures-de-running', - 'Chelsea boots' => 'chelsea-boots', - 'Chemises' => 'chemises', - 'Chiens' => 'chiens', - 'Chocolat' => 'chocolat', - 'Chuck Taylor' => 'chuck-taylor', - 'Cinéma' => 'cinema', - 'Cire dépilatoire' => 'cire-depilatoire', - 'Cirque & arts de rue' => 'cirque-et-arts-de-rue', - 'Citytrips' => 'citytrips', - 'Civilization' => 'civilization', - 'Civilization VI' => 'civilization-vi', - 'CK One' => 'ck-one', - 'Clarks' => 'clarks', - 'Claviers' => 'claviers', - 'Claviers (musique)' => 'claviers-musique', - 'Claviers gamer' => 'claviers-gamer', - 'Claviers Logitech' => 'claviers-logitech', - 'Claviers mécaniques' => 'claviers-mecaniques', - 'Claviers sans fil' => 'claviers-sans-fil', - 'Clés USB' => 'cles-usb', - 'Climatisation' => 'climatisation', - 'Climatiseurs' => 'climatiseurs', - 'Cocottes' => 'cocottes', - 'Coffrets de livres' => 'coffrets-de-livres', - 'Coffrets DVD' => 'coffrets-dvd', - 'Coffrets maquillage' => 'coffrets-maquillage', - 'Colliers & laisses' => 'colliers-et-laisses', - 'Compléments alimentaires' => 'complements-alimentaires', - 'Composteurs' => 'composteurs', - 'Concerts' => 'concerts', - 'Concours' => 'concours', - 'Congélateurs' => 'congelateurs', - 'Connectiques' => 'connectiques', - 'Console Google Stadia' => 'google-stadia', - 'Console Nintendo Classic Mini' => 'nintendo-classic-mini', - 'Console Nintendo Classic Mini: SNES' => 'nintendo-classic-mini-snes', - 'Console Nintendo Switch' => 'nintendo-switch', - 'Console Nintendo Switch Lite' => 'nintendo-switch-lite', - 'Console PS4' => 'playstation-4', - 'Console PS4 Pro' => 'playstation-4-pro', - 'Console PS5' => 'playstation-5', - 'Consoles' => 'consoles', - 'Consoles & jeux vidéo' => 'consoles-jeux-video', - 'Console Sega Mega Drive Mini' => 'sega-mega-drive-mini', - 'Console Xbox One S' => 'xbox-one-s', - 'Console Xbox One X' => 'xbox-one-x', - 'Console Xbox Series S' => 'xbox-series-s', - 'Console Xbox Series X' => 'xbox-series-x', - 'Consommables imprimantes' => 'consommables-imprimantes', - 'Converse' => 'converse', - 'Coques iPhone' => 'coques-iphone', - 'Corsair Void PRO' => 'corsair-void-pro', - 'Costumes' => 'costumes', - 'Costumes & déguisements' => 'costumes-et-deguisements', - 'Couches' => 'couches', - 'Couettes' => 'couettes', - 'Coupes menstruelles' => 'coupes-menstruelles', - 'Cours & formations' => 'cours-et-formations', - 'Courses hippiques' => 'courses-hippiques', - 'Couteaux de cuisine' => 'couteaux-de-cuisine', - 'Couture' => 'couture', - 'Couverts' => 'couverts', - 'Couverts pour bébés' => 'couverts-pour-bebes', - 'Covoiturage' => 'covoiturage', - 'Crash Team Racing Nitro-Fueled' => 'crash-team-racing-nitro-fueled', - 'Cravates' => 'cravates', - 'Crédits' => 'credits', - 'Crèmes hydratantes' => 'cremes-hydratantes', - 'Crèmes solaires' => 'cremes-solaires', - 'Croisières' => 'croisieres', - 'Croquettes pour chat' => 'croquettes-pour-chat', - 'Croquettes pour chien' => 'croquettes-pour-chien', - 'Cuiseurs à riz' => 'cuiseur-riz', - 'Cuisinières' => 'cuisinieres', - 'Culottes menstruelles' => 'culottes-menstruelles', - 'Culture & divertissement' => 'culture-divertissement', - 'Cyberpunk 2077' => 'cyberpunk-2077', - 'Cyclisme' => 'cyclisme', - 'Cyclisme & sports urbains' => 'cyclisme-sports-urbains', - 'Darksiders' => 'darksiders', - 'Dashcams' => 'dashcams', - 'DDR3' => 'ddr3', - 'DDR4' => 'ddr4', - 'Dead Rising' => 'dead-rising', - 'Death Stranding' => 'death-stranding', - 'Décoration' => 'decoration', - 'Décorations de Noël' => 'decoration-noel', - 'Deebot' => 'ecovacs-deebot', - 'Deezer' => 'deezer', - 'Dell' => 'dell', - 'Dell XPS' => 'dell-xps', - 'Delsey' => 'delsey', - 'Demandes de deals' => 'les-demandes-de-deals', - 'Denon' => 'denon', - 'Dentifrices' => 'dentifrices', - 'Déodorants' => 'deodorants', - 'Désherbants' => 'desherbants', - 'Déshumidificateurs' => 'deshumidificateurs', - 'Désinfectant' => 'desinfectants', - 'Désodorisants & parfums d'intérieur' => 'desodorisants-et-parfums-d-interieur', - 'Destiny' => 'destiny', - 'Destiny 2' => 'destiny-2', - 'Détecteurs de fumée' => 'detecteurs-de-fumee', - 'Detroit: Become Human' => 'detroit-become-human', - 'Deus Ex' => 'deus-ex', - 'Deus Ex: Mankind Divided' => 'deus-ex-mankind-divided', - 'Devil May Cry 5' => 'devil-may-cry-5', - 'Dishonored' => 'dishonored', - 'Dishonored 2' => 'dishonored-2', - 'Disney+' => 'disney-plus', - 'Disneyland Paris' => 'disneyland-paris', - 'Disques durs (internes)' => 'hdd', - 'Disques durs externes' => 'disques-durs-externes', - 'Divers' => 'divers', - 'DJI' => 'dji', - 'DJI Mavic Air 2' => 'dji-mavic-air-2', - 'DJI Mavic Mini' => 'dji-mavic-mini', - 'Dolce Gusto' => 'dolce-gusto', - 'Domotique' => 'smart-home', - 'Doom Eternal' => 'doom-eternal', - 'Dosettes Dolce Gusto' => 'dosettes-dolce-guste', - 'Dosettes Nespresso' => 'dosettes-nespresso', - 'Dosettes Senseo' => 'dosettes-senseo', - 'Dosettes Tassimo' => 'dosettes-tassimo', - 'Dr. Martens' => 'dr-martens', - 'Dragon Age' => 'dragon-age', - 'Dragon Ball' => 'dragon-ball', - 'Dragon Ball FighterZ' => 'dragon-ball-fighterz', - 'Dragon Ball Z: Kakarot' => 'dragon-ball-z-kakarot', - 'Dragon Quest' => 'dragon-quest', - 'Dragon Quest Builders' => 'dragon-quest-builders', - 'Dragon Quest Builders 2' => 'dragon-quest-builders-2', - 'Draisiennes' => 'draisiennes', - 'Draps & housses' => 'draps-et-housses', - 'Dreame V10' => 'xiaomi-dreame-v10', - 'Dreame V11' => 'xiaomi-dreame-v11', - 'Drones' => 'drones', - 'Durex' => 'durex', - 'DVD' => 'dvd', - 'Dying Light' => 'dying-light', - 'Dying Light 2' => 'dying-light-2', - 'Dyson' => 'dyson', - 'Dyson V10' => 'dyson-v10', - 'Dyson V11' => 'dyson-v11', - 'Eastpak' => 'eastpak', - 'Ebooks' => 'ebooks', - 'Écharpes & foulards' => 'echarpes-et-foulards', - 'Éclairage intelligent' => 'smart-light', - 'Écouteurs' => 'ecouteurs', - 'Écouteurs sans fil' => 'ecouteurs-sans-fil', - 'Écouteurs sport' => 'ecouteurs-sport', - 'Ecovacs' => 'ecovacs', - 'Ecovacs Deebot 900' => 'ecovacs-deebot-900', - 'Ecovacs Deebot OZMO 930' => 'ecovacs-deebot-ozmo-930', - 'Écrans' => 'ecrans', - 'Écrans 4K / UHD' => 'ecrans-4k-uhd', - 'Écrans 21" et moins' => 'ecrans-21-pouces-et-moins', - 'Écrans 24"' => 'ecrans-24-pouces', - 'Écrans 27"' => 'ecrans-27-pouces', - 'Écrans 29" et plus' => 'ecrans-29-pouces-et-plus', - 'Écrans Acer' => 'ecrans-acer', - 'Écrans Asus' => 'ecrans-asus', - 'Écrans BenQ' => 'ecrans-benq', - 'Écrans Dell' => 'ecrans-dell', - 'Écrans de projection' => 'ecrans-de-projection', - 'Écrans FreeSync' => 'ecrans-freesync', - 'Écrans gaming' => 'ecrans-gamer', - 'Écrans incurvés' => 'ecrans-incurves', - 'Écrans Philips' => 'ecrans-philips', - 'Écrans Samsung' => 'ecrans-samsung', - 'Électricité (matériel)' => 'electricite', - 'Electrolux' => 'electrolux', - 'Électroménager' => 'electromenager', - 'Embauchoirs' => 'embauchoirs', - 'Enceintes' => 'enceintes', - 'Enceintes Bluetooth' => 'enceintes-bluetooth', - 'Enceintes connectées' => 'enceintes-connectees', - 'Enceintes portables sans fil' => 'enceintes-portables-sans-fil', - 'Énergie' => 'energie', - 'Engrais' => 'engrais', - 'Épicerie & courses' => 'epicerie-courses-supermarches', - 'Épilateurs à lumière pulsée' => 'epilateurs-a-lumiere-pulsee', - 'Épilateurs électriques' => 'epilateurs-electriques', - 'Épilation' => 'epilation', - 'Équipement motard' => 'equipement-motard', - 'Équipement running' => 'equipement-running', - 'Équipement sportif' => 'equipement-sportif', - 'Érotisme' => 'erotisme', - 'Escarpins' => 'escarpins', - 'Événements sportifs' => 'evenements-sportifs', - 'Expositions' => 'expositions', - 'Extracteurs de jus' => 'extracteurs-de-jus', - 'F1 2017' => 'f1-2017', - 'F1 2019' => 'f1-2019', - 'Facom' => 'facom', - 'Fallout' => 'fallout', - 'Fallout 4' => 'fallout-4', - 'Fallout 76' => 'fallout-76', - 'Famille & enfants' => 'famille-enfants', - 'Far Cry' => 'far-cry', - 'Far Cry New Dawn' => 'far-cry-new-dawn', - 'Fards à paupières' => 'fards-a-paupieres', - 'Fast-foods' => 'fast-foods', - 'Fauteuils' => 'fauteuils', - 'Fauteuils gamer' => 'fauteuils-gaming', - 'Fe' => 'fe', - 'Fers à lisser / à friser' => 'fers-a-lisser-a-friser', - 'Fers à repasser' => 'fers-a-repasser', - 'Fers à souder' => 'fers-a-souder', - 'Festivals' => 'festivals', - 'Feutres' => 'feutres', - 'FIFA' => 'fifa', - 'FIFA 17' => 'fifa-17', - 'FIFA 18' => 'fifa-18', - 'FIFA 19' => 'fifa-19', - 'FIFA 20' => 'fifa-20', - 'FIFA 21' => 'fifa-21', - 'Figurines' => 'figurines', - 'Films & Séries' => 'films', - 'Final Fantasy' => 'final-fantasy', - 'Final Fantasy XII' => 'final-fantasy-xii', - 'Finances & Assurances' => 'finances-assurances', - 'fitbit' => 'fitbit', - 'Fitness & yoga' => 'fitness-yoga', - 'Flash' => 'flash', - 'Fluval' => 'fluval', - 'Foires & salons' => 'foires-et-salons', - 'Fonds de teint' => 'fonds-de-teint', - 'Football' => 'football', - 'Forfaits de ski' => 'forfaits-ski', - 'Forfaits mobiles' => 'forfaits-mobiles', - 'Forfaits mobiles et internet' => 'telecommunications', - 'For Honor' => 'for-honor', - 'Formations premiers secours' => 'formations-premiers-secours', - 'Formule 1' => 'formule-1', - 'Fortnite' => 'fortnite', - 'Fortnite: Pack Feu Obscur' => 'fortnite-pack-feu-obscur', - 'Forza' => 'forza', - 'Forza Horizon' => 'forza-horizon', - 'Forza Horizon 3' => 'forza-horizon-3', - 'Forza Horizon 4' => 'forza-horizon-4', - 'Forza Motorsport' => 'forza-motosport', - 'Forza Motorsport 7' => 'forza-motorsport-7', - 'Fossil' => 'fossil', - 'Fournitures scolaires' => 'fournitures-scolaires', - 'Fours' => 'fours', - 'Fours à poser' => 'fours-a-poser', - 'Fours encastrables' => 'fours-encastrables', - 'Friandises pour chat' => 'friandises-pour-chat', - 'Friandises pour chien' => 'friandises-pour-chien', - 'Friskies' => 'friskies', - 'Friteuses' => 'friteuses', - 'Friteuses sans huile' => 'friteuses-sans-huile', - 'Fruits & légumes' => 'fruits-et-legumes', - 'Fujifilm' => 'fujifilm', - 'Funko Pop' => 'funko-pop', - 'FURminator' => 'furminator', - 'Futuroscope' => 'futuroscope', - 'Gamelles' => 'gamelles', - 'Game of Thrones' => 'game-of-thrones', - 'Gaming' => 'le-laboratoire-des-gamers', - 'Gants' => 'gants', - 'Gants moto' => 'gants-moto', - 'Garmin' => 'garmin', - 'Garmin Fenix' => 'garmin-fenix', - 'Garmin Forerunner' => 'garmin-forerunner', - 'Garmin Vivoactive' => 'garmin-vivoactive', - 'Garmin Vivomove' => 'garmin-vivomove', - 'Gâteaux & biscuits' => 'gateaux-et-biscuits', - 'Gears 5' => 'gears-5', - 'Gel hydroalcoolique' => 'gel-hydroalcoolique', - 'Gels douche' => 'gels-douche', - 'Geox' => 'geox', - 'Ghost of Tsushima' => 'ghost-of-tsushima', - 'Gigoteuses' => 'gigoteuses', - 'Gillette Fusion' => 'gillette-fusion', - 'Gillette Mach3' => 'gillette-mach3', - 'Glaces' => 'glaces', - 'Glacières' => 'glacieres', - 'Glisse urbaine' => 'glisse-urbaine', - 'God of War' => 'god-of-war', - 'Google Chromecast' => 'google-chromecast', - 'Google Home' => 'google-home', - 'Google Home Max' => 'google-home-max', - 'Google Home Mini' => 'google-home-mini', - 'Google Nest Hub' => 'google-nest-hub', - 'Google Nest Mini' => 'google-nest-mini', - 'Google Pixel' => 'google-pixel', - 'Google Pixel 2' => 'google-pixel-2', - 'Google Pixel 2 XL' => 'google-pixel-2-xl', - 'Google Pixel 3' => 'google-pixel-3', - 'Google Pixel 3 XL' => 'google-pixel-3-xl', - 'Google Pixel 3a' => 'google-pixel-3a', - 'Google Pixel 4' => 'google-pixel-4', - 'Google Pixel 4 XL' => 'google-pixel-4xl', - 'Google Pixel 4a' => 'google-pixel-4a', - 'Google Pixel 5' => 'google-pixel-5', - 'Google Pixel XL' => 'google-pixel-xl', - 'GoPro' => 'gopro-hero', - 'GoPro Hero 9' => 'gopro-hero-9', - 'Gran Turismo' => 'gran-turismo', - 'Grille-pain' => 'grille-pain', - 'Grossesse & maternité' => 'grossesse-maternite', - 'GTA' => 'gta', - 'GTA V' => 'gta-v', - 'GTX 1060' => 'nvidia-geforce-gtx-1060', - 'GTX 1070' => 'nvidia-geforce-gtx-1070', - 'GTX 1080' => 'nvidia-geforce-gtx-1080', - 'GTX 1080 Ti' => 'nvidia-geforce-gtx-1080-ti', - 'GTX 1650' => 'gtx-1650', - 'GTX 1660' => 'gtx-1660', - 'GTX 1660 Ti' => 'gtx-1660-ti', - 'Guerlain La Petite Robe Noire' => 'guerlain-petite-robe-noire', - 'Guirlandes lumineuses' => 'guirlandes-lumineuses', - 'Guitares' => 'guitares', - 'Gyropodes' => 'gyropodes', - 'Half Life' => 'half-life', - 'Half Life 2' => 'half-life-2', - 'Half Life Alyx' => 'half-life-alyx', - 'Halloween' => 'halloween', - 'Haltères & poids' => 'halteres-et-poids', - 'Hama' => 'hama', - 'Hamacs' => 'hamacs', - 'Hand spinners' => 'hand-spinners', - 'Harnais pour chien' => 'harnais-pour-chien', - 'Harry Potter' => 'harry-potter', - 'Havaianas' => 'havaianas', - 'High-Tech' => 'high-tech', - 'High-tech & informatique' => 'le-laboratoire-high-tech-informatique', - 'Hisense' => 'hisense', - 'Home Cinéma' => 'home-cinema', - 'Honor' => 'honor', - 'Honor 6X' => 'honor-6x', - 'Honor 8' => 'honor-8', - 'Honor 8 Pro' => 'honor-8-pro', - 'Honor 8X' => 'honor-8x', - 'Honor 8X Max' => 'honor-8x-max', - 'Honor 9' => 'honor-9', - 'Honor 10' => 'honor-10', - 'Honor 20' => 'honor-20', - 'Honor 20 Lite' => 'honor-20-lite', - 'Honor 20 Pro' => 'honor-20-pro', - 'Honor Band 5' => 'honor-band-5', - 'Honor MagicBook' => 'honor-magicbook', - 'Honor MagicWatch 2' => 'honor-magicwatch-2', - 'Honor View 20' => 'honor-view-20', - 'Horizon Zero Dawn' => 'horizon-zero-dawn', - 'Hôtels & Hébergements' => 'hotels', - 'Hoverboards' => 'hoverboards', - 'HTC 10' => 'htc-10', - 'HTC Desire' => 'htc-desire', - 'HTC One M9' => 'htc-one-m9', - 'HTC U11' => 'htc-u11', - 'HTC U Play' => 'htc-u-play', - 'HTC U Ultra' => 'htc-u-ultra', - 'HTC Vive' => 'htc-vive', - 'Huawei' => 'huawei', - 'Huawei FreeBuds 3' => 'huawei-freebuds-3', - 'Huawei Mate 9' => 'huawei-mate-9', - 'Huawei Mate 10' => 'huawei-mate-10', - 'Huawei Mate 10 Pro' => 'huawei-mate-10-pro', - 'Huawei Mate 20' => 'huawei-mate-20', - 'Huawei Mate 20 Lite' => 'huawei-mate-20-lite', - 'Huawei Mate 20 Pro' => 'huawei-mate-20-pro', - 'Huawei Mate 20 RS' => 'huawei-mate-20-rs', - 'Huawei Mate 30' => 'huawei-mate-30', - 'Huawei Mate 30 Lite' => 'huawei-mate-30-lite', - 'Huawei Mate 30 Pro' => 'huawei-mate-30-pro', - 'Huawei P8 Lite' => 'huawei-p8-lite', - 'Huawei P9 Lite' => 'huawei-p9-lite', - 'Huawei P10' => 'huawei-p10', - 'Huawei P10 Lite' => 'huawei-p10-lite', - 'Huawei P10 Plus' => 'huawei-p10-plus', - 'Huawei P20' => 'huawei-p20', - 'Huawei P20 Lite' => 'huawei-p20-lite', - 'Huawei P20 Pro' => 'huawei-p20-pro', - 'Huawei P30' => 'huawei-p30', - 'Huawei P30 Lite' => 'huawei-p30-lite', - 'Huawei P30 Pro' => 'huawei-p30-pro', - 'Huawei P40' => 'huawei-p40', - 'Huawei P40 Lite' => 'huawei-p40-lite', - 'Huawei P40 Pro' => 'huawei-p40-pro', - 'Huawei Watch' => 'huawei-watch', - 'Huawei Watch 2' => 'huawei-watch-2', - 'Hubs' => 'hubs', - 'Hugo Boss Bottled' => 'hugo-boss-bottled', - 'Huile moteur' => 'huile-moteur', - 'Hygiène & soins' => 'hygiene-soins', - 'Hygiène de la maison' => 'hygiene-de-la-maison', - 'Hygiène des bébés' => 'hygiene-des-bebes', - 'Hygiène intime' => 'hygiene-intime', - 'iMac' => 'mac-de-bureau', - 'iMac 2021' => 'imac-2021', - 'Image, son, photo' => 'le-laboratoire-audiovisuel', - 'Impressions photo' => 'impressions-photo', - 'Imprimantes' => 'imprimantes', - 'Imprimantes 3D' => 'imprimantes-3d', - 'Imprimantes Brother' => 'imprimantes-brother', - 'Imprimantes Canon' => 'imprimantes-canon', - 'Imprimantes Epson' => 'imprimantes-epson', - 'Imprimantes HP' => 'imprimantes-hp', - 'Imprimantes laser' => 'imprimantes-laser', - 'Imprimantes multifonctions' => 'imprimantes-multifonctions', - 'Informatique' => 'informatique', - 'Instax Mini' => 'instax-mini', - 'Instruments de musique' => 'instruments-de-musique', - 'Intel i5' => 'intel-i5', - 'Intel i7' => 'intel-i7', - 'Intel i9' => 'intel-i9', - 'iPad' => 'apple-ipad', - 'iPad 2019' => 'ipad-2019', - 'iPad 2020' => 'ipad-2020', - 'iPad Air' => 'ipad-air', - 'iPad Air 2019' => 'ipad-air-2019', - 'iPad Air 2020' => 'ipad-air-2020', - 'iPad Mini' => 'apple-ipad-mini', - 'iPad Pro' => 'apple-ipad-pro', - 'iPad Pro 11' => 'ipad-pro-11', - 'iPad Pro 12.9' => 'ipad-pro-12-9', - 'iPad Pro 2020' => 'ipad-pro-2020', - 'iPhone' => 'apple-iphone', - 'iPhone 6' => 'apple-iphone-6', - 'iPhone 7' => 'apple-iphone-7', - 'iPhone 7 Plus' => 'apple-iphone-7-plus', - 'iPhone 8' => 'apple-iphone-8', - 'iPhone 8 Plus' => 'apple-iphone-8-plus', - 'iPhone 11' => 'iphone-11', - 'iPhone 11 Pro' => 'iphone-11-pro', - 'iPhone 11 Pro Max' => 'iphone-11-pro-max', - 'iPhone 12' => 'iphone-12', - 'iPhone 12 Mini' => 'iphone-12-mini', - 'iPhone 12 Pro' => 'iphone-12-pro', - 'iPhone 12 Pro Max' => 'iphone-12-pro-max', - 'iPhone SE' => 'apple-iphone-se', - 'iPhone X' => 'apple-iphone-x', - 'iPhone XR' => 'apple-iphone-xr', - 'iPhone XS' => 'apple-iphone-xs', - 'iPhone XS Max' => 'apple-iphone-xs-max', - 'iRobot Roomba' => 'irobot-roomba', - 'Isolation' => 'isolation', - 'Jabra Elite 75t' => 'jabra-elite-75t', - 'Jabra Elite 85h' => 'jabra-elite-85h', - 'Jabra Elite 85t' => 'jabra-elite-85t', - 'Jabra Elite Active 65t' => 'jabra-elite-active-65t', - 'Jacuzzis' => 'jacuzzis', - 'Jardin' => 'jardin', - 'Jardin & bricolage' => 'jardin-bricolage', - 'Jardinage' => 'entretien-du-jardin', - 'JBL' => 'jbl', - 'JBL Charge 4' => 'jbl-charge-4', - 'JBL Flip' => 'jbl-flip', - 'JBL GO' => 'jbl-go', - 'JBL Xtreme 2' => 'jbl-xtreme-2', - 'Jeans' => 'jeans', - 'Jets dentaires' => 'jets-dentaires', - 'Jeux & jouets' => 'jeux-jouets', - 'Jeux & sports de café' => 'jeux-sports-cafe-bar', - 'Jeux d'adresse' => 'jeux-adresse', - 'Jeux d'apprentissage' => 'jeux-d-apprentissage', - 'Jeux d'eau' => 'jeux-jouets-eau', - 'Jeux d'extérieur' => 'jeux-d-exterieur', - 'Jeux d'imitation' => 'jeux-d-imitation', - 'Jeux de cartes et de plateau' => 'jeux-cartes-plateau-societe', - 'Jeux de construction' => 'jeux-de-construction', - 'Jeux de hasard & paris' => 'jeux-et-paris', - 'Jeux de société' => 'jeux-de-societe', - 'Jeux Nintendo 3DS' => 'jeux-3ds', - 'Jeux Nintendo Switch' => 'jeux-nintendo-switch', - 'Jeux PC' => 'jeux-pc', - 'Jeux PC dématérialisés' => 'jeux-pc-dematerialises', - 'Jeux pour bébés' => 'jeux-pour-bebes', - 'Jeux PS4' => 'jeux-playstation-4', - 'Jeux PS4 dématérialisés' => 'jeux-ps4-dematerialises', - 'Jeux PS5' => 'jeux-playstation-5', - 'Jeux PS5 dématérialisés' => 'jeux-playstation-5-dematerialises', - 'Jeux PS Plus' => 'jeux-ps-plus', - 'Jeux vidéo' => 'jeux-video', - 'Jeux VR' => 'jeux-vr', - 'Jeux Wii U' => 'jeux-wii-u', - 'Jeux Xbox One' => 'jeux-xbox-one', - 'Jeux Xbox One dématérialisés' => 'jeux-xbox-dematerialises', - 'Jeux Xbox Series X' => 'jeux-xbox-series-x', - 'Jeux Xbox with Gold' => 'jeux-xbox-with-gold', - 'Jouets' => 'jouets', - 'Jouets pour chat' => 'jouets-pour-chat', - 'Jouets pour chien' => 'jouets-pour-chien', - 'Journaux numériques' => 'journaux-numeriques', - 'Journaux papier' => 'journaux-papier', - 'Joy-Con' => 'manettes-nintendo-switch-joy-con', - 'Jungle Speed' => 'jungle-speed', - 'Just Cause' => 'just-cause', - 'Just Cause 3' => 'just-cause-3', - 'Just Cause 4' => 'just-cause-4', - 'Kärcher' => 'karcher', - 'Kaspersky' => 'kaspersky', - 'Kinder' => 'kinder', - 'Kindle Oasis' => 'kindle-oasis', - 'Kindle Paperwhite' => 'kindle-paperwhite', - 'Kindle Voyage' => 'kindle-voyage', - 'Kingdom Hearts' => 'kingdom-hearts', - 'Kingdom Hearts 3' => 'kingdom-hearts-3', - 'Kingston HyperX Cloud II' => 'kingston-hyperx-cloud-2', - 'Kits premiers secours' => 'premiers-secours', - 'Kobo' => 'kobo', - 'Kobo Aura 2' => 'kobo-aura-2', - 'Kobo Aura H2o' => 'kobo-aura-h2o', - 'Kobo Aura One' => 'kobo-aura-one', - 'L'annale du destin' => 'l-annale-du-destin', - 'L'ombre de la guerre' => 'l-ombre-de-la-guerre', - 'L'ombre du Mordor' => 'l-ombre-du-mordor', - 'Lacoste' => 'lacoste', - 'Lampadaires' => 'lampadaires', - 'Lampes' => 'lampes', - 'Lampes de table' => 'lampes-de-table', - 'Lampes solaires' => 'lampes-solaires', - 'Lancôme La Vie est Belle' => 'lancome-la-vie-est-belle', - 'Lapeyre' => 'lapeyre', - 'La Terre du Milieu' => 'la-terre-du-milieu', - 'Lavage auto' => 'lavage-auto', - 'Lavazza' => 'lavazza', - 'Lave-linge' => 'lave-linge', - 'Lave-linge frontal' => 'lave-linge-frontal', - 'Lave-linge séchant' => 'lave-linge-sechant', - 'Lave-linge top' => 'lave-linge-top', - 'Lave-vaisselle' => 'lave-vaisselle', - 'Lay-Z-Spa' => 'lay-z-spa', - 'Leasing voiture' => 'leasing-voiture', - 'Le bâton de la vérité' => 'le-baton-de-la-verite', - 'Lecteurs Blu-Ray' => 'lecteurs-blu-ray', - 'Lecteurs CD' => 'lecteurs-cd', - 'Lecteurs DVD' => 'lecteurs-dvd', - 'Lego' => 'lego', - 'Lego Architecture' => 'lego-architecture', - 'Lego Batman' => 'lego-batman', - 'Lego City' => 'lego-city', - 'Lego Creator' => 'lego-creator', - 'Lego Dimensions' => 'lego-dimensions', - 'Lego Duplo' => 'lego-duplo', - 'Lego Friends' => 'lego-friends', - 'Lego Harry Potter' => 'lego-harry-potter', - 'Lego Ideas' => 'lego-ideas', - 'Lego Marvel' => 'lego-marvel', - 'Lego Nexo Knights' => 'lego-nexo-knights', - 'Lego Ninjago' => 'lego-ninjago', - 'Lego Star Wars' => 'lego-star-wars', - 'Lego Technic' => 'lego-technic', - 'Lenovo' => 'lenovo', - 'Lenovo IdeaPad' => 'lenovo-ideapad', - 'Lenovo K6 Note' => 'lenovo-k6-note', - 'Lenovo P8' => 'lenovo-p8', - 'Lenovo Tab 3' => 'lenovo-tab-3', - 'Lenovo Tab 4' => 'lenovo-tab-4', - 'Lenovo ThinkPad' => 'lenovo-thinkpad', - 'Lenovo Yoga' => 'lenovo-yoga', - 'Lenovo Yoga Tab 3' => 'lenovo-yoga-tab-3', - 'Lentilles de contact' => 'lentilles-de-contact', - 'Le Seigneur des anneaux' => 'le-seigneur-des-anneaux', - 'Les Sims' => 'les-sims', - 'Les Sims 4' => 'les-sims-4', - 'Lessive' => 'lessive', - 'Levi's' => 'levi-s', - 'LG' => 'lg', - 'LG G4' => 'lg-g4', - 'LG G5' => 'lg-g5', - 'LG G6' => 'lg-g6', - 'LG OLED TV' => 'lg-oled-tv', - 'LG Q6' => 'lg-q6', - 'LG Q8' => 'lg-q8', - 'Life is Strange' => 'life-is-strange', - 'Linge de maison' => 'linge-de-maison', - 'Lingerie' => 'lingerie', - 'Lingettes désinfectantes' => 'lingettes-desinfectantes', - 'Lingettes pour bébés' => 'lingettes-pour-bebes', - 'Liseuses' => 'liseuses', - 'Litière pour chat' => 'litiere-pour-chat', - 'Lits' => 'lits', - 'Lits pour bébé' => 'lits-pour-bebe', - 'Lits pour enfants' => 'lits-pour-enfants', - 'Little Nightmares' => 'little-nightmares', - 'Livraison de repas' => 'service-de-livraison-de-repas', - 'Livres & littérature' => 'livres-litterature', - 'Livres & Magazines' => 'livres', - 'Livres audio' => 'livres-audio', - 'Livres photo' => 'livres-photo', - 'Location de voiture' => 'location-de-voiture', - 'Logiciels' => 'logiciels', - 'Logiciels de sécurité' => 'logiciels-de-securite', - 'Logiciels Microsoft' => 'logiciels-microsoft', - 'Logitech' => 'logitech', - 'Logitech G502' => 'logitech-g502', - 'Logitech G703' => 'logitech-g703', - 'Logitech G Pro X' => 'logitech-g-pro-x', - 'Logitech Harmony' => 'logitech-harmony', - 'Logitech MX Master' => 'logitech-mx-master', - 'Logitech MX Master 2S' => 'logitech-mx-master-2s', - 'Loisirs créatifs' => 'loisirs-creatifs', - 'Lolita Lempicka' => 'lolita-lempicka-premier-parfum', - 'Loup-Garou' => 'loup-garou', - 'Lubrifiants' => 'lubrifiants', - 'Luges' => 'luges', - 'Luigi's Mansion 3' => 'luigi-mansion-3', - 'Luminaires' => 'luminaires', - 'Lunettes de natation' => 'lunettes-de-natation', - 'Lunettes de soleil' => 'lunettes-de-soleil', - 'M&M's' => 'metm-s', - 'MacBook' => 'macbook', - 'MacBook Air' => 'apple-macbook-air', - 'MacBook Pro' => 'apple-macbook-pro', - 'MacBook Pro 13' => 'macbook-pro-13', - 'MacBook Pro 15' => 'macbook-pro-15', - 'MacBook Pro 16' => 'macbook-pro-16', - 'Machines à café à dosettes' => 'machines-a-cafe-a-dosettes', - 'Machines à café en grain' => 'machines-a-cafe-en-grain', - 'Machines à coudre' => 'machines-a-coudre', - 'Machines à pain' => 'machines-a-pain', - 'Machines de sport' => 'machines-sport', - 'Machines Dolce Gusto' => 'machines-dolce-gusto', - 'Machines Nespresso' => 'machines-nespresso', - 'Machines Senseo' => 'machines-senseo', - 'Machines Tassimo' => 'machines-tassimo', - 'Mac mini' => 'mac-mini', - 'Madden NFL 20' => 'madden-nfl-20', - 'Magasins d'usine' => 'magasins-usine', - 'Magazines' => 'magazines', - 'Maillots de bain' => 'maillots-de-bain', - 'Maillots de football' => 'maillots-de-football', - 'Maison & Habitat' => 'maison-habitat', - 'Maisons de poupées' => 'maisons-poupees', - 'Makita' => 'makita', - 'Manettes' => 'manettes-accessoires-consoles', - 'Manettes DualSense' => 'manettes-playstation-5', - 'Manettes Nintendo Switch' => 'manettes-nintendo-switch', - 'Manettes Nintendo Switch Pro' => 'manettes-nintendo-switch-pro', - 'Manettes PlayStation 4' => 'manettes-playstation-4', - 'Manettes Xbox' => 'manettes-xbox', - 'Manettes Xbox One' => 'manettes-xbox-one', - 'Manettes Xbox One Elite' => 'manettes-xbox-one-elite', - 'Manettes Xbox Series X' => 'manettes-xbox-series-x', - 'Manix' => 'manix', - 'Manteaux' => 'manteaux', - 'Maquillage' => 'maquillage', - 'Marchands et leurs offres' => 'vos-avisdemandes-sur-les-marchands-et-leurs-offres', - 'Mario & Sonic aux Jeux Olympiques de Tokyo 2020' => 'mario-sonic-jeux-olympiques-tokyo-2020', - 'Mario Kart' => 'mario-kart', - 'Marques' => 'marques', - 'Marteaux & maillets' => 'marteaux-et-maillets', - 'Marvel's Avengers' => 'marvels-avengers', - 'Mascara' => 'mascara', - 'Masques cheveux' => 'masques-cheveux', - 'Masques de protection' => 'masques-de-protection-respiratoire', - 'Masques de ski' => 'masques-de-ski', - 'Mass Effect' => 'mass-effect', - 'Mass Effect: Andromeda' => 'mass-effect-andromeda', - 'Matchs de football' => 'matchs-de-football', - 'Matelas' => 'matelas', - 'Matelas gonflables' => 'matelas-gonflables', - 'Matériaux de construction' => 'materiaux-de-construction', - 'Matériel de ski' => 'materiel-de-ski', - 'Medion' => 'medion', - 'Metro' => 'metro', - 'Metro 2033' => 'metro-2033', - 'Metro Exodus' => 'metro-exodus', - 'Meubles pour aquarium' => 'meubles-pour-aquarium', - 'Meubles pour chat' => 'meubles-pour-chat', - 'Meubles salle de bain' => 'salle-de-bain', - 'Micro-casques gaming' => 'micro-casques-gaming', - 'Micro-ondes' => 'micro-ondes', - 'Microphones' => 'microphones', - 'Micro SD' => 'micro-sd', - 'Microsoft Flight Simulator' => 'microsoft-flight-simulator', - 'Microsoft Office' => 'microsoft-office', - 'Microsoft Surface Book' => 'microsoft-surface-book', - 'Microsoft Surface Pro 6' => 'microsoft-surface-pro-6', - 'Microsoft Surface Pro 7' => 'microsoft-surface-pro-7', - 'Miele' => 'miele', - 'Minecraft' => 'minecraft', - 'Mini PC' => 'mini-pc', - 'Mini réfrigérateurs' => 'mini-refrigerateurs', - 'Miroirs' => 'miroirs', - 'Mixeurs & Blenders' => 'mixeurs-blenders', - 'Mixeurs plongeants' => 'mixeur-plongeant', - 'Mobilier' => 'mobilier', - 'Mobilier de bureau' => 'fournitures-de-bureau', - 'Mobilier de jardin' => 'mobilier-jardin', - 'Mobilier de salon' => 'mobilier-salon', - 'Mobvoi Ticwatch' => 'mobvoi-ticwatch', - 'Mode' => 'mode', - 'Mode & accessoires' => 'mode-accessoires', - 'Mode & beauté' => 'le-laboratoire-de-la-mode-beaute', - 'Mode enfants' => 'mode-enfants', - 'Mode femme' => 'mode-femme', - 'Mode homme' => 'mode-homme', - 'Modélisme' => 'modelisme', - 'Monopoly' => 'monopoly', - 'Montage PC' => 'montage-pc', - 'Montre connectée Amazfit' => 'montres-connectees-amazfit', - 'Montre connectée Garmin' => 'montres-connectees-garmin', - 'Montre connectée Honor' => 'montres-connectees-honor', - 'Montre connectée Samsung' => 'smartwatch-samsung', - 'Montres' => 'montres', - 'Montres connectées' => 'smartwatch', - 'Mortal Kombat' => 'mortal-kombat', - 'Mortal Kombat 11' => 'mortal-kombat-11', - 'Moto C Plus' => 'moto-c-plus', - 'Moto E4' => 'moto-e4', - 'Moto G5' => 'moto-g5', - 'Moto G5 Plus' => 'moto-g5-plus', - 'Moto G5S' => 'moto-g5s', - 'Moto G5S Plus' => 'moto-g5s-plus', - 'Moto G6' => 'moto-g6', - 'Moto G6 Play' => 'moto-g6-play', - 'Moto G6 Plus' => 'moto-g6-plus', - 'Moto G7 Play' => 'moto-g7-play', - 'Moto G7 Plus' => 'moto-g7-plus', - 'Moto G7 Power' => 'moto-g7-power', - 'Moto M' => 'moto-m', - 'Motorola' => 'motorola', - 'Moto Z2' => 'moto-z2', - 'Moto Z2 Force' => 'moto-z2-force', - 'Moto Z2 Play' => 'moto-z2-play', - 'Moto Z3' => 'moto-z3', - 'Moto Z3 Play' => 'moto-z3-play', - 'Moulinex' => 'moulinex', - 'Mousses à raser' => 'mousses-a-raser', - 'MSI' => 'msi', - 'Musées' => 'musees', - 'Musique' => 'musique', - 'NAS' => 'nas', - 'Natation' => 'natation', - 'Nature & sports d'hiver' => 'nature-sports-hiver', - 'Navigation' => 'navigation', - 'NBA 2K' => 'nba-2k', - 'NBA 2K20' => 'nba-2k20', - 'NERF' => 'nerf', - 'Nescafé' => 'nescafe', - 'Nespresso' => 'nespresso', - 'Nest Learning Thermostat' => 'nest-learning-thermostat', - 'Nest Protect' => 'nest-protect', - 'Netflix' => 'netflix', - 'Nettoyeurs haute-pression' => 'nettoyeurs-haute-pression', - 'Nettoyeurs haute pression Karcher' => 'nettoyeurs-haute-pression-karcher', - 'Nettoyeurs vapeur' => 'nettoyeurs-vapeur', - 'New Balance' => 'new-balance', - 'New Balance 574' => 'new-balance-574', - 'NHL 20' => 'nhl-20', - 'Nike' => 'nike', - 'Nike Air Force' => 'nike-air-force', - 'Nike Air Jordan' => 'nike-air-jordan', - 'Nike Air Max' => 'nike-air-max', - 'Nike Air Max 90' => 'nike-air-max-90', - 'Nike Air Max 200' => 'nike-air-max-200', - 'Nike Air Max 270' => 'nike-air-max-270', - 'Nike Air Max 720' => 'nike-air-max-720', - 'Nike Free' => 'nike-free', - 'Nike Huarache' => 'nike-huarache', - 'Nike Roshe Run' => 'nike-roshe-run', - 'Nikon' => 'nikon', - 'Nikon D3500' => 'nikon-d3500', - 'Ni no Kuni' => 'ni-no-kuni', - 'Ni No Kuni: Wrath of the White Witch' => 'ni-no-kuni-wrath-white-witch', - 'Ni No Kuni II: Revenant Kingdom' => 'ni-no-kuni-2-revenant-kingdom', - 'Nintendo' => 'nintendo', - 'Nioh' => 'nioh', - 'Nivea' => 'nivea', - 'Nocciolata' => 'nocciolata', - 'Nokia' => 'nokia', - 'Nokia 5' => 'nokia-5', - 'Nokia 6' => 'nokia-6', - 'Nokia 8' => 'nokia-8', - 'Nokia 9 PureView' => 'nokia-9-pureview', - 'Nougats' => 'nougats', - 'Nourriture pour chat' => 'nourriture-pour-chat', - 'Nourriture pour chien' => 'nourriture-pour-chien', - 'Nourriture pour poissons' => 'nourriture-pour-poissons', - 'Nutella' => 'nutella', - 'Nvidia' => 'nvidia', - 'Nvidia GeForce' => 'nvidia-geforce', - 'Nvidia Shield' => 'nvidia-shield', - 'Objectifs' => 'objectifs', - 'Objets connectés' => 'objets-connectes', - 'Oculus Go' => 'oculus-go', - 'Oculus Rift' => 'oculus-rift', - 'Oiseaux' => 'oiseaux', - 'One Piece: Pirate Warriors' => 'one-piece-pirate-warriors', - 'OnePlus 5' => 'oneplus-5', - 'OnePlus 5T' => 'oneplus-5t', - 'OnePlus 6' => 'oneplus-6', - 'OnePlus 6T' => 'oneplus-6t', - 'OnePlus 7' => 'oneplus-7', - 'OnePlus 7 Pro' => 'oneplus-7-pro', - 'OnePlus 7T' => 'oneplus-7t', - 'OnePlus 7T Pro' => 'oneplus-7t-pro', - 'OnePlus 8' => 'oneplus-8', - 'OnePlus 8 Pro' => 'oneplus-8-pro', - 'OnePlus 8T' => 'oneplus-8t', - 'OnePlus 9' => 'oneplus-9', - 'OnePlus 9 Pro' => 'oneplus-9-pro', - 'OnePlus Nord' => 'oneplus-nord', - 'Onkyo' => 'onkyo', - 'Oppo Find X2 Lite' => 'oppo-find-x2-lite', - 'Oppo Find X2 Neo' => 'oppo-find-x2-neo', - 'Oppo Find X2 Pro' => 'oppo-find-x2-pro', - 'Oppo Reno' => 'oppo-reno', - 'Optique' => 'optique', - 'Oral-B' => 'oral-b', - 'Ordinateurs de bureau' => 'ordinateurs-de-bureau', - 'Ordinateurs tout-en-un' => 'pc-de-bureau-complets', - 'Oreillers' => 'oreillers', - 'Osram Smart+' => 'osram-smart-plus', - 'Outillage' => 'outillage', - 'Outils à main' => 'outils-main', - 'Outils de jardinage' => 'outils-de-jardinage', - 'Outils électriques' => 'outils-electriques', - 'Overwatch' => 'overwatch', - 'Packs clavier-souris' => 'packs-clavier-souris', - 'Packs consoles' => 'packs-consoles', - 'Paco Rabanne Invictus' => 'paco-rabanne-invictus', - 'Paco Rabanne Lady Million' => 'paco-rabanne-lady-million', - 'Paco Rabanne One Million' => 'paco-rabanne-one-million', - 'Pain & pâtisseries' => 'pain-patisseries', - 'Pampers' => 'pampers', - 'Panasonic' => 'panasonic', - 'Panasonic Lumix' => 'panasonic-lumix', - 'Panier Plus' => 'panier-plus', - 'Pantalons' => 'pantalons', - 'Papeterie' => 'papeterie', - 'Papeterie et bureautique' => 'papeterie-bureautique', - 'Papier bureautique' => 'papier-bureautique', - 'Papier peint' => 'papier-peint', - 'Papier toilette' => 'papier-toilette', - 'Parapharmacie' => 'parapharmacie', - 'Parasols' => 'parasols', - 'Parc Astérix' => 'parc-asterix', - 'Parcs d'attraction' => 'parcs-d-attraction', - 'Parfums' => 'parfums', - 'Parfums femme' => 'parfums-femme', - 'Parfums homme' => 'parfums-homme', - 'Parkas' => 'parkas', - 'Parrot' => 'parrot', - 'Partitions' => 'partitions', - 'Pâtée pour chat' => 'patee-pour-chat', - 'Pâtée pour chien' => 'patee-pour-chien', - 'Pâtes à tartiner' => 'pates-tartiner', - 'Pâtisserie' => 'patisserie', - 'PC Barebones' => 'pc-barebones', - 'PC gamer fixe' => 'pc-gamer-complets', - 'PC gaming' => 'pc-gaming', - 'PC hybrides' => 'hybrides', - 'PC Microsoft Surface' => 'pc-microsoft-surface', - 'PC portables' => 'pc-portables', - 'PC portables Acer' => 'pc-portables-acer', - 'PC portables ASUS' => 'pc-portables-asus', - 'PC portables Dell' => 'pc-portables-dell', - 'PC portables gaming' => 'portables-gamer', - 'PC portables Honor' => 'pc-portables-honor', - 'PC portables HP' => 'pc-portables-hp', - 'PC portables Lenovo' => 'pc-portables-lenovo', - 'PC portables Lenovo Legion' => 'lenovo-legion', - 'PC portables Xiaomi' => 'pc-portables-xiaomi', - 'Pêche' => 'peche', - 'Peignes & brosses à cheveux' => 'peignes-et-brosses-a-cheveux', - 'Peignoirs' => 'peignoirs', - 'Peintures' => 'peintures', - 'Peluches' => 'peluches', - 'Perceuses' => 'perceuses', - 'Périphériques PC' => 'peripheriques-pc', - 'Persona 5' => 'persona-5', - 'Persona 5 Royal' => 'persona-5-royal', - 'PES' => 'pro-evolution-soccer', - 'Pèse-personnes' => 'pese-personnes', - 'Petites voitures' => 'petites-voitures', - 'Pharmacie & parapharmacie' => 'pharmacie-parapharmacie', - 'Philips' => 'philips', - 'Philips Hue' => 'philips-hue', - 'Philips Hue E14' => 'philips-hue-e14', - 'Philips Hue E27' => 'philips-hue-e27', - 'Philips Hue Go' => 'philips-hue-go', - 'Philips Hue GU10' => 'philips-hue-gu10', - 'Philips Hue LightStrip' => 'philips-hue-lightstrip', - 'Philips Hue Play HDMI Sync Box' => 'philips-hue-play-hdmi-sync-box', - 'Philips Lumea' => 'philips-lumea', - 'Philips OneBlade' => 'philips-one-blade', - 'Philips Sonicare' => 'philips-sonicare', - 'Photo' => 'photo', - 'Pièces auto' => 'pieces-auto', - 'Pièces moto' => 'pieces-moto', - 'Pièces vélo' => 'pieces-velo', - 'Piles' => 'piles', - 'Piles rechargeables' => 'piles-rechargeables', - 'Pinceaux maquillage' => 'pinceaux-maquillage', - 'Pinces' => 'pinces', - 'Ping-pong' => 'ping-pong', - 'Pioneer' => 'pioneer', - 'Piscines' => 'piscines', - 'Pizza' => 'pizza', - 'Places de cinéma' => 'places-de-cinema', - 'Plafonniers' => 'plafonniers', - 'Plancha' => 'planchas', - 'Plantes & semis' => 'plantes', - 'Plaques de cuisson' => 'plaques-de-cuisson', - 'Platines vinyle' => 'platines-vinyle', - 'Plats & moules' => 'plats-et-moules', - 'PlayerUnknown's Battlegrounds' => 'playerunknown-s-battleground', - 'Playmobil' => 'playmobil', - 'PlayStation' => 'playstation', - 'Pneus' => 'pneus', - 'PocketBook' => 'pocketbook', - 'PocketBook Touch Lux 3' => 'pocketbook-touch-lux-3', - 'POCO F2 Pro' => 'poco-f2-pro', - 'POCO F3' => 'poco-f3', - 'POCO M3' => 'poco-m3', - 'POCO X3' => 'poco-x3', - 'POCO X3 Pro' => 'poco-x3-pro', - 'Poêles' => 'poeles', - 'Pokémon' => 'pokemon', - 'Pokémon: Let's Go' => 'pokemon-letsgo', - 'Pokémon Épée et Bouclier' => 'pokemon-epee-bouclier', - 'Pokémon Tournament' => 'pokemon-tournament', - 'Pokémon Ultra Sun / Moon' => 'pokemon-ultra-sun-moon', - 'Polaroid' => 'polaroid', - 'Polos' => 'polos', - 'Pompes à vélo' => 'pompes-velo', - 'Porte-bébé' => 'porte-bebe', - 'Portefeuilles' => 'portefeuilles', - 'Posters' => 'posters', - 'Potager' => 'potager', - 'Pots & cache-pots' => 'pots-et-cache-pots', - 'Poubelles' => 'poubelles', - 'Poulaillers' => 'poulaillers', - 'Poupées' => 'poupees', - 'Poussettes' => 'poussettes-bebe', - 'Présentez-vous !' => 'mieux-se-connaitre-presentez-vous', - 'Préservatifs' => 'preservatifs', - 'Princesse Tam-Tam' => 'princesse-tam-tam', - 'Prises connectées' => 'prises-connectees', - 'Processeurs' => 'processeurs', - 'Produit pour lentilles' => 'produit-pour-lentilles', - 'Produits de massage' => 'produits-de-massage', - 'Produits frais' => 'produits-frais', - 'Produits reconditionnés' => 'reconditionne', - 'Produits vétérinaires' => 'produits-veterinaires', - 'Programme d'Entraînement Cérébral du Dr. Kawashima' => 'dr-kawashima-brain-training', - 'Project Cars 2' => 'project-cars-2', - 'Protection de la maison' => 'protection-de-la-maison', - 'Protections intimes' => 'protections-intimes', - 'Protection solaire' => 'protection-solaire', - 'Puériculture' => 'puericulture', - 'Pulls' => 'pulls', - 'Puma' => 'puma', - 'Purificateurs d'air' => 'purificateurs-d-air', - 'Purina' => 'purina', - 'Puzzles' => 'puzzles', - 'Pyjamas' => 'pyjamas', - 'Pyjamas & chemises de nuit' => 'pyjamas-chemises-de-nuit', - 'Pyjamas pour bébés' => 'pyjamas-pour-bebes', - 'Qobuz' => 'qobuz', - 'Quiksilver' => 'quiksilver', - 'Radiateurs' => 'radiateurs', - 'Ralph Lauren' => 'ralph-lauren', - 'RAM' => 'ram', - 'Randonnée' => 'randonnee', - 'Raquettes de ping-pong' => 'raquettes-de-ping-pong', - 'Raquettes de tennis' => 'raquettes-de-tennis', - 'Rasage et épilation' => 'rasage-epilation', - 'Rasoirs Braun' => 'rasoirs-braun', - 'Rasoirs électriques' => 'rasoirs-electriques', - 'Rasoirs Gillette' => 'gillette', - 'Rasoirs manuels' => 'rasoirs-manuels', - 'Rasoirs Philips' => 'rasoirs-philips', - 'Rasoirs Wilkinson' => 'rasoirs-wilkinson-sword', - 'Raspberry Pi' => 'raspberry-pi', - 'Ray-Ban' => 'ray-ban', - 'Razer' => 'razer', - 'Razer DeathAdder' => 'razer-deathadder', - 'Realme 5 Pro' => 'realme-5-pro', - 'Realme X2 Pro' => 'realme-x2-pro', - 'Red Dead Redemption' => 'red-dead-redemption', - 'Red Dead Redemption 2' => 'red-dead-redemption-2', - 'Réductions étudiants & jeunes' => 'reductions-etudiants-et-jeunes', - 'Reebok' => 'reebok', - 'Reebok Club C' => 'reebok-club-c', - 'Réfrigérateurs' => 'refrigerateurs', - 'Réfrigérateurs américains' => 'refrigerateurs-americains', - 'Refroidissement PC' => 'refroidissement-pc', - 'Réhausseurs' => 'rehausseurs', - 'Remington' => 'remington', - 'Repas de fête' => 'repas-fete-reveillon', - 'Repassage' => 'repassage', - 'Répéteurs' => 'repeteurs', - 'Réseau' => 'reseau', - 'Resident Evil' => 'resident-evil', - 'Resident Evil 3' => 'resident-evil-3', - 'Resident Evil 7' => 'resident-evil-7', - 'Restaurants' => 'restaurants', - 'Revêtements de sols' => 'revetements-de-sols', - 'Revêtements muraux' => 'revetements-muraux', - 'Rhum' => 'rhum', - 'Richelieus' => 'richelieus', - 'Ring Fit Adventure' => 'ring-fit-adventure', - 'Risk' => 'risk', - 'Robes & jupes' => 'robes-et-jupes', - 'Roborock' => 'roborock', - 'Roborock S5 MAX' => 'roborock-s5-max', - 'Roborock S6' => 'roborock-s6', - 'Robots cuiseurs' => 'robots-cuiseurs', - 'Robots ménagers' => 'robots-menagers', - 'Robot tondeuse' => 'robot-tondeuse', - 'ROCCAT' => 'roccat', - 'Rollers' => 'rollers', - 'Rouges à lèvres' => 'rouges-a-levres', - 'Routeurs' => 'routeurs', - 'Rowenta' => 'rowenta', - 'Royal Canin' => 'royal-canin', - 'RTX 2060' => 'rtx-2060', - 'RTX 2070' => 'rtx-2070', - 'RTX 2080' => 'rtx-2080', - 'RTX 2080 Ti' => 'rtx-2080-ti', - 'RTX 3070' => 'rtx-3070', - 'RTX 3080' => 'rtx-3080', - 'RTX 3090' => 'rtx-3090', - 'RX 480' => 'rx-480', - 'RX 580' => 'rx-580', - 'RX 590' => 'radeon-rx-590', - 'RX Vega 56' => 'rx-vega-56', - 'RX Vega 64' => 'rx-vega-64', - 'Sacs à déjections' => 'sacs-a-dejections', - 'Sacs à dos' => 'sacs-a-dos', - 'Sacs à langer' => 'sacs-a-langer', - 'Sacs à main' => 'sacs-a-main', - 'Sacs bandoulière' => 'sacs-bandouliere', - 'Sacs de couchage' => 'sacs-de-couchage', - 'Sacs de randonnée' => 'sacs-de-randonnee', - 'Sacs de sport' => 'sacs-de-sport', - 'Sacs de voyage' => 'sacs-de-voyage', - 'Salle à manger' => 'salle-manger', - 'Samsonite' => 'samsonite', - 'Samsung' => 'samsung', - 'Samsung Galaxy A5' => 'samsung-galaxy-a5', - 'Samsung Galaxy A50' => 'samsung-galaxy-a50', - 'Samsung Galaxy A51' => 'samsung-galaxy-a51', - 'Samsung Galaxy A51 5G' => 'samsung-galaxy-a51-5g', - 'Samsung Galaxy A70' => 'samsung-galaxy-a70', - 'Samsung Galaxy A80' => 'samsung-galaxy-a80', - 'Samsung Galaxy Buds' => 'samsung-galaxy-buds', - 'Samsung Galaxy Buds+' => 'samsung-galaxy-buds-plus', - 'Samsung Galaxy Buds Live' => 'samsung-galaxy-buds-live', - 'Samsung Galaxy Buds Pro' => 'samsung-galaxy-buds-pro', - 'Samsung Galaxy Fold' => 'samsung-galaxy-fold', - 'Samsung Galaxy Note 8' => 'samsung-galaxy-note-8', - 'Samsung Galaxy Note 9' => 'samsung-galaxy-note-9', - 'Samsung Galaxy Note 10' => 'samsung-galaxy-note-10', - 'Samsung Galaxy Note 10 Lite' => 'samsung-galaxy-note-10-lite', - 'Samsung Galaxy Note 10 Plus' => 'samsung-galaxy-note-10-plus', - 'Samsung Galaxy Note20' => 'samsung-galaxy-note-20', - 'Samsung Galaxy Note20 Ultra' => 'samsung-galaxy-note-20-ultra', - 'Samsung Galaxy S7' => 'samsung-galaxy-s7', - 'Samsung Galaxy S7 Edge' => 'samsung-galaxy-s7-edge', - 'Samsung Galaxy S8' => 'samsung-galaxy-s8', - 'Samsung Galaxy S8+' => 'samsung-galaxy-s8plus', - 'Samsung Galaxy S9' => 'samsung-galaxy-s9', - 'Samsung Galaxy S9 Plus' => 'samsung-galaxy-s9-plus', - 'Samsung Galaxy S10' => 'samsung-galaxy-s10', - 'Samsung Galaxy S10 Lite' => 'samsung-galaxy-s10-lite', - 'Samsung Galaxy S10+' => 'samsung-galaxy-s10-plus', - 'Samsung Galaxy S10e' => 'samsung-galaxy-s10e', - 'Samsung Galaxy S20' => 'samsung-galaxy-s20', - 'Samsung Galaxy S20 FE' => 'samsung-galaxy-s20-fe', - 'Samsung Galaxy S20 Ultra' => 'samsung-galaxy-s20-ultra', - 'Samsung Galaxy S20+' => 'samsung-galaxy-s20-plus', - 'Samsung Galaxy S21 5G' => 'samsung-galaxy-s21-5g', - 'Samsung Galaxy S21 Ultra 5G' => 'samsung-galaxy-s21-ultra-5g', - 'Samsung Galaxy S21+ 5G' => 'samsung-galaxy-s21-plus-5g', - 'Samsung Galaxy Tab A' => 'samsung-galaxy-tab-a', - 'Samsung Galaxy Tab S2' => 'samsung-galaxy-tab-s2', - 'Samsung Galaxy Tab S3' => 'samsung-galaxy-tab-s3', - 'Samsung Galaxy Tab S4' => 'samsung-galaxy-tab-s4', - 'Samsung Galaxy Tab S5e' => 'samsung-galaxy-tab-s5e', - 'Samsung Galaxy Tab S6' => 'samsung-galaxy-tab-s6', - 'Samsung Galaxy Tab S7' => 'samsung-galaxy-tab-s7', - 'Samsung Galaxy Watch' => 'samsung-galaxy-watch', - 'Samsung Galaxy Watch3' => 'samsung-galaxy-watch-3', - 'Samsung Galaxy Watch Active 2' => 'samsung-galaxy-watch-active2', - 'Samsung Galaxy Z Flip' => 'galaxy-z-flip', - 'Samsung Gear' => 'samsung-gear', - 'Samsung Gear S3' => 'samsung-gear-s3', - 'Samsung Gear VR' => 'samsung-gear-vr', - 'Sandales' => 'sandales', - 'SanDisk' => 'sandisk', - 'Sanitaires et robinetterie' => 'sanitaires-robinetterie', - 'Santé & Cosmétiques' => 'sante-et-cosmetiques', - 'Sapins de Noël' => 'sapins-noel', - 'Savons' => 'savons', - 'Scanners' => 'scanners', - 'Scanners A3' => 'scanners-a3', - 'Scanners A4' => 'scanners-a4', - 'Scies' => 'scies', - 'Scooters' => 'scooters', - 'Seagate' => 'seagate', - 'Sécateurs' => 'secateurs', - 'Sèche-cheveux' => 'seche-cheveux', - 'Sèche-linge' => 'seche-linge', - 'Seiko' => 'seiko', - 'Séjours' => 'sejours', - 'Sekiro: Shadows Die Twice' => 'sekiro', - 'Semis & graines' => 'semis-et-graines', - 'Sennheiser' => 'sennheiser', - 'Senseo' => 'senseo', - 'Séries TV' => 'series-tv', - 'Service & réparation auto-moto' => 'service-reparation-auto-moto', - 'Services' => 'services-divers', - 'Services auto' => 'services-auto', - 'Services de livraison' => 'services-livraisons', - 'Services moto' => 'services-moto', - 'Services photo' => 'services-photo', - 'Serviettes' => 'serviettes', - 'Serviettes hygiéniques' => 'serviettes-hygieniques', - 'Sextoys' => 'sextoys', - 'Shadow of the Colossus' => 'shadow-of-the-colossus', - 'Shadow of the Tomb Raider' => 'shadow-tomb-raider', - 'Shalimar' => 'shalimar', - 'Shampooings & soins' => 'shampooings-et-soins', - 'Shenmue' => 'shenmue', - 'Shenmue I & II' => 'shenmue-i-ii', - 'Shenmue III' => 'shenmue-iii', - 'Shorts' => 'shorts', - 'Shorts de bain' => 'shorts-de-bain', - 'Sièges auto' => 'sieges-auto', - 'Siemens' => 'siemens', - 'Skates & longboards' => 'skates-et-longboards', - 'Skechers' => 'sketchers', - 'Ski' => 'ski', - 'Skyrim' => 'skyrim', - 'Slips & boxers' => 'slips-et-boxers', - 'Smartphones' => 'smartphones', - 'Smartphones à moins de 100€' => 'smartphones-moins-de-100', - 'Smartphones à moins de 200€' => 'smartphones-moins-de-200', - 'Smartphones Android' => 'smartphones-android', - 'Smartphones Asus' => 'smartphones-asus', - 'Smartphones Google' => 'smartphones-google', - 'Smartphones Honor' => 'smartphones-honor', - 'Smartphones HTC' => 'smartphones-htc', - 'Smartphones Huawei' => 'smartphones-huawei', - 'Smartphones Lenovo Motorola' => 'smartphones-lenovo-motorola', - 'Smartphones LG' => 'smartphones-lg', - 'Smartphones Nokia' => 'smartphones-nokia', - 'Smartphones OnePlus' => 'smartphones-oneplus', - 'Smartphones Oppo' => 'smartphones-oppo', - 'Smartphones Realme' => 'smartphones-realme', - 'Smartphones Samsung' => 'smartphones-samsung', - 'Smartphones Sony' => 'smartphones-sony', - 'Smartphones Xiaomi' => 'smartphones-xiaomi', - 'Smartphones ZTE' => 'smartphones-zte', - 'Smart TV' => 'smart-tv', - 'Sneakers' => 'sneakers', - 'SodaStream' => 'sodastream', - 'Sofas gonflable' => 'sofas-gonflable', - 'Soin barbe et rasage' => 'soin-barbe-rasage', - 'Soin de la peau' => 'soin-peau', - 'Soin des cheveux' => 'soin-des-cheveux', - 'Soin des ongles' => 'soin-ongles', - 'Soins dentaires' => 'soins-dentaires', - 'Sonos' => 'sonos', - 'Sonos Beam' => 'sonos-beam', - 'Sonos Move' => 'sonos-move', - 'Sonos One' => 'sonos-one', - 'Sonos PLAY:1' => 'sonos-play-1', - 'Sonos PLAY:3' => 'sonos-play-3', - 'Sonos PLAY:5' => 'sonos-play-5', - 'Sonos PLAYBAR' => 'sonos-playbar', - 'Sony' => 'sony', - 'Sony PlayStation VR' => 'sony-playstation-vr', - 'Sony Pulse 3D sans fil' => 'casque-audio-sony-pulse-3d', - 'Sony WF-1000XM3' => 'sony-wf-1000xm3', - 'Sony WH-1000XM3' => 'sony-wh-1000xm3', - 'Sony WH-1000XM4' => 'sony-wh-1000xm4', - 'Sony Xperia XA1' => 'sony-xperia-xa1', - 'Sony Xperia X Compact' => 'sony-xperia-x-compact', - 'Sony Xperia XZ1' => 'sony-xperia-xz1', - 'Sony Xperia XZ1 Compact' => 'sony-xperia-xz1-compact', - 'Sony Xperia XZ Premium' => 'sony-xperia-xz-premium', - 'Sony Xperia Z3' => 'sony-xperia-z3', - 'Soulcalibur' => 'soulcalibur', - 'Souris' => 'souris', - 'Souris gamer' => 'souris-gamer', - 'Souris Logitech' => 'souris-logitech', - 'Souris sans fil' => 'souris-sans-fil', - 'Sous-vêtements' => 'sous-vetements', - 'Sous-vêtements de sport' => 'sous-vetements-de-sport', - 'South Park' => 'south-park', - 'Soutiens-gorge' => 'soutiens-gorge', - 'Spas' => 'spa', - 'Spectacles' => 'spectacles', - 'Spectacles & Billetterie' => 'sorties', - 'Spectacles comiques' => 'spectacles-comiques', - 'Spectacles pour enfants' => 'spectacles-pour-enfants', - 'Sports & plein air' => 'sports-plein-air', - 'Sports collectifs' => 'sports-collectifs', - 'Sports nautiques' => 'sports-nautiques', - 'Sportswear' => 'sportswear', - 'Spotify' => 'spotify', - 'SSD' => 'ssd', - 'Star Wars: Jedi Fallen Order' => 'star-wars-jedi-fallen-order', - 'Star Wars: Squadrons' => 'star-wars-squadrons', - 'Star Wars Battlefront' => 'star-wars-battlefront', - 'Stations météo' => 'stations-meteo', - 'Stickers muraux' => 'stickers-muraux', - 'Stihl' => 'stihl', - 'Stockage externe' => 'stockage', - 'Streaming' => 'streaming', - 'Streaming musical' => 'streaming-musical', - 'Streaming vidéo' => 'streaming-video', - 'Stylos' => 'stylos', - 'Sucettes' => 'sucettes', - 'Super Mario' => 'super-mario', - 'Super Mario 3D All-Stars' => 'super-mario-3d-all-stars', - 'Super Mario Maker 2' => 'super-mario-maker-2', - 'Super Mario Party' => 'super-mario-party', - 'Super Smash Bros. Ultimate' => 'super-smash-bros-ultimate', - 'Support GPS & smartphone' => 'support-gps-et-smartphone', - 'Supports TV' => 'supports-tv', - 'Surface Pro 4' => 'surface-pro-4', - 'Surgelés' => 'surgeles', - 'Surveillance' => 'surveillance', - 'Suspensions' => 'suspensions', - 'Swatch' => 'swatch', - 'Switch réseau' => 'switch-reseau', - 'Systèmes d'exploitation' => 'systemes-d-exploitation', - 'Systèmes multiroom' => 'systemes-multiroom', - 'T-shirts' => 't-shirts', - 'Tables' => 'tables', - 'Tables à langer' => 'tables-a-langer', - 'Tables à repasser' => 'tables-a-repasser', - 'Tables basses' => 'tables-basses', - 'Tables de camping' => 'tables-de-camping', - 'Tables de mixage' => 'tables-de-mixage', - 'Tables de ping-pong' => 'tables-ping-pong', - 'Tablettes' => 'tablettes', - 'Tablettes graphiques' => 'tablettes-graphiques', - 'Tablettes graphiques Huion' => 'huion', - 'Tablettes graphiques Wacom' => 'wacom', - 'Tablettes Huawei' => 'tablettes-huawei', - 'Tablettes Lenovo' => 'tablettes-lenovo', - 'Tablettes Microsoft Surface' => 'tablettes-microsoft-surface', - 'Tablettes Samsung' => 'tablettes-samsung', - 'Tablettes Xiaomi' => 'tablettes-xiaomi', - 'Tampons' => 'tampons', - 'Tapis' => 'tapis', - 'Tapis de souris' => 'tapis-de-souris', - 'Tassimo' => 'tassimo', - 'Taxis' => 'taxis', - 'Tefal' => 'tefal', - 'Tekken' => 'tekken', - 'Tekken 7' => 'tekken-7', - 'Télécommandes' => 'telecommandes', - 'Téléphones fixes' => 'telephones-fixes', - 'Téléphonie' => 'telephonie', - 'Téléviseurs' => 'televiseurs', - 'Tentes' => 'tentes', - 'Tentes Quechua' => 'tentes-quechua', - 'Têtes de brosse à dents de rechange' => 'tetes-de-brosse-a-dents-de-rechange', - 'Théâtre' => 'theatre', - 'The Last of Us' => 'the-last-of-us', - 'The Last of Us Part II' => 'the-last-of-us-part-2', - 'The Legend of Zelda' => 'the-legend-of-zelda', - 'The Legend of Zelda: Breath of the Wild' => 'zelda-breath-of-the-wild', - 'The Legend of Zelda: Link's Awakening' => 'legend-of-zelda-link-s-awakening', - 'The Legend of Zelda: Skyward Sword HD' => 'the-legend-of-zelda-skyward-sword-hd', - 'Thermomètres' => 'thermometres', - 'Thermomix' => 'thermomix', - 'Thermostats connectés' => 'thermostat-connecte', - 'Thés' => 'thes', - 'Thés glacés' => 'thes-glaces', - 'The Walking dead' => 'the-walking-dead', - 'The Witcher' => 'the-witcher', - 'The Witcher 3' => 'the-witcher-3', - 'Time's Up!' => 'time-s-up', - 'Tokyo Laundry' => 'tokyo-laundry', - 'Tomb Raider' => 'tomb-raider', - 'Tom Clancy's' => 'tom-clancy-s', - 'Tom Clancy's Ghost Recon: Wildlands' => 'tom-clancy-s-ghost-recon-wildlands', - 'Tom Clancy's Ghost Recon Breakpoint' => 'tom-clancy-s-ghost-recon-breakpoint', - 'Tom Clancy's The Division' => 'tom-clancy-s-the-division', - 'TomTom' => 'tomtom', - 'Tondeuses' => 'tondeuses', - 'Tondeuses à gazon' => 'tondeuses-a-gazon', - 'Toner' => 'toner', - 'Tongs' => 'tongs', - 'Torchons' => 'torchons', - 'Toshiba' => 'toshiba', - 'Total War' => 'total-war', - 'Total War: Warhammer' => 'total-war-warhammer', - 'Total War: Warhammer II' => 'total-war-warhammer-ii', - 'Tournevis' => 'tournevis-et-visseuses', - 'TP-Link' => 'tp-link', - 'Trains & Bus' => 'trains-bus', - 'Trampolines' => 'trampolines', - 'Transats & cosys' => 'transats-et-cosys', - 'Transport bébé' => 'poussettes', - 'Transport d'animaux' => 'transport-d-animaux', - 'Transports en commun' => 'transports-en-commun', - 'Transports urbains' => 'transports-urbains', - 'Travaux & matériaux' => 'travaux-materiaux', - 'Trépieds' => 'trepieds', - 'Trixie' => 'trixie', - 'Tronçonneuses' => 'tronconneuses', - 'Tropico' => 'tropico', - 'Tropico 6' => 'tropico-6', - 'Trottinettes' => 'trottinettes', - 'Trottinettes électriques' => 'trottinettes-electriques', - 'Trottinettes électriques en libre-service' => 'location-trottinettes-electriques', - 'Trottinettes Xiaomi' => 'trottinettes-xiaomi', - 'TV & Vidéo' => 'tv-video', - 'TV 4K' => 'tv-4k', - 'TV 40'' à 64''' => 'tv-40-pouces-a-64-pouces', - 'TV 65'' et plus' => 'tv-65-pouces-et-plus', - 'TV Hisense' => 'tv-hisense', - 'TV LG' => 'tv-lg', - 'TV OLED' => 'tv-oled', - 'TV Panasonic' => 'tv-panasonic', - 'TV Philips' => 'tv-philips', - 'TV Samsung' => 'tv-samsung', - 'TV Samsung QLED' => 'tv-samsung-qled', - 'TV Samsung The Frame' => 'tv-samsung-the-frame', - 'TV Sony' => 'tv-sony', - 'TV TCL' => 'tv-tcl', - 'TV Toshiba' => 'tv-toshiba', - 'TV Xiaomi' => 'tv-xiaomi', - 'UE Boom 2' => 'ue-boom-2', - 'UE Boom 3' => 'ue-boom-3', - 'UE Megaboom' => 'ue-megaboom', - 'UE Megaboom 3' => 'ue-megaboom-3', - 'UE Wonderboom' => 'ue-wonderboom', - 'Ultraportables' => 'ultraportables', - 'Uncharted' => 'uncharted', - 'Uncharted 4' => 'uncharted-4', - 'Uncharted: The Lost Legacy' => 'uncharted-the-lost-legacy', - 'Under Armour' => 'under-armour', - 'Until Dawn' => 'until-dawn', - 'Ustensiles de cuisine' => 'ustensiles-de-cuisine', - 'Ustensiles de cuisson' => 'ustensiles-de-cuisson', - 'Vacances et séjours' => 'vacances-sejours', - 'Vaisselle' => 'vaisselle', - 'Valises' => 'valises', - 'Valises cabine' => 'valises-cabine', - 'Valises rigides' => 'valises-rigides', - 'Vans Old Skool' => 'vans-old-skool', - 'Variétés & revues' => 'varietes-et-revues', - 'Vases' => 'vases', - 'Veet' => 'veet', - 'Veilleuses' => 'veilleuses', - 'Vélos' => 'velos', - 'Vélos d'appartement' => 'velos-d-appartement', - 'Vélos électriques' => 'velos-electriques', - 'Ventilateurs' => 'ventilateurs', - 'Ventirad' => 'ventirad', - 'Vernis à ongles' => 'vernis-a-ongles', - 'Verres' => 'verres', - 'Vestes' => 'vestes', - 'Vestes polaires' => 'vestes-polaires', - 'Vêtements d'été' => 'vetements-d-ete', - 'Vêtements d'hiver' => 'vetements-d-hiver', - 'Vêtements de grossesse' => 'vetements-de-grossesse', - 'Vêtements de montagne' => 'vetements-techniques', - 'Vêtements de running' => 'vetements-de-running', - 'Vêtements de ski' => 'vetements-de-ski', - 'Vêtements de sport' => 'vetements-de-sport', - 'Vêtements pour bébé' => 'vetements-pour-bebe', - 'Vidéoprojecteurs' => 'projecteurs', - 'Vidéoprojecteurs 3D' => 'videoprojecteurs-3d', - 'Vidéoprojecteurs Acer' => 'videoprojecteurs-acer', - 'Vidéoprojecteurs BenQ' => 'videoprojecteurs-benq', - 'Vidéoprojecteurs Epson' => 'videoprojecteurs-epson', - 'Vidéoprojecteurs HD' => 'videoprojecteurs-hd', - 'Vidéoprojecteurs LG' => 'videoprojecteurs-lg', - 'Vidéoprojecteurs Optoma' => 'videoprojecteurs-optoma', - 'Vins' => 'vins', - 'Visites & patrimoine' => 'visites-et-patrimoine', - 'Visseuses' => 'visseuses', - 'VOD' => 'vod', - 'Voitures & motos' => 'voitures-motos', - 'Voitures télécommandées' => 'voitures-telecommandees', - 'Volants' => 'volants-de-course', - 'Vols' => 'billets-d-avion', - 'Voyages' => 'voyages', - 'Voyages & loisirs' => 'le-laboratoire-des-voyages-loisirs', - 'VPN' => 'vpn', - 'VTC' => 'vtc', - 'VTT' => 'vtt', - 'Wacom Cintiq' => 'cintiq', - 'Watch Dogs' => 'watch-dogs', - 'Watch Dogs 2' => 'watch-dogs-2', - 'Watch Dogs: Legion' => 'watch-dogs-legion', - 'Watercooling' => 'watercooling', - 'WD (Western Digital)' => 'western-digital', - 'Wearables' => 'wearables', - 'Webcams' => 'webcams', - 'Whey' => 'whey', - 'Whirlpool' => 'whirlpool', - 'Whiskas' => 'whiskas', - 'Whisky' => 'whisky', - 'Wiko' => 'wiko', - 'Wilkinson Sword Hydro 5' => 'wilkinson-sword-hydro-5', - 'Windows' => 'windows', - 'WindScribe' => 'windscribe', - 'Wolfenstein' => 'wolfenstein', - 'Wolfenstein II: The New Colossus' => 'wolfenstein-ii-the-new-colossus', - 'Xbox' => 'xbox', - 'Xbox Game Pass' => 'xbox-game-pass', - 'Xbox Live' => 'xbox-live', - 'XCOM' => 'xcom', - 'XCOM 2' => 'xcom-2', - 'Xiaomi' => 'xiaomi', - 'Xiaomi AirDots' => 'xiaomi-airdots', - 'Xiaomi Black Shark' => 'xiaomi-black-shark', - 'Xiaomi Black Shark 2' => 'xiaomi-black-shark-2', - 'Xiaomi Mi6' => 'xiaomi-mi6', - 'Xiaomi Mi8' => 'xiaomi-mi8', - 'Xiaomi Mi8 Lite' => 'xiaomi-mi8-lite', - 'Xiaomi Mi8 Pro' => 'xiaomi-mi8-pro', - 'Xiaomi Mi8 SE' => 'xoaimi-mi8-se', - 'Xiaomi Mi9' => 'xiaomi-mi9', - 'Xiaomi Mi 9 Lite' => 'xiaomi-mi-9-lite', - 'Xiaomi Mi 9 Pro' => 'xiaomi-mi-9-pro', - 'Xiaomi Mi 9 SE' => 'xiaomi-mi-9-se', - 'Xiaomi Mi 9T' => 'xiaomi-mi-9t', - 'Xiaomi Mi 9T Pro' => 'xiaomi-mi-9t-pro', - 'Xiaomi Mi 10' => 'xiaomi-mi-10', - 'Xiaomi Mi 10 Lite' => 'xiaomi-mi-10-lite', - 'Xiaomi Mi 10 Pro' => 'xiaomi-mi-10-pro', - 'Xiaomi Mi 10T' => 'xiaomi-mi-10t', - 'Xiaomi Mi 10T Lite' => 'xiaomi-mi-10t-lite', - 'Xiaomi Mi 10T Pro' => 'xiaomi-mi-10t-pro', - 'Xiaomi Mi 11' => 'xiaomi-mi-11', - 'Xiaomi Mi 11 Lite' => 'xiaomi-mi-11-lite', - 'Xiaomi Mi A1' => 'xiaomi-mi-a1', - 'Xiaomi Mi A2' => 'xiaomi-mi-a2', - 'Xiaomi Mi A2 Lite' => 'xiaomi-mi-a2-lite', - 'Xiaomi Mi Airdots Pro' => 'xiaomi-mi-airdots-pro', - 'Xiaomi Mi Band' => 'xiaomi-mi-band', - 'Xiaomi Mi Band 4' => 'xiaomi-mi-band-4', - 'Xiaomi Mi Band 5' => 'xiaomi-mi-band-5', - 'Xiaomi Mi Band 6' => 'xiaomi-mi-band-6', - 'Xiaomi Mi Box' => 'xiaomi-mi-box', - 'Xiaomi Mi Electric Scooter M365' => 'xiaomi-mi-electric-scooter-m365', - 'Xiaomi Mi Max' => 'xiaomi-mi-max', - 'Xiaomi Mi Mix' => 'xiaomi-mi-mix', - 'Xiaomi Mi Mix 2' => 'xiaomi-mi-mix-2', - 'Xiaomi Mi Note 10' => 'xiaomi-mi-note-10', - 'Xiaomi Mi Note 10 Pro' => 'xiaomi-mi-note-10-pro', - 'Xiaomi Mi Pad 3' => 'xiaomi-mi-pad-3', - 'Xiaomi Mi Watch' => 'xiaomi-mi-watch', - 'Xiaomi Pocophone F1' => 'xiaomi-pocophone-f1', - 'Xiaomi Redmi 4A' => 'xiaomi-redmi-4a', - 'Xiaomi Redmi 4X' => 'xiaomi-redmi-4x', - 'Xiaomi Redmi 7' => 'xiaomi-redmi-7', - 'Xiaomi Redmi 9' => 'xiaomi-redmi-9', - 'Xiaomi Redmi AirDots' => 'xiaomi-redmi-airdots', - 'Xiaomi Redmi Note 4' => 'xiaomi-redmi-note-4', - 'Xiaomi Redmi Note 5' => 'xiaomi-redmi-note-5', - 'Xiaomi Redmi Note 6' => 'xiaomi-redmi-note-6', - 'Xiaomi Redmi Note 7' => 'xiaomi-redmi-note-7', - 'Xiaomi Redmi Note 8' => 'xiaomi-redmi-note-8', - 'Xiaomi Redmi Note 8 Pro' => 'xiaomi-redmi-note-8-pro', - 'Xiaomi Redmi Note 9' => 'xiaomi-redmi-note-9', - 'Xiaomi Redmi Note 9 Pro' => 'xiaomi-redmi-note-9-pro', - 'Xiaomi Redmi Note 9S' => 'xiaomi-redmi-note-9s', - 'Xiaomi Redmi Note 10' => 'xiaomi-redmi-note-10', - 'Xiaomi Redmi Note 10 Pro' => 'xiaomi-redmi-10-pro', - 'Xiaomi Smart Home' => 'xiaomi-smart-home', - 'Yamaha' => 'yamaha', - 'Yeelight' => 'xiaomi-yeelight', - 'Yoshi's Crafted World' => 'yoshi-crafted-world', - 'Zoos' => 'zoos', - ] - ], + 'type' => 'text', + 'exampleValue' => 'abonnements-internet', + 'title' => 'Nom du groupe dans l\'URL : Il faut entrer le nom du groupe qui est présent après "https://www.dealabs.com/groupe/" et avant tout éventuel "?" +Exemple : Si l\'URL du groupe affichées dans le navigateur est : +https://www.dealabs.com/groupe/abonnements-internet?sortBy=lowest_price +Il faut alors saisir : +abonnements-internet', + ], 'order' => [ 'name' => 'Trier par', 'type' => 'list', @@ -1923,5 +100,7 @@ class DealabsBridge extends PepperBridgeAbstract 'title-talk' => 'Surveillance Discussion', 'deal-type' => 'Type de deal', 'localdeal' => 'Deal Local', + 'context-hot' => '-hot', + 'context-new' => '-nouveaux', ]; } diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index 6958220e..9d56fa97 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -40,3202 +40,13 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'Deals per group' => [ 'group' => [ 'name' => 'Group', - 'type' => 'list', - 'title' => 'Group whose deals must be displayed', - 'values' => [ - '3D Blu-ray' => '3d-bluray', - '3D Printer' => '3d-printer', - '3D TV' => '3d-tv', - '4K Blu-ray' => '4k-bluray', - '4K Monitor' => '4k-monitor', - '4K TV' => '4k-tv', - '5G Phones' => '5g-phones', - '7 Up' => '7up', - '8K TV' => '8k-tv', - '32 inch TV' => '32-inch-tv', - '40 inch TV' => '40-inch-tv', - '55 inch TV' => '55-inch-tv', - '65 inch TV' => '65-inch-tv', - '75 inch TV' => '75-inch-tv', - '144Hz Monitor' => '144hz', - 'A4 Paper' => 'a4-paper', - 'AAA Battery' => 'aaa', - 'AA Battery' => 'aa', - 'Abercrombie' => 'abercrombie', - 'Aberlour' => 'aberlour', - 'Accommodation' => 'accomodation', - 'Accurist' => 'accurist', - 'Ace Combat 7: Skies Unknown' => 'ace-combat-7', - 'Acer' => 'acer', - 'Acer Aspire' => 'acer-aspire', - 'Acer Laptop' => 'acer-laptop', - 'Acer PC Monitor' => 'acer-pc-monitor', - 'Acer Predator' => 'acer-predator', - 'Action Camera' => 'action-camera', - 'Action Figure & Playsets' => 'playsets', - 'Activewear' => 'sports-clothes', - 'Activia' => 'activia', - 'adidas' => 'adidas', - 'adidas Continental' => 'continental', - 'Adidas Gazelle' => 'gazelle', - 'Adidas Originals' => 'adidas-originals', - 'Adidas Samba' => 'samba', - 'Adidas Stan Smith' => 'stan-smith', - 'Adidas Superstar' => 'adidas-superstar', - 'Adidas Trainers' => 'adidas-shoes', - 'Adidas Ultraboost' => 'adidas-ultraboost', - 'Adidas ZX Flux' => 'adidas-zx-flux', - 'Adobe' => 'adobe', - 'Adobe Lightroom' => 'lightroom', - 'Adobe Photoshop' => 'photoshop', - 'Adult Products' => 'adult', - 'Advent Calendar' => 'advent-calendar', - 'Adventure Time' => 'adventure-time', - 'AEG' => 'aeg', - 'Aftershave' => 'aftershave', - 'Age Of Empires' => 'age-of-empires', - 'Air Bed' => 'air-bed', - 'Air Conditioner' => 'air-con', - 'Airer' => 'airer', - 'Airfix' => 'airfix', - 'Air Fryer' => 'air-fryer', - 'Airline' => 'airline', - 'Airport' => 'airport', - 'Airport Parking' => 'airport-parking', - 'Air Purifier' => 'air-purifier', - 'AirTag' => 'airtag', - 'Air Treatment' => 'air-treatment', - 'AKG' => 'akg', - 'Alarm Clock' => 'alarm-clock', - 'Alarm System' => 'alarm-system', - 'Alcatel' => 'alcatel', - 'Alcohol' => 'alcohol', - 'Alesis' => 'alesis', - 'Alien: Isolation' => 'alien-isolation', - 'Alienware' => 'alienware', - 'All-in-One PC' => 'all-in-one-pc', - 'All-in-One Printer' => 'all-in-one-printer', - 'Alloy Wheel' => 'alloy-wheels', - 'All Saints' => 'all-saints', - 'Almonds' => 'almonds', - 'Alpro' => 'alpro', - 'Alton Towers' => 'alton-towers', - 'Amazfit' => 'xiaomi-amazfit', - 'Amazfit Bip' => 'xiaomi-amazfit-bip', - 'Amazfit GTS' => 'amazfit-gts', - 'Amazfit Verge' => 'amazfit-verge', - 'Amazfit Verge Lite' => 'amazfit-verge-lite', - 'Amazfit Watch' => 'amazfit-watch', - 'Amazon Add On Item' => 'add-on-item', - 'Amazon Business' => 'amazon-business', - 'Amazon Echo' => 'amazon-echo', - 'Amazon Echo Dot' => 'amazon-echo-dot', - 'Amazon Echo Plus' => 'amazon-echo-plus', - 'Amazon Echo Show' => 'amazon-echo-show', - 'Amazon Echo Show 5' => 'echo-show-5', - 'Amazon Echo Show 8' => 'amazon-echo-show-8', - 'Amazon Echo Spot' => 'amazon-echo-spot', - 'Amazon Fire 7' => 'amazon-fire-7', - 'Amazon Fire HD 8' => 'amazon-fire-hd-7', - 'Amazon Fire HD 10 Tablet' => 'amazon-fire-hd-10', - 'Amazon Fire Tablet' => 'amazon-tablet', - 'Amazon Fire TV Cube' => 'fire-tv-cube', - 'Amazon Fire TV Stick' => 'amazon-fire-stick', - 'Amazon Pantry' => 'amazon-pantry', - 'Amazon Prime' => 'amazon-prime', - 'Amazon Prime Video' => 'amazon-video', - 'Amazon Warehouse' => 'amazon-warehouse', - 'AMD' => 'amd', - 'AMD Radeon' => 'radeon', - 'AMD Ryzen' => 'amd-ryzen', - 'AMD Ryzen 5 5600X' => 'amd-ryzen-5-5600x', - 'AMD Ryzen 7 5800X' => 'amd-ryzen-7-5800x', - 'AMD Ryzen 9 5900X' => 'amd-ryzen-9-5900x', - 'AMD Ryzen 9 5950X' => 'amd-ryzen-9-5950x', - 'Amex' => 'amex', - 'Amiibo' => 'amiibo', - 'Amplifier' => 'amplifier', - 'Anchor Butter' => 'anchor-butter', - 'Andrex' => 'andrex', - 'Android Apps' => 'android-app', - 'Android Smartphone' => 'android-smartphone', - 'Android Tablet' => 'android-tablet', - 'Angelcare' => 'angelcare', - 'Angle Grinder' => 'grinder', - 'Anglepoise' => 'anglepoise', - 'Angry Birds' => 'angry-birds', - 'Animal Crossing' => 'animal-crossing', - 'Anime' => 'anime', - 'Anker' => 'anker', - 'Ankle Boots' => 'ankle-boots', - 'Anno 1800' => 'anno-1800', - 'Anthem' => 'anthem', - 'Antibacterial Hand Gel' => 'hand-gel', - 'Antibacterial Wipes' => 'cleaning-wipes', - 'Antivirus' => 'antivirus', - 'Antler' => 'antler', - 'AOC' => 'aoc', - 'Apex Legends' => 'apex-legends', - 'A Plague Tale: Innocence' => 'a-plague-tale-innocence', - 'App' => 'app', - 'Apple' => 'apple', - 'Apple AirPods' => 'apple-airpods', - 'Apple Airpods 2' => 'airpods-2', - 'Apple Airpods Max' => 'airpods-max', - 'Apple Airpods Pro' => 'airpods-pro', - 'Apple EarPods' => 'earpods', - 'Apple Headphones' => 'apple-headphones', - 'Apple HomePod' => 'apple-homepod', - 'Apple HomePod mini' => 'apple-homepod-mini', - 'Apple Keyboard' => 'apple-keyboard', - 'Apple Pencil' => 'apple-pencil', - 'Apple TV' => 'apple-tv', - 'Apple TV 4K' => 'apple-tv-4k', - 'Apple Watch' => 'apple-watch', - 'Apple Watch 3' => 'apple-watch-3', - 'Apple Watch 4' => 'apple-watch-4', - 'Apple Watch 5' => 'apple-watch-5', - 'Apple Watch 6' => 'apple-watch-6', - 'Apple Watch SE' => 'apple-watch-se', - 'Apron' => 'apron', - 'Aquadoodle' => 'aquadoodle', - 'Aqua Optima' => 'aqua-optima', - 'Aquarium' => 'aquarium', - 'Aramis' => 'aramis', - 'Argan Oil' => 'argan-oil', - 'Ariel' => 'ariel', - 'Ark' => 'ark', - 'Armani' => 'armani', - 'Armchair' => 'armchair', - 'Armed Forces Discount' => 'armed-forces', - 'Arsenal F. C.' => 'arsenal', - 'Arts and Crafts' => 'craft', - 'Asics' => 'asics', - 'Ask' => 'ask', - 'ASRock' => 'asrock', - 'Assassin's Creed' => 'assassins-creed', - 'Assassin's Creed: Odyssey' => 'assassins-creed-odyssey', - 'Assassin's Creed: Origins' => 'assassins-creed-origins', - 'Assassin's Creed: Unity' => 'assassins-creed-unity', - 'Assassin's Creed: Valhalla' => 'assasins-creed-valhalla', - 'Astral Chain' => 'astral-chain', - 'ASTRO Gaming' => 'astro-gaming', - 'Astro Gaming A40' => 'astro-gaming-a40', - 'Astro Gaming A50' => 'astro-gaming-a50', - 'Asus' => 'asus', - 'ASUS Laptop' => 'asus-laptop', - 'ASUS Monitor' => 'asus-monitor', - 'ASUS ROG' => 'asus-rog', - 'Asus ROG Phone' => 'asus-rog-phone', - 'Asus ROG Phone 2' => 'asus-rog-phone-2', - 'ASUS Router' => 'asus-router', - 'Asus Smartphone' => 'asus-smartphone', - 'ASUS Vivobook' => 'asus-vivobook', - 'ASUS Zenbook' => 'zenbook', - 'Asus ZenFone 6' => 'asus-zenfone-6', - 'Atari' => 'atari', - 'Audi' => 'audi', - 'Audio & Hi-Fi' => 'audio', - 'Audio Accessories' => 'audio-accessories', - 'Audiobook' => 'audiobook', - 'Audio Technica' => 'audio-technica', - 'Aukey' => 'aukey', - 'Aussie' => 'aussie', - 'Autoglym' => 'autoglym', - 'Aveeno' => 'aveeno', - 'Avengers' => 'avengers', - 'AVG' => 'avg', - 'Aviva' => 'aviva', - 'Avon' => 'avon', - 'AV Receiver' => 'av-receiver', - 'Axe' => 'axe', - 'Baby Annabell' => 'baby-annabell', - 'Baby Bath' => 'baby-bath', - 'Baby Born' => 'baby-born', - 'Baby Bottle' => 'baby-bottles', - 'Baby Bouncer' => 'bouncer', - 'Baby Carrier' => 'baby-carrier', - 'Baby Clothes' => 'baby-clothes', - 'Baby Food' => 'baby-food', - 'Baby Gym' => 'baby-gym', - 'Baby Jogger' => 'baby-jogger', - 'Babyliss' => 'babyliss', - 'Baby Monitor' => 'baby-monitor', - 'Baby Shoes' => 'baby-shoes', - 'Baby Swing' => 'baby-swing', - 'Baby Walker' => 'baby-walker', - 'Baby Wipes' => 'wipes', - 'Bacardi' => 'bacardi', - 'Backpack' => 'backpack', - 'Back to the Future' => 'back-to-the-future', - 'Bacon' => 'bacon', - 'Badminton' => 'badminton', - 'Bag' => 'bag', - 'Bagless Vacuum Cleaner' => 'bagless-vacuum-cleaner', - 'Bahco' => 'bahco', - 'Baileys' => 'baileys', - 'Baked Beans' => 'baked-beans', - 'Bakery Products' => 'bakery-products', - 'Baking' => 'baking', - 'Ball Pit' => 'ball-pit', - 'Ballpoint Pen' => 'pen', - 'Band of Brothers' => 'band-of-brothers', - 'Bang & Olufsen' => 'bang-olufsen', - 'Bank' => 'bank', - 'Bank Account' => 'bank-account', - 'Banks & Credit Cards' => 'bank-credit-card', - 'Barbell' => 'barbell', - 'Barbie' => 'barbie', - 'Barbour' => 'barbour', - 'Barclaycard' => 'barclaycard', - 'Barclays' => 'barclays', - 'Barebones PC' => 'barebones', - 'bareMinerals' => 'bareminerals', - 'Barry M' => 'barry-m', - 'Bar Stools' => 'bar-stools', - 'Base Layer' => 'base-layer', - 'Basket' => 'basket', - 'Basketball' => 'basketball', - 'Basmati Rice' => 'basmati-rice', - 'Bath Mat' => 'bath-mat', - 'Bathroom Accessories' => 'bathroom', - 'Bathroom Cabinet' => 'bathroom-cabinet', - 'Bathroom Scale' => 'bathroom-scales', - 'Bathroom Tap' => 'tap', - 'Batman' => 'batman', - 'Battery' => 'battery', - 'Battleborn' => 'battleborn', - 'Battlefield' => 'battlefield', - 'Battlefield 1' => 'battlefield-1', - 'Battlefield 4' => 'battlefield-4', - 'Battlefield 5' => 'battlefield-5', - 'Battlestar Galactica' => 'battlestar-galactica', - 'Baylis & Harding' => 'baylis-and-harding', - 'Bayonetta' => 'bayonetta', - 'Bayonetta 2' => 'bayonetta-2', - 'Baywatch' => 'baywatch', - 'BB-8' => 'bb-8', - 'BBC' => 'bbc', - 'BBQ Food' => 'bbq', - 'BBQs and Grills' => 'grill', - 'Bean Bag' => 'bean-bag', - 'Beanie Hat' => 'beanie-hat', - 'Bean to Cup Machine' => 'bean-to-cup', - 'Beard Trimmer' => 'beard-trimmer', - 'Beats by Dre' => 'beats-by-dre', - 'Beats Solo 3' => 'beats-solo-3', - 'Beats Studio 3' => 'beats-studio-3', - 'Beauty' => 'beauty-care', - 'Beauty and the Beast' => 'beauty-and-the-beast', - 'Becks' => 'becks', - 'Bed' => 'bed', - 'Bedding' => 'bedding', - 'Bedding & Linens' => 'bedding-linens', - 'Bed Frame' => 'bed-frame', - 'Bedroom' => 'bedroom-furniture', - 'Beef' => 'beef', - 'Beer' => 'beer', - 'Beer Advent Calendar' => 'beer-advent-calendar', - 'Beko' => 'beko', - 'Belkin' => 'belkin', - 'Belstaff' => 'belstaff', - 'Belt' => 'belt', - 'BelVita' => 'belvita', - 'Ben & Jerry's' => 'ben-jerrys', - 'Benefit Cosmetics' => 'benefit-cosmetics', - 'BenQ' => 'benq', - 'BenQ Monitor' => 'benq-monitor', - 'Ben Sherman' => 'ben-sherman', - 'BeoPlay Headphones' => 'beoplay-headphones', - 'Beoplay Speakers' => 'beoplay', - 'Berghaus' => 'berghaus', - 'Bestway' => 'bestway', - 'Betting' => 'betting', - 'Beyerdynamic' => 'beyerdynamic', - 'Bic' => 'bic', - 'Bike' => 'bike', - 'Bike Accessories' => 'bike-accessories', - 'Bike Brake' => 'brakes', - 'Bike Computer' => 'bike-computer', - 'Bike Helmet' => 'bicycle-helmet', - 'Bike Inner Tube' => 'inner-tube', - 'Bike Lights' => 'bike-lights', - 'Bike Lock' => 'bike-lock', - 'Bike Parts' => 'bike-parts', - 'Bike Pump' => 'bike-pump', - 'Biker Equipment' => 'biker-equipment', - 'Bike Saddle' => 'saddle', - 'Biking & Urban Sports' => 'biking-urban-sports', - 'Bikini' => 'bikini', - 'Billabong' => 'billabong', - 'Bin' => 'bin', - 'Binatone' => 'binatone', - 'Bingo' => 'bingo', - 'Binoculars' => 'binoculars', - 'Bio Oil' => 'bio-oil', - 'Bioshock' => 'bioshock', - 'Birds Eye' => 'birds-eye', - 'Birkenstock' => 'birkenstock', - 'Biscuits' => 'biscuits', - 'Bissell' => 'bissell', - 'Bistro Set' => 'bistro-set', - 'Bitdefender' => 'bitdefender', - 'Black & Decker' => 'black-decker', - 'Blackberry Smartphone' => 'blackberry', - 'Blanket' => 'blanket', - 'Blaupunkt' => 'blaupunkt', - 'Blazer' => 'blazer', - 'Bleach' => 'bleach', - 'Blended Malt' => 'malt', - 'Blender' => 'blender', - 'Blinds' => 'blinds', - 'Blink XT2 Smart Security Camera' => 'blink-xt2', - 'Blizzard' => 'blizzard', - 'Blood & Truth' => 'blood-and-truth', - 'Bloodborne' => 'bloodborne', - 'Blood Pressure Monitor' => 'blood-pressure', - 'Blu-ray' => 'blu-ray', - 'Blu-ray Player' => 'blu-ray-player', - 'Bluetooth Headphones' => 'bluetooth-headphones', - 'Bluetooth Speaker' => 'bluetooth-speaker', - 'BMW' => 'bmw', - 'BMW Mini Cooper' => 'mini-cooper', - 'BMX' => 'bmx', - 'Board Game' => 'board-game', - 'Boardman' => 'boardman', - 'Boat Shoes' => 'boat-shoes', - 'Bodum' => 'bodum', - 'Bogof' => 'bogof', - 'Boiler' => 'boiler', - 'Bold' => 'bold', - 'Bombay Sapphire' => 'bombay-sapphire', - 'Bomber Jacket' => 'bomber-jacket', - 'Bonne Maman' => 'bonne-maman', - 'Bonsai' => 'bonsai', - 'Book' => 'book', - 'Bookcase' => 'bookcase', - 'Books & Magazines' => 'books-magazines', - 'Booster Seat' => 'booster-seat', - 'Boots' => 'boots', - 'Borderlands' => 'borderlands', - 'Borderlands 3' => 'borderlands-3', - 'Bosch' => 'bosch', - 'Bosch Dishwasher' => 'bosch-dishwasher', - 'Bosch Drill' => 'bosch-drill', - 'Bosch Fridge' => 'bosch-fridge', - 'Bosch Rotak' => 'rotak', - 'Bosch Washing Machine' => 'bosch-washing-machine', - 'Bose' => 'bose', - 'Bose Headphones' => 'bose-headphones', - 'Bose Noise Cancelling Headphones 700' => 'bose-headphones-700', - 'Bose QuietComfort' => 'bose-quietcomfort', - 'Bose QuietComfort 35 II' => 'bose-quietcomfort-35-ii', - 'Bose SoundLink' => 'bose-soundlink', - 'Bose SoundLink Around-Ear II' => 'bose-soundlink-2', - 'Bose SoundTouch' => 'bose-soundtouch', - 'BOSS' => 'hugo-boss', - 'Boss Bottled' => 'boss-bottled', - 'Bouncy Castle' => 'bouncy-castle', - 'Bourbon' => 'bourbon', - 'Bourjois' => 'bourjois', - 'Bowers & Wilkins' => 'bowers-wilkins', - 'Bowling' => 'bowling', - 'Bowmore' => 'bowmore', - 'Boxers' => 'boxers', - 'Boxing' => 'boxing', - 'Boxing Gloves' => 'boxing-gloves', - 'Boy's Clothes' => 'clothes-for-boys', - 'Bra' => 'bra', - 'Brabantia' => 'brabantia', - 'Bracelet' => 'bracelet', - 'Brands' => 'brand', - 'Brandy' => 'brandy', - 'Branston' => 'branston', - 'Branston Beans' => 'branston-beans', - 'Braun' => 'braun', - 'Braun Series 3' => 'braun-series-3', - 'Braun Series 5' => 'braun-series-5', - 'Braun Series 7' => 'braun-series-7', - 'Braun Series 9' => 'braun-series-9', - 'Braun Shaver' => 'braun-shaver', - 'Bread' => 'bread', - 'Breadmaker' => 'breadmaker', - 'Breakdown Cover' => 'breakdown', - 'Breaking Bad' => 'breaking-bad', - 'Breast Pump' => 'breast-pump', - 'Breville' => 'breville', - 'Breville Blend Active' => 'blendactive', - 'Brewdog' => 'brewdog', - 'Bridge Camera' => 'bridge-camera', - 'Briefcase' => 'briefcase', - 'Brita' => 'brita', - 'Britax' => 'britax', - 'British Airways' => 'british-airways', - 'Broadband' => 'broadband', - 'Broadband & Phone Contracts' => 'broadband-phone-service', - 'Brogues' => 'brogues', - 'Brother' => 'brother', - 'Brother Printer' => 'brother-printer', - 'Brownie' => 'brownie', - 'BT' => 'bt', - 'BT Sport' => 'bt-sport', - 'Budweiser' => 'budweiser', - 'Buffalo' => 'buffalo', - 'Bugaboo' => 'bugaboo', - 'Buggy' => 'buggy', - 'Build-A-Bear' => 'build-a-bear', - 'Bulb' => 'bulbs', - 'Bulletstorm' => 'bulletstorm', - 'Bulmers' => 'bulmers', - 'Bulova' => 'bulova', - 'Burberry' => 'burberry', - 'Burger' => 'burger', - 'Burnout Paradise' => 'burnout-paradise', - 'Burt's Bees' => 'burts-bees', - 'Bus and Coach Ticket' => 'bus', - 'Bush' => 'bush', - 'Bushmills' => 'bushmills', - 'Butter' => 'butter', - 'Buying From Abroad' => 'buying-from-abroad', - 'Bvlgari' => 'bvlgari', - 'Cabin Case' => 'cabin-case', - 'Cabinet' => 'cabinet', - 'Cable Reel' => 'cable-reel', - 'Cables' => 'cables', - 'Cadbury's' => 'cadbury', - 'Café Rouge' => 'cafe-rouge', - 'Cafetière' => 'cafetiere', - 'Caffè Nero' => 'cafe-nero', - 'Cake' => 'cake', - 'Calculator' => 'calculator', - 'Calendar' => 'calendar', - 'Call of Duty' => 'call-of-duty', - 'Call of Duty: Black Ops' => 'black-ops', - 'Call of Duty: Black Ops 3' => 'black-ops-3', - 'Call of Duty: Black Ops 4' => 'black-ops-4', - 'Call of Duty: Black Ops Cold War' => 'call-of-duty-black-ops-cold-war', - 'Call of Duty: Infinite Warfare' => 'call-of-duty-infinite-warfare', - 'Call of Duty: Modern Warfare' => 'modern-warfare', - 'Call of Duty: WW2' => 'call-of-duty-ww2', - 'Calpol' => 'calpol', - 'Calvin Klein' => 'calvin-klein', - 'Camcorder' => 'camcorder', - 'Camelbak' => 'camelbak', - 'Camera' => 'camera', - 'Camera Accessories' => 'camera-accessories', - 'Camera Bag' => 'camera-bag', - 'Camera Lens' => 'lens', - 'Camping' => 'camping', - 'Campingaz' => 'campingaz', - 'Candle' => 'candle', - 'Cannondale' => 'cannondale', - 'Canon' => 'canon', - 'Canon Camera' => 'canon-camera', - 'Canon EOS' => 'canon-eos', - 'Canon Lens' => 'canon-lens', - 'Canon Pixma' => 'canon-pixma', - 'Canon PowerShot' => 'canon-powershot', - 'Canon PowerShot SX430 IS' => 'canon-powershot-sx430-is', - 'Canon Printer' => 'canon-printer', - 'Canterbury' => 'canterbury', - 'Canton' => 'canton', - 'Canvas Print' => 'canvas-print', - 'Cap' => 'cap', - 'Capsule Machine' => 'capsule-machine', - 'Captain America' => 'captain-america', - 'Captain Morgan' => 'captain-morgan', - 'Captain Toad: Treasure Tracker' => 'captain-toad-treasure-tracker', - 'Car' => 'car', - 'Car & Motorcycle' => 'car-motorcycle', - 'Car Accessories' => 'car-accessories', - 'Caravan' => 'caravan', - 'Car Battery' => 'car-battery', - 'Carbon Monoxide Detector' => 'carbon-monoxide', - 'Car Care' => 'car-care', - 'Car Charger' => 'car-charger', - 'Cardhu' => 'cardhu', - 'Cardigan' => 'cardigan', - 'Card Reader' => 'card-reader', - 'Carex' => 'carex', - 'Carhartt' => 'carhartt', - 'Car Hire' => 'car-hire', - 'Car Insurance' => 'car-insurance', - 'Car Leasing' => 'car-lease', - 'Carling' => 'carling', - 'Car Lock' => 'lock', - 'Carlsberg' => 'carlsberg', - 'Car Mats' => 'car-mats', - 'Carolina Herrera' => 'carolina-herrera', - 'Car Parts' => 'car-parts', - 'Carpet' => 'carpet', - 'Carpet Cleaner' => 'carpet-cleaner', - 'CarPlan' => 'carplan', - 'Car Polish' => 'car-polish', - 'Carrera Bikes' => 'carrera', - 'Car Seat' => 'car-seat', - 'Car Service' => 'car-service', - 'Car Stereo' => 'car-stereo', - 'Car Wash' => 'car-wash', - 'Car Wax' => 'car-wax', - 'Casio' => 'casio', - 'Casio Eco-Drive' => 'eco-drive', - 'Casio Edifice' => 'edifice', - 'Casio G-Shock' => 'g-shock', - 'Casserole' => 'casserole', - 'Cast Iron Pots and Pans' => 'cast-iron', - 'Castrol' => 'castrol', - 'Caterpillar' => 'caterpillar', - 'Cat Flap' => 'cat-flap', - 'Cat Food' => 'cat-food', - 'Cath Kidston' => 'cath-kidston', - 'Cat Supplies' => 'cat-supplies', - 'CCTV' => 'cctv', - 'CD' => 'cd', - 'CD Player' => 'cd-player', - 'Ceiling Light' => 'ceiling-light', - 'Celebrations' => 'celebrations', - 'Cereal' => 'cereal', - 'Cetirizine' => 'cetirizine', - 'Chad Valley' => 'chad-valley', - 'Chainsaw' => 'chainsaw', - 'Champagne' => 'champagne', - 'Champneys' => 'champneys', - 'Chanel' => 'chanel', - 'Chanel Coco Mademoiselle' => 'coco-mademoiselle', - 'Changing Bag' => 'changing-bag', - 'Channel 4' => 'channel-4', - 'Charger' => 'charger', - 'Cheese' => 'cheese', - 'Chelsea Boots' => 'chelsea-boots', - 'Chelsea F. C.' => 'chelsea', - 'Chess' => 'chess', - 'Chessington' => 'chessington', - 'Chest Freezer' => 'chest-freezer', - 'Chest of Drawers' => 'chest-of-drawers', - 'Chicco' => 'chicco', - 'Chicken' => 'chicken', - 'Childcare' => 'baby', - 'Children's Books' => 'childrens-books', - 'Chino' => 'chino', - 'Chisel' => 'chisel', - 'Chloe' => 'chloe', - 'Chocolate' => 'chocolate', - 'Chocolate Advent Calendar' => 'chocolate-advent-calendar', - 'Chopper' => 'chopper', - 'Chopping Board' => 'chopping-board', - 'Christmas Card' => 'christmas-card', - 'Christmas Decoration' => 'christmas-decorations', - 'Christmas Gift' => 'christmas-gifts', - 'Christmas Jumper' => 'christmas-jumper', - 'Christmas Lights' => 'christmas-lights', - 'Christmas Stocking Fillers' => 'christmas-stocking-fillers', - 'Christmas Toys' => 'christmas-toys', - 'Christmas Tree' => 'christmas-tree', - 'Chromebook' => 'chromebook', - 'Chromecast' => 'chromecast', - 'Chromecast Ultra' => 'chromecast-ultra', - 'Chromecast with Google TV' => 'chromecast-google-tv', - 'Chronograph' => 'chronograph', - 'Chupa Chups' => 'chupa-chups', - 'Chuwi' => 'chuwi', - 'Cider' => 'cider', - 'Cinema' => 'cinema', - 'Cineworld' => 'cineworld', - 'Circular Saw' => 'circular-saw', - 'Circulon' => 'circulon', - 'Ciroc' => 'ciroc', - 'Cities Skylines' => 'cities-skylines', - 'Citizen' => 'citizen', - 'Citroen' => 'citroen', - 'City Break' => 'city-breaks', - 'Civilization' => 'civilization', - 'Clarins' => 'clarins', - 'Clarks' => 'clarks', - 'Clearance' => 'clearance', - 'Climbing' => 'climbing', - 'Climbing Frame' => 'climbing-frame', - 'Clinique' => 'clinique', - 'Clothes' => 'clothes', - 'Cloud Service' => 'cloud', - 'Clutch Bag' => 'clutch', - 'Coat' => 'coat', - 'Coca Cola' => 'coke', - 'Cocktail' => 'cocktail', - 'Coconut Oil' => 'coconut', - 'Coffee' => 'coffee', - 'Coffee Beans' => 'coffee-beans', - 'Coffee Machine' => 'coffee-machine', - 'Coffee Pods' => 'coffee-pods', - 'Coffee Table' => 'coffee-table', - 'Cognac' => 'cognac', - 'Cola' => 'cola', - 'Coleman' => 'coleman', - 'Colgate' => 'colgate', - 'Combi Drill' => 'combi', - 'Comfort' => 'comfort', - 'Comic' => 'comic', - 'Command & Conquer' => 'command-and-conquer', - 'Compact Camera' => 'compact-camera', - 'Compact Flash' => 'compact-flash', - 'Competitions' => 'competitions', - 'Compost' => 'compost', - 'Compressor' => 'compressor', - 'Computer Accessories' => 'computer-accessories', - 'Computers & Tablets' => 'computers', - 'Concert' => 'concert', - 'Condé Nast' => 'conde-nast', - 'Conditioner' => 'conditioner', - 'Condom' => 'condom', - 'Connectors' => 'connectors', - 'Contact Lenses' => 'contact-lenses', - 'Contents Insurance' => 'contents-insurance', - 'Controller' => 'controller', - 'Converse' => 'converse', - 'Converse Chuck Taylor' => 'chuck-taylor', - 'Cooker' => 'cooker', - 'Cooking Oil' => 'cooking-oil', - 'Cookware' => 'cooking', - 'Cookware Set' => 'cookware-set', - 'Cookworks' => 'cookworks', - 'Cool Box' => 'cool-box', - 'Coors Light' => 'coors-light', - 'Cordless Drill' => 'cordless-drill', - 'Cordless Phone' => 'cordless-phone', - 'Cornetto' => 'cornetto', - 'Corona Beer' => 'corona', - 'Corsair' => 'corsair', - 'Cosatto' => 'cosatto', - 'Costa Coffee' => 'costa-coffee', - 'Costume' => 'costume', - 'Cot' => 'cot', - 'Counter Strike' => 'counter-strike', - 'Courses and Training' => 'education', - 'Cow & Gate' => 'cow-and-gate', - 'Cozy Coupe' => 'cozy-coupe', - 'CPU' => 'cpu', - 'CPU Cooler' => 'cpu-cooler', - 'Craghoppers' => 'craghoppers', - 'Crash Bandicoot' => 'crash-bandicoot', - 'Crash Team Racing Nitro-Fueled' => 'crash-team-racing-nitro-fueled', - 'Crayola' => 'crayola', - 'Creatine' => 'creatine', - 'Credit Card' => 'credit-card', - 'Creme Egg' => 'creme-egg', - 'Cricket' => 'cricket', - 'Crisps' => 'crisps', - 'Crocs' => 'crocs', - 'Cross Trainer' => 'cross-trainer', - 'Crown Paint' => 'crown', - 'Crucial' => 'crucial', - 'Cruelty Free Makeup' => 'cruelty-free-makeup', - 'Cruises' => 'cruise', - 'Cube Bikes' => 'cube', - 'Cubot' => 'cubot', - 'Cufflinks' => 'cufflinks', - 'Culture & Leisure' => 'entertainment', - 'Cuphead' => 'cuphead', - 'Cuprinol' => 'cuprinol', - 'Curling Wand' => 'curling-wand', - 'Curtain' => 'curtain', - 'Cushelle' => 'cushelle', - 'Cushion' => 'cushion', - 'Cutlery' => 'cutlery', - 'CyberLink' => 'cyberlink', - 'Cyberpunk 2077' => 'cyberpunk-2077', - 'Cybex' => 'cybex', - 'Cycling' => 'cycling', - 'Cycling Jacket' => 'cycling-jacket', - 'D-Link' => 'd-link', - 'DAB Radio' => 'dab-radio', - 'Dacia' => 'dacia', - 'Daily Mail' => 'daily-mail', - 'Dairy Milk' => 'dairy-milk', - 'Darksiders' => 'darksiders', - 'Dark Souls' => 'dark-souls', - 'Dark Souls 3' => 'dark-souls-3', - 'Dartboard' => 'dartboard', - 'Darts' => 'darts', - 'Dash Cam' => 'dash-cam', - 'Data Storage' => 'storage', - 'Davidoff' => 'davidoff', - 'Days Gone' => 'days-gone', - 'Days Out' => 'days-out', - 'Daz' => 'daz', - 'DC Comic' => 'dc', - 'DDR3' => 'ddr3', - 'DDR4' => 'ddr4', - 'Dead Island' => 'dead-island', - 'Dead or Alive 6' => 'dead-or-alive-6', - 'Deadpool' => 'deadpool', - 'Dead Rising' => 'dead-rising', - 'Death Stranding' => 'death-stranding', - 'Deezer' => 'deezer', - 'Dehumidifier' => 'dehumidifier', - 'Dell' => 'dell', - 'Dell Laptop' => 'dell-laptop', - 'Dell Monitor' => 'dell-monitor', - 'Dell XPS' => 'xps', - 'Delonghi' => 'delonghi', - 'Demon's Souls' => 'demon-souls', - 'Denby' => 'denby', - 'Denon' => 'denon', - 'Deodorant' => 'deodorant', - 'Desk' => 'desk', - 'Desperados Beer' => 'desperados', - 'Despicable Me' => 'despicable-me', - 'Destiny' => 'destiny', - 'Destiny 2' => 'destiny-2', - 'Detergent' => 'detergent', - 'Detroit: Become Human' => 'detroit-become-human', - 'Dettol' => 'dettol', - 'Deus Ex' => 'deus-ex', - 'Deus Ex: Mankind Divided' => 'deus-ex-mankind-divided', - 'Development Boards' => 'development-boards', - 'Devil May Cry 5' => 'devil-may-cry-5', - 'DeWalt' => 'dewalt', - 'DFDS' => 'dfds', - 'Diablo 3' => 'diablo-3', - 'Diary' => 'diary', - 'Dickies' => 'dickies', - 'Diesel' => 'diesel', - 'Diet' => 'diet', - 'Diggerland' => 'diggerland', - 'Digihome' => 'digihome', - 'Digimon' => 'digimon', - 'Digital Camera' => 'digital-camera', - 'Digital Watch' => 'digital-watch', - 'Dildo' => 'dildo', - 'Dimplex' => 'dimplex', - 'Dining Room' => 'dining-room', - 'Dining Room Chair' => 'chair', - 'Dining Set' => 'dining-set', - 'Dining Table' => 'dining-table', - 'Dinner Plate' => 'plates', - 'Dinner Set' => 'dinner-set', - 'Dinosaur' => 'dinosaur', - 'Dior' => 'dior', - 'Dior Sauvage' => 'dior-sauvage', - 'Dirt' => 'dirt', - 'Dirt 4' => 'dirt-4', - 'DIRT 5' => 'dirt-5', - 'Dirt Rally 2.0' => 'dirt-rally-2', - 'Disaronno' => 'disaronno', - 'Discord Nitro' => 'discord-nitro', - 'Disgaea' => 'disgaea', - 'Dishonored' => 'dishonored', - 'Dishonored 2' => 'dishonored-2', - 'Dishwasher' => 'dishwasher', - 'Dishwasher Tablets' => 'dishwasher-tablets', - 'Disinfectants' => 'disinfectants', - 'Disney' => 'disney', - 'Disney's Cars' => 'disney-cars', - 'Disney's Frozen' => 'disney-frozen', - 'Disney+' => 'disney-plus', - 'Disney Infinity' => 'disney-infinity', - 'Disneyland' => 'disneyland', - 'Disney Princess' => 'disney-princess', - 'Disney Tsum Tsum' => 'tsum-tsum', - 'Disney World' => 'disney-world', - 'Divan' => 'divan', - 'DIY' => 'diy', - 'DJ Equipment' => 'dj', - 'DJI Phantom' => 'dji-phantom', - 'DKNY' => 'dkny', - 'Doctor Who' => 'doctor-who', - 'Dog Bed' => 'dog-bed', - 'Dog Food' => 'dog-food', - 'Dog Supplies' => 'dog', - 'Dolce & Gabbana' => 'dolce', - 'Dolce Gusto' => 'dolce-gusto', - 'Dolce Gusto Coffee Machine' => 'dolce-gusto-coffee-machine', - 'Doll' => 'doll', - 'Dolls House' => 'dolls-house', - 'Domain Service' => 'domain', - 'Doogee' => 'doogee', - 'Doom' => 'doom', - 'Door' => 'door', - 'Doorbell' => 'doorbell', - 'Door Handles' => 'door-handles', - 'Doormat' => 'doormat', - 'Doritos' => 'doritos', - 'Dove' => 'dove', - 'Down Jacket' => 'down-jacket', - 'Downton Abbey' => 'downton-abbey', - 'Dr. Martens' => 'dr-martens', - 'Dragon Age' => 'dragon-age', - 'Dragon Ball' => 'dragon-ball', - 'Dragon Ball: FighterZ' => 'dragon-ball-fighterz', - 'Dragon Quest' => 'dragon-quest', - 'Dragon Quest Builders' => 'dragon-quest-builders', - 'Dragon Quest Builders 2' => 'dragon-quest-builders-2', - 'Dragon Quest XI: Echoes of an Elusive Age' => 'dragon-quest-xi', - 'Draper' => 'draper', - 'Drayton Manor' => 'drayton-manor', - 'Dreame T20' => 'dreame-t20', - 'Dreame V9' => 'dreame-v9', - 'Dreame V9P' => 'dreame-v9p', - 'Dreame V10' => 'dreame-v10', - 'Dreame V11' => 'dreame-v11', - 'Dreame Vacuum Cleaner' => 'xiaomi-vacuum-cleaner', - 'Dremel' => 'dremel', - 'Dress' => 'dress', - 'Dressing Gown' => 'dressing-gown', - 'Drill' => 'drill', - 'Drill Driver' => 'driver', - 'Drinks' => 'drinks', - 'Driveclub' => 'driveclub', - 'Driving Lessons' => 'driving-lessons', - 'Drone' => 'drone', - 'Dryer' => 'dryer', - 'DSLR Camera' => 'dslr', - 'Dual Fuel Cooker' => 'dual-fuel', - 'Dualit' => 'dualit', - 'Dual Sim' => 'sim', - 'Dulux' => 'dulux', - 'Duracell' => 'duracell', - 'Durex' => 'durex', - 'Duvet' => 'duvet', - 'DVD' => 'dvd', - 'DVD Player' => 'dvd-player', - 'Dying Light' => 'dying-light', - 'Dymo' => 'dymo', - 'Dyson' => 'dyson', - 'Dyson Supersonic' => 'dyson-supersonic', - 'Dyson V6' => 'dyson-v6', - 'Dyson V7' => 'dyson-v7', - 'Dyson V8' => 'dyson-v8', - 'Dyson V10' => 'dyson-v10', - 'Dyson V11' => 'dyson-v11', - 'Dyson Vacuum Cleaner' => 'dyson-vacuum-cleaner', - 'e-Reader' => 'ereader', - 'EA' => 'ea', - 'EA Access' => 'ea-access', - 'Earphones' => 'earphones', - 'Earrings' => 'earrings', - 'EA Sports' => 'ea-sports', - 'EA Sports UFC' => 'ufc', - 'Easter Eggs' => 'egg', - 'Eastpak' => 'eastpak', - 'eBook' => 'ebook', - 'Ecovacs' => 'ecovacs', - 'Ecover' => 'ecover', - 'Educational Toys' => 'educational-toys', - 'EE' => 'ee', - 'eFootball PES 2021' => 'pes-2021', - 'ELC Happyland' => 'happyland', - 'Electrical Accessories' => 'electrical-accessories', - 'Electric Bike' => 'electric-bike', - 'Electric Blanket' => 'electric-blanket', - 'Electric Cooker' => 'electric-cooker', - 'Electric Fires' => 'electric-fire', - 'Electric Scooter' => 'electric-scooter', - 'Electric Shower' => 'electric-shower', - 'Electric Toothbrush' => 'electric-toothbrush', - 'Electronic Accessories' => 'electronics-accessories', - 'Electronics' => 'electronics', - 'Elemis' => 'elemis', - 'Elephone' => 'elephone', - 'Elgato' => 'elgato', - 'Elite Dangerous' => 'elite-dangerous', - 'Elizabeth Arden' => 'elizabeth-arden', - 'Emirates' => 'emirates', - 'Endura' => 'endura', - 'Eneloop' => 'eneloop', - 'Energizer' => 'energizer', - 'Energy' => 'energy', - 'Energy, Heating & Gas' => 'energy-heating-gas', - 'Energy Drinks' => 'energy-drinks', - 'Engine Oil' => 'engine-oil', - 'Epilator' => 'epilator', - 'Epson' => 'epson', - 'Epson Printer' => 'epson-printer', - 'Espresso' => 'espresso', - 'Espresso Machine' => 'espresso-machine', - 'Esprit' => 'esprit', - 'Estée Lauder' => 'estee-lauder', - 'Ethernet' => 'ethernet', - 'Etnies' => 'etnies', - 'Eurostar Ticket' => 'eurostar', - 'Eurotunnel' => 'eurotunnel', - 'Everton F. C.' => 'everton', - 'EVGA' => 'evga', - 'Evian' => 'evian', - 'Exercise Equipment' => 'exercise-equipment', - 'Exercise Weights' => 'weight', - 'Extension Lead' => 'extension-lead', - 'External Hard Drive' => 'external-hard-drive', - 'F1' => 'formula-one', - 'F1 2017' => 'f1-2017', - 'F1 2018' => 'f1-2018', - 'F1 2019' => 'f1-2019', - 'F1 2020' => 'f1-2020', - 'Fabric Conditioner' => 'fabric-conditioner', - 'Face Cream' => 'face-cream', - 'Face Mask' => 'face-mask', - 'Fairy' => 'fairy', - 'Fairy Light' => 'fairy-light', - 'Fallout' => 'fallout', - 'Fallout 4' => 'fallout-4', - 'Fallout 76' => 'fallout-76', - 'Family & Kids' => 'kids', - 'Family Break' => 'family-break', - 'Family Guy' => 'family-guy', - 'Famous Grouse' => 'famous-grouse', - 'Fancy Dress' => 'fancy-dress', - 'Fans' => 'fan', - 'Fanta' => 'fanta', - 'Far Cry' => 'far-cry', - 'Far Cry 4' => 'far-cry-4', - 'Far Cry 5' => 'far-cry-5', - 'Far Cry New Dawn' => 'far-cry-new-dawn', - 'Far Cry Primal' => 'far-cry-primal', - 'Farming Simulator' => 'farming-simulator', - 'Fashion & Accessories' => 'fashion', - 'Fashion Accessories' => 'fashion-accessories', - 'Fashion for Men' => 'mens-clothing', - 'Fashion for Women' => 'womens-clothes', - 'Fast and Furious' => 'fast-and-furious', - 'Father's Day' => 'fathers-day', - 'FatMax' => 'fatmax', - 'FC Barcelona' => 'fc-barcelona', - 'Felix' => 'felix', - 'Fence' => 'fence', - 'Fender Guitar' => 'fender', - 'Ferrero Rocher' => 'ferrero-rocher', - 'Ferry' => 'ferry', - 'Festival' => 'festival', - 'Fever Thermometer' => 'thermometer', - 'Fiat' => 'fiat', - 'Fidget Spinner' => 'spinner', - 'FIFA' => 'fifa', - 'FIFA 17' => 'fifa-17', - 'FIFA 18' => 'fifa-18', - 'FIFA 19' => 'fifa-19', - 'FIFA 20' => 'fifa-20', - 'FIFA 21' => 'fifa-21', - 'FightStick' => 'fightstick', - 'Figures' => 'figures', - 'Fila Trainers' => 'fila-trainers', - 'Filing Cabinet' => 'filing-cabinet', - 'Final Fantasy' => 'final-fantasy', - 'Final Fantasy 15' => 'final-fantasy-15', - 'Finance & Insurance' => 'personal-finance', - 'Finish' => 'finish', - 'Finlux' => 'finlux', - 'Fiorelli' => 'fiorelli', - 'Fire Emblem' => 'fire-emblem', - 'Fire Pit' => 'fire-pit', - 'Fireplace' => 'fireplace', - 'Firewall: Zero Hour' => 'firewall-zero-hour', - 'First Aid' => 'first-aid', - 'Fish & Seafood' => 'fish-and-seafood', - 'Fish and Aquatic Pet Supplies' => 'fish', - 'Fisher Price' => 'fisher-price', - 'Fisher Price Imaginext' => 'imaginext', - 'Fisher Price Jumperoo' => 'jumperoo', - 'Fisher Price Little People' => 'little-people', - 'Fishing' => 'fishing', - 'Fiskars' => 'fiskars', - 'Fitbit' => 'fitbit', - 'Fitbit Alta' => 'fitbit-alta', - 'Fitbit Blaze' => 'fitbit-blaze', - 'Fitbit Charge 2' => 'fitbit-charge-2', - 'Fitbit Inspire' => 'fitbit-inspire', - 'Fitbit Versa' => 'fitbit-versa', - 'Fitness & Running' => 'fitness', - 'Fitness App' => 'fitness-app', - 'Fitness Tracker' => 'fitness-tracker', - 'Flamingo Land' => 'flamingo-land', - 'Flea Treatment' => 'flea', - 'Fleece Clothing' => 'fleece', - 'Flights' => 'flight', - 'Flip Flops' => 'flip-flops', - 'Floodlight' => 'floodlight', - 'Flooring' => 'flooring', - 'Flowers' => 'flowers', - 'Flymo' => 'flymo', - 'FM Transmitter' => 'fm-transmitter', - 'Food' => 'food', - 'Food Containers' => 'food-containers', - 'Food Processor' => 'food-processor', - 'Food Server' => 'food-server', - 'Football' => 'football', - 'Football Boots' => 'football-boots', - 'Football Manager' => 'football-manager', - 'Football Matches' => 'football-matches', - 'Football Shirt' => 'football-shirt', - 'Foot Pump' => 'foot-pump', - 'Ford' => 'ford', - 'For Honor' => 'for-honor', - 'Fortnite' => 'fortnite', - 'Fortnite: Darkfire' => 'fortnite-darkfire', - 'Forza' => 'forza', - 'Forza 7' => 'forza-7', - 'Forza Horizon' => 'forza-horizon', - 'Forza Horizon 3' => 'forza-horizon-3', - 'Forza Horizon 4' => 'forza-horizon-4', - 'Forza Motorsport' => 'forza-motorsport', - 'Foscam' => 'foscam', - 'Fossil' => 'fossil', - 'Foster's' => 'fosters', - 'Foundation' => 'foundation', - 'Fountain Pen' => 'fountain-pen', - 'Fred Perry' => 'fred-perry', - 'Freesat' => 'freesat', - 'Freeview' => 'freeview', - 'Freezer' => 'freezer', - 'Fridge' => 'fridge', - 'Fridge Freezer' => 'fridge-freezer', - 'Frontline' => 'frontline', - 'Frozen Food' => 'frozen', - 'Fruit' => 'fruit', - 'Fruit and Vegetables' => 'fruit-and-vegetable', - 'Fruit of the Loom' => 'fruit-of-the-loom', - 'Fryer' => 'fryer', - 'Frying Pan' => 'frying-pan', - 'Fujifilm' => 'fuji', - 'Fujitsu' => 'fujitsu', - 'Funko Pop' => 'funko-pop', - 'Furby' => 'furby', - 'Furniture' => 'furniture', - 'G-Star' => 'g-star', - 'G-Sync Monitor' => 'g-sync', - 'Gaggia' => 'gaggia', - 'Gambling' => 'gambling', - 'Game App' => 'game-app', - 'Game of Thrones' => 'game-of-thrones', - 'Games & Board Games' => 'board-games', - 'Games Consoles' => 'console', - 'Gaming' => 'gaming', - 'Gaming Accessories' => 'gaming-accessories', - 'Gaming Chair' => 'gaming-chair', - 'Gaming Headset' => 'gaming-headset', - 'Gaming Keyboard' => 'gaming-keyboard', - 'Gaming Laptop' => 'gaming-laptop', - 'Gaming Monitor' => 'gaming-monitor', - 'Gaming Mouse' => 'gaming-mouse', - 'Gaming PC' => 'gaming-pc', - 'Gant' => 'gant', - 'Garage' => 'garage', - 'Garage & Service' => 'garage-service', - 'Garden' => 'garden', - 'Garden & Do It Yourself' => 'garden-diy', - 'Garden Furniture' => 'garden-furniture', - 'Gardening' => 'gardening', - 'Garden Storage' => 'garden-storage', - 'Garden Table' => 'table', - 'Garmin' => 'garmin', - 'Garmin Fenix' => 'garmin-fenix', - 'Garmin Fenix 6' => 'garmin-fenix-6', - 'Garmin Fenix 6 Pro' => 'garmin-fenix-6-pro', - 'Garmin Forerunner' => 'garmin-forerunner', - 'Garmin Vivoactive' => 'garmin-vivoactive', - 'Garmin Watch' => 'garmin-watch', - 'Garnier' => 'garnier', - 'Gas' => 'gas', - 'Gas Canister' => 'butane', - 'Gas Cooker' => 'gas-cooker', - 'Gatwick' => 'gatwick', - 'Gazebo' => 'gazebo', - 'GBK' => 'gbk', - 'Gears 5' => 'gears-5', - 'Gears of War' => 'gears-of-war', - 'Gears of War 4' => 'gears-of-war-4', - 'George Foreman' => 'george-foreman', - 'Geox' => 'geox', - 'GHD' => 'ghd', - 'Ghostbusters' => 'ghostbusters', - 'Ghostbusters: The Video Game Remastered' => 'ghostbusters-the-video-game', - 'Ghost of Tsushima' => 'ghost-of-tsushima', - 'Gibson Guitar' => 'gibson', - 'giffgaff' => 'giffgaff', - 'Gift Card' => 'gift-card', - 'Gift Hamper' => 'hamper', - 'Gifts' => 'gifts', - 'Gift Set' => 'gift-set', - 'GIGABYTE' => 'gigabyte', - 'Gigaset' => 'gigaset', - 'Gilet' => 'gilet', - 'Gillette Fusion' => 'fusion', - 'Gillette Mach3' => 'mach-3', - 'Gillette Razor' => 'gillette', - 'Gimbal' => 'gimbal', - 'Gin' => 'gin', - 'Girl's Clothes' => 'girls-clothes', - 'Glasses' => 'glasses', - 'Glassware' => 'glassware', - 'Glenfiddich' => 'glenfiddich', - 'Glenlivet' => 'glenlivet', - 'Glenmorangie' => 'glenmorangie', - 'Gloves' => 'gloves', - 'Glue' => 'glue', - 'Glue Gun' => 'glue-gun', - 'Gluten-Free' => 'gluten-free', - 'God of War' => 'god-of-war', - 'Go Kart' => 'go-kart', - 'Golf' => 'golf', - 'Golf Balls' => 'golf-balls', - 'Golf Clubs' => 'golf-clubs', - 'Goodfellas' => 'goodfellas', - 'Goodmans' => 'goodmans', - 'Goodyear' => 'goodyear', - 'Google' => 'google', - 'Google Home' => 'google-home', - 'Google Home Max' => 'google-home-max', - 'Google Home Mini' => 'google-home-mini', - 'Google Nest' => 'nest', - 'Google Nest Audio' => 'google-nest-audio', - 'Google Nest Hub' => 'google-home-hub', - 'Google Nest Mini' => 'nest-mini', - 'Google Nest Protect' => 'google-nest-protect', - 'Google Nexus' => 'nexus', - 'Google Pixel' => 'google-pixel', - 'Google Pixel 2' => 'google-pixel-2', - 'Google Pixel 2 XL' => 'google-pixel-2-xl', - 'Google Pixel 3' => 'google-pixel-3', - 'Google Pixel 3 XL' => 'google-pixel-3-xl', - 'Google Pixel 3a' => 'google-pixel-3a', - 'Google Pixel 3a XL' => 'google-pixel-3a-xl', - 'Google Pixel 4' => 'google-pixel-4', - 'Google Pixel 4 XL' => 'google-pixel-4-xl', - 'Google Pixel 4a' => 'google-pixel-4a', - 'Google Pixel 4a 5G' => 'google-pixel-4a-5g', - 'Google Pixel 5' => 'google-pixel-5', - 'Google Pixelbook' => 'google-pixelbook', - 'Google Pixel XL' => 'google-pixel-xl', - 'Google Smartphone' => 'google-smartphone', - 'Google Stadia' => 'google-stadia', - 'GoPro' => 'gopro', - 'GoPro HERO 6' => 'gopro-hero-6', - 'GoPro HERO 7' => 'gopro-hero-7', - 'GoPro HERO 8' => 'gopro-hero-8', - 'GoPro HERO 9' => 'gopro-hero-9', - 'Gore-Tex Clothing and Shoes' => 'gore-tex', - 'Graco' => 'graco', - 'Grand National' => 'grand-national', - 'Gran Turismo' => 'gran-turismo', - 'Gran Turismo Sport' => 'gran-turismo-sport', - 'Graphics Card' => 'graphics-card', - 'Gravity Rush' => 'gravity-rush', - 'Graze' => 'graze', - 'GreedFall' => 'greedfall', - 'Greenhouse' => 'greenhouse', - 'Greeting Cards and Wrapping Paper' => 'wrapping-paper-and-cards', - 'Greggs' => 'greggs', - 'Grey Goose' => 'grey-goose', - 'Griffin Technology' => 'griffin', - 'GroBag' => 'grobag', - 'Groceries' => 'groceries', - 'Gruffalo' => 'gruffalo', - 'Grundig' => 'grundig', - 'GTA' => 'gta', - 'GTA V' => 'gta-v', - 'GTX 970' => 'gtx-970', - 'GTX 980' => 'gtx-980', - 'GTX 1060' => 'gtx-1060', - 'GTX 1070' => 'gtx-1070', - 'GTX 1080' => 'gtx-1080', - 'GTX 1080 Ti' => 'gtx-1080-ti', - 'GTX 1660' => 'gtx-1660', - 'GTX 1660 Ti' => 'gtx-1660-ti', - 'Guardians of the Galaxy' => 'guardians-of-the-galaxy', - 'Gucci' => 'gucci', - 'Guinness' => 'guinness', - 'Guitar' => 'guitar', - 'Guitar Amp' => 'guitar-amp', - 'Guitar Hero' => 'guitar-hero', - 'Gulliver's' => 'gullivers', - 'Gym' => 'gym', - 'Gym Membership' => 'gym-membership', - 'H1Z1' => 'h1z1', - 'Häagen Dazs' => 'haagen-dazs', - 'Habitat' => 'habitat', - 'Hacksaw' => 'hacksaw', - 'Hair Brush' => 'hair-brush', - 'Hair Care' => 'hair', - 'Hair Clipper' => 'hair-clipper', - 'Hair Colour' => 'hair-colour', - 'Haircut' => 'haircut', - 'Hair Dryer' => 'hair-dryer', - 'Hair Dye' => 'hair-dye', - 'Hair Removal Devices' => 'hair-removal-devices', - 'Halifax' => 'halifax', - 'Hall' => 'hall', - 'Halloween' => 'halloween', - 'Halo' => 'halo', - 'Halo 5' => 'halo-5', - 'Ham' => 'ham', - 'Hammer' => 'hammer', - 'Hammer Drill' => 'hammer-drill', - 'Hammock' => 'hammock', - 'Handbag' => 'handbag', - 'Hand Blender' => 'hand-blender', - 'Hand Cream' => 'hand-cream', - 'Hand Mixer' => 'hand-mixer', - 'Hand Tools' => 'hand-tools', - 'Handwash' => 'handwash', - 'Hard Drive' => 'hard-drive', - 'Haribo' => 'haribo', - 'Harman Kardon' => 'harman-kardon', - 'Harry Potter' => 'harry-potter', - 'Hasbro' => 'hasbro', - 'Hat' => 'hat', - 'Hatchimals' => 'hatchimals', - 'Hats & Caps' => 'hats-caps', - 'Hauck' => 'hauck', - 'Hayfever Remedies' => 'hayfever', - 'Headboard' => 'headboard', - 'Headphones' => 'headphones', - 'Headset' => 'headset', - 'Health & Beauty' => 'beauty', - 'Healthcare' => 'health-care', - 'Heart Rate Monitor' => 'heart-rate-monitor', - 'Heater' => 'heater', - 'Heating' => 'heating', - 'Heating Appliances' => 'heating-appliances', - 'Hedge Trimmer' => 'hedge-trimmer', - 'Heineken' => 'heineken', - 'Heinz' => 'heinz', - 'Heinz Beanz' => 'heinz-baked-beans', - 'Hello Kitty' => 'hello-kitty', - 'Hello Neighbour' => 'hello-neighbour', - 'Helly Hansen' => 'helly-hansen', - 'Henry Hoover' => 'henry-hoover', - 'Hermes' => 'hermes', - 'High5' => 'high-5', - 'Highchair' => 'highchair', - 'Hiking' => 'hiking', - 'Hilton' => 'hilton', - 'Hisense' => 'hisense', - 'Hisense TVs' => 'hisense-tv', - 'Hitachi' => 'hitachi', - 'Hitman' => 'hitman', - 'Hive' => 'hive', - 'Hive Active Heating' => 'hive-active-heating', - 'Hob' => 'hob', - 'Hobbit' => 'hobbit', - 'Hockey' => 'hockey', - 'Holiday Inn' => 'holiday-inn', - 'Holiday Park' => 'holiday-parks', - 'Holidays and Trips' => 'holidays-and-trips', - 'Hollow Knight' => 'hollow-knight', - 'Home & Living' => 'home', - 'Home Accessories' => 'home-accessories', - 'Home Appliances' => 'home-appliances', - 'Home Care' => 'home-care', - 'Home Cinema' => 'home-cinema', - 'HoMedics' => 'homedics', - 'Homefront' => 'homefront', - 'Home Networking' => 'network', - 'Homeplug' => 'homeplug', - 'Home Security' => 'home-security', - 'Homeware' => 'homeware', - 'Honda' => 'honda', - 'Honey' => 'honey', - 'Honeywell' => 'honeywell', - 'Honor 6X' => 'honor-6x', - 'Honor 7' => 'honor-7', - 'Honor 8S' => 'honor-8s', - 'Honor 8X' => 'honor-8x', - 'Honor 8X Max' => 'honor-8x-max', - 'Honor 9' => 'honor-9', - 'Honor 9X' => 'honor-9x', - 'Honor 10' => 'honor-10', - 'Honor Band 5' => 'honor-band-5', - 'Honor Play' => 'honor-play', - 'Honor Smartphone' => 'honor', - 'Honor View 20' => 'honor-view-20', - 'Hoodie' => 'hoodie', - 'Hoover' => 'hoover', - 'Hori' => 'hori', - 'Horizon: Zero Dawn' => 'horizon-zero-dawn', - 'Hornby' => 'hornby', - 'Horse Races' => 'horse-races', - 'Hose' => 'hose', - 'HOTAS' => 'hotas', - 'Hotel' => 'hotel', - 'Hotpoint' => 'hotpoint', - 'Hotspot' => 'hotspot', - 'Hot Tub' => 'hot-tub', - 'Hot Water Bottle' => 'hot-water-bottle', - 'Hot Wheels' => 'hot-wheels', - 'Hozelock' => 'hozelock', - 'HP' => 'hp', - 'HP Envy' => 'hp-envy', - 'HP Laptop' => 'hp-laptop', - 'HP Omen' => 'hp-omen', - 'HP Printer' => 'hp-printer', - 'HTC' => 'htc', - 'HTC 10' => 'htc-10', - 'HTC Desire' => 'htc-desire', - 'HTC One' => 'htc-one', - 'HTC Smartphone' => 'htc-smartphone', - 'HTC U11' => 'htc-u11', - 'HTC Vive' => 'htc-vive', - 'Huawei' => 'huawei', - 'Huawei Freebuds 3' => 'huawei-freebuds-3', - 'Huawei Headphones' => 'huawei-headphones', - 'Huawei Mate 20' => 'huawei-mate-20', - 'Huawei Mate 20 Pro' => 'huawei-mate-20-pro', - 'Huawei Mate 30' => 'huawei-mate-30', - 'Huawei Mate 30 Lite' => 'huawei-mate-30-lite', - 'Huawei Mate 30 Pro' => 'huawei-mate-30-pro', - 'Huawei Matebook' => 'huawei-matebook', - 'Huawei MediaPad M3' => 'huawei-mediapad-m3', - 'Huawei MediaPad M5' => 'huawei-mediapad-m5', - 'Huawei MediaPad T3' => 'huawei-mediapad-t3', - 'Huawei MediaPad T5' => 'huawei-mediapad-t5', - 'Huawei P9' => 'huawei-p9', - 'Huawei P10' => 'huawei-p10', - 'Huawei P20' => 'huawei-p20', - 'Huawei P20 Lite' => 'huawei-p20-lite', - 'Huawei P20 Pro' => 'huawei-p20-pro', - 'Huawei P30' => 'huawei-p30', - 'Huawei P30 Lite' => 'huawei-p30-lite', - 'Huawei P30 Pro' => 'huawei-p30-pro', - 'Huawei P40' => 'huawei-p40', - 'Huawei P40 Lite' => 'huawei-p40-lite', - 'Huawei P40 Pro' => 'huawei-p40-pro', - 'Huawei P Smart' => 'huawei-p-smart', - 'Huawei Smartphone' => 'huawei-smartphone', - 'Huawei Smartwatch' => 'huawei-smartwatch', - 'Huawei Tablet' => 'huawei-tablet', - 'Huawei Watch 2' => 'huawei-watch-2', - 'Huawei Watch GT' => 'huawei-watch-gt', - 'Huawei Watch GT2' => 'huawei-watch-gt2', - 'Huawei Watch GT 2 Pro' => 'huawei-watch-gt-2-pro', - 'Huawei Y7' => 'huawei-y7', - 'Huggies' => 'huggies', - 'Hulk' => 'hulk', - 'Humax' => 'humax', - 'Humidifier' => 'humidifier', - 'Hunter' => 'hunter', - 'HyperX' => 'hyperx', - 'Hyrule Warriors' => 'hyrule-warriors', - 'Hyundai' => 'hyundai', - 'IAMS' => 'iams', - 'iCandy' => 'icandy', - 'Ice-Watch' => 'ice-watch', - 'Ice Cream' => 'ice-cream', - 'Ice Cream Maker' => 'ice-cream-maker', - 'iMac' => 'apple-imac', - 'iMac 2021' => 'imac-2021', - 'Impact Driver' => 'impact-driver', - 'Indesit' => 'indesit', - 'Inflatable Boats' => 'boat', - 'Inflatable Toys' => 'inflatable', - 'Injustice' => 'injustice', - 'Injustice 2' => 'injustice-2', - 'Ink Cartridge' => 'ink', - 'Inkjet Printer' => 'inkjet-printer', - 'Innocent' => 'innocent', - 'Instant Cameras' => 'instant-cameras', - 'Instant Ink' => 'instant-ink', - 'Instax Mini 9' => 'instax-mini-9', - 'Insulation' => 'insulation', - 'Insurance' => 'insurance', - 'Intel' => 'intel', - 'Intel Atom' => 'atom', - 'Intel i3' => 'i3', - 'Intel i5' => 'i5', - 'Intel i7' => 'i7', - 'Intel i9' => 'intel-i9', - 'Internet' => 'internet', - 'Internet Security' => 'internet-security', - 'In the Night Garden' => 'in-the-night-garden', - 'Intimate Care' => 'intimate-care', - 'Introduce Yourself' => 'introduce-yourself', - 'iOS Apps' => 'ios-apps', - 'iPad' => 'ipad', - 'iPad 2019' => 'ipad-2019', - 'iPad 2020' => 'ipad-2020', - 'iPad Air' => 'ipad-air', - 'iPad Air 2019' => 'ipad-air-2019', - 'iPad Air 2020' => 'ipad-air-2020', - 'iPad Case' => 'ipad-case', - 'iPad mini' => 'ipad-mini', - 'iPad Pro' => 'ipad-pro', - 'iPad Pro 11' => 'ipad-pro-11', - 'iPad Pro 12.9' => 'ipad-pro-12-9', - 'iPad Pro 2020' => 'ipad-pro-2020', - 'iPad Pro 2021' => 'ipad-pro-2021', - 'IP Camera' => 'ip-camera', - 'iPhone' => 'iphone', - 'iPhone 5s' => 'iphone-5s', - 'iPhone 6' => 'iphone-6', - 'iPhone 6 Plus' => 'iphone-6-plus', - 'iPhone 6s' => 'iphone-6s', - 'iPhone 6s Plus' => 'iphone-6s-plus', - 'iPhone 7' => 'iphone-7', - 'iPhone 7 Plus' => 'iphone-7-plus', - 'iPhone 8' => 'iphone-8', - 'iPhone 8 Plus' => 'iphone-8-plus', - 'iPhone 11' => 'iphone-11', - 'iPhone 11 Pro' => 'iphone-11-pro', - 'iPhone 11 Pro Max' => 'iphone-11-pro-max', - 'iPhone 12' => 'iphone-12', - 'iPhone 12 mini' => 'iphone-12-mini', - 'iPhone 12 Pro' => 'iphone-12-pro', - 'iPhone 12 Pro Max' => 'iphone-12-pro-max', - 'iPhone Accessories' => 'iphone-accessories', - 'iPhone Case' => 'iphone-case', - 'iPhone SE' => 'iphone-se', - 'iPhone X' => 'iphone-x', - 'iPhone Xr' => 'iphone-xr', - 'iPhone Xs' => 'iphone-xs', - 'iPhone Xs Max' => 'iphone-xs-max', - 'iPod' => 'ipod', - 'iPod Nano' => 'ipod-nano', - 'iPod Shuffle' => 'ipod-shuffle', - 'iPod Touch' => 'ipod-touch', - 'Irish Whiskey' => 'irish-whisky', - 'Irn Bru' => 'irn-bru', - 'iRobot' => 'irobot', - 'Iron' => 'iron', - 'Ironing' => 'ironing', - 'Ironing Board' => 'ironing-board', - 'Iron Man' => 'iron-man', - 'Issey Miyake' => 'issey-miyake', - 'ITV' => 'itv', - 'Jabra' => 'jabra', - 'Jabra Elite 85h' => 'jabra-elite-85h', - 'Jabra Elite Active 65t' => 'jabra-elite-active-65t', - 'Jabra Elite Active 75t' => 'jabra-elite-active-75t', - 'Jabra Headphones' => 'jabra-headphones', - 'Jack & Jones' => 'jack-and-jones', - 'Jack Daniel's' => 'jack-daniels', - 'Jacket' => 'jacket', - 'Jack Wills' => 'jack-wills', - 'Jack Wolfskin' => 'jack-wolfskin', - 'Jaffa Cakes' => 'jaffa-cakes', - 'Jägermeister' => 'jagermeister', - 'Jameson' => 'jameson', - 'Jamie Oliver' => 'jamie-oliver', - 'Jaybird' => 'jaybird', - 'JBL' => 'jbl', - 'JBL Flip' => 'jbl-flip', - 'JBL GO' => 'jbl-go', - 'JBL Headphones' => 'jbl-headphones', - 'JBL Link' => 'jbl-link', - 'JBL Live' => 'jbl-live', - 'JBL Tune' => 'jbl-tune', - 'JCB' => 'jcb', - 'Jean Paul Gaultier' => 'jean-paul-gautier', - 'Jean Paul Gaultier Le Male' => 'le-male', - 'Jeans' => 'jeans', - 'Jelly Belly' => 'jelly-belly', - 'Jewellery' => 'jewellery', - 'Jigsaw' => 'jigsaw', - 'Jim Beam' => 'jim-beam', - 'Jimmy Choo' => 'jimmy-choo', - 'JML' => 'jml', - 'Jogging Bottoms' => 'jogging-bottoms', - 'Johnnie Walker' => 'johnnie-walker', - 'Johnson's' => 'johnsons', - 'John West' => 'john-west', - 'John Wick' => 'john-wick', - 'JoJo Siwa' => 'jojo', - 'Joop' => 'joop', - 'Joseph Joseph' => 'joseph-joseph', - 'Joules' => 'joules', - 'Juice' => 'juice', - 'Juicer' => 'juicer', - 'Jumper' => 'jumper', - 'Jurassic World' => 'jurassic-world', - 'Jura Whisky' => 'jura', - 'Just Cause' => 'just-cause', - 'Just Cause 3' => 'just-cause-3', - 'Just Cause 4' => 'just-cause-4', - 'Just Dance' => 'just-dance', - 'JVC' => 'jvc', - 'K-Swiss' => 'k-swiss', - 'Karcher' => 'karcher', - 'Karcher Window Vacuum' => 'karcher-window-cleaner', - 'Karen Millen' => 'karen-millen', - 'Karrimor' => 'karrimor', - 'Kaspersky' => 'kaspersky', - 'Kayak' => 'kayak', - 'Keg' => 'keg', - 'Kellogg's' => 'kelloggs', - 'Kellogg's Cornflakes' => 'cornflakes', - 'Kellogg's Crunchy Nut' => 'crunchy-nut', - 'Kenco' => 'kenco', - 'Kenwood' => 'kenwood', - 'Kenwood kMix' => 'kmix', - 'Kenzo' => 'kenzo', - 'Ketchup' => 'ketchup', - 'Keter' => 'keter', - 'Kettle' => 'kettle', - 'Kettlebell' => 'kettlebell', - 'Keyboard' => 'keyboard', - 'KIA' => 'kia', - 'Kickers' => 'kickers', - 'Kid's Bike' => 'kids-bike', - 'Kid's Clothes' => 'kids-clothes', - 'Kid's Room' => 'kids-rooms', - 'Kid's Shoes' => 'kids-shoes', - 'Kidizoom' => 'kidizoom', - 'Killzone' => 'killzone', - 'Kilner' => 'kilner', - 'Kinder' => 'kinder', - 'Kindle' => 'kindle', - 'Kindle Book' => 'kindle-book', - 'Kindle Fire' => 'kindle-fire', - 'Kindle Oasis' => 'kindle-oasis', - 'Kindle Paperwhite' => 'kindle-paperwhite', - 'Kingdom Come: Deliverance' => 'kingdom-come-deliverance', - 'Kingdom Hearts' => 'kingdom-hearts', - 'Kingdom Hearts 3' => 'kingdom-hearts-3', - 'Kingdom Hearts: The Story So Far' => 'kingdom-hearts-the-story-so-far', - 'King Kong' => 'king-kong', - 'King Size Bed' => 'king-size', - 'Kingsmill' => 'kingsmill', - 'Kingston' => 'kingston', - 'Kitchen' => 'kitchen', - 'KitchenAid' => 'kitchenaid', - 'Kitchen Appliances' => 'kitchen-appliances', - 'Kitchen Knife' => 'knife', - 'Kitchen Roll' => 'kitchen-roll', - 'Kitchen Scale' => 'kitchen-scales', - 'Kitchen Tap' => 'kitchen-tap', - 'Kitchen Utensils' => 'kitchen-utensils', - 'Kite' => 'kite', - 'KitSound' => 'kitsound', - 'Knickers' => 'knickers', - 'Kobo' => 'kobo', - 'Kodak' => 'kodak', - 'Kodi' => 'kodi', - 'Kohinoor' => 'kohinoor', - 'Kopparberg' => 'kopparberg', - 'Kraken' => 'kraken', - 'Krispy Kreme' => 'krispy-kreme', - 'Krups' => 'krups', - 'KTC' => 'ktc', - 'Kurt Geiger' => 'kurt-geiger', - 'L'Occitane' => 'loccitane', - 'L.O.L. Surprise!' => 'lol-surprise', - 'Lacoste' => 'lacoste', - 'Ladder' => 'ladder', - 'Lamaze' => 'lamaze', - 'Lamb' => 'lamb', - 'Laminate' => 'laminate', - 'Laminator' => 'laminator', - 'Lamp' => 'lamp', - 'Lancôme' => 'lancome', - 'Landmann' => 'landmann', - 'Lantern' => 'lantern', - 'Laphroaig' => 'laphroaig', - 'Laptop' => 'laptop', - 'Laptop Accessories' => 'laptop-accessories', - 'Laptop Case' => 'laptop-case', - 'Laptop Sleeve' => 'laptop-sleeve', - 'Laser Printer' => 'laser-printer', - 'Last Minute' => 'last-minute', - 'Laundry Basket' => 'laundry-basket', - 'Laura Ashley' => 'laura-ashley', - 'Lavazza' => 'lavazza', - 'Lavender' => 'lavender', - 'Lawnmower' => 'lawnmower', - 'Lay-Z-Spa' => 'lay-z-spa', - 'LeapFrog' => 'leapfrog', - 'Le Creuset' => 'le-creuset', - 'LED Bulb' => 'led-bulbs', - 'LED Light' => 'led-light', - 'LED Strip Lights' => 'led-strip-lights', - 'LED TV' => 'led-tv', - 'Lee Stafford' => 'lee-stafford', - 'Leffe' => 'leffe', - 'Leggings' => 'leggings', - 'Lego' => 'lego', - 'Lego Advent Calendar' => 'lego-advent-calendar', - 'Lego Architecture' => 'lego-architecture', - 'Lego Art' => 'lego-art', - 'Lego Batman' => 'lego-batman', - 'Lego BrickHeadz' => 'lego-brickheadz', - 'Lego City' => 'lego-city', - 'Lego Classic' => 'lego-classic', - 'Lego Creator' => 'lego-creator', - 'Lego Dimensions' => 'lego-dimensions', - 'Lego Disney' => 'lego-disney', - 'Lego Dots' => 'lego-dots', - 'Lego Duplo' => 'lego-duplo', - 'Lego Friends' => 'lego-friends', - 'LEGO Harry Potter' => 'lego-harry-potter', - 'Lego Hidden Side' => 'lego-hidden-side', - 'Legoland' => 'legoland', - 'Lego Marvel' => 'lego-marvel', - 'Lego Mindstorms' => 'lego-mindstorms', - 'Lego Nexo Knights' => 'lego-nexo-knights', - 'Lego Ninjago' => 'lego-ninjago', - 'Lego Porsche' => 'lego-porsche', - 'Lego Simpsons' => 'lego-simpsons', - 'Lego Speed Champions' => 'lego-speed-champions', - 'Lego Star Wars' => 'lego-star-wars', - 'Lego Star Wars Millennium Falcon' => 'lego-star-wars-millennium-falcon', - 'Lego Super Mario' => 'lego-mario', - 'Lego Technic' => 'lego-technic', - 'Lego VIDIYO' => 'lego-vidiyo', - 'Lemonade' => 'lemonade', - 'Lenor' => 'lenor', - 'Lenovo' => 'lenovo', - 'Lenovo IdeaPad' => 'lenovo-ideapad', - 'Lenovo Laptop' => 'lenovo-laptop', - 'Lenovo Tablet' => 'lenovo-tablet', - 'Lenovo Thinkpad' => 'thinkpad', - 'Lenovo Yoga Laptop' => 'lenovo-yoga-laptop', - 'Lenovo Yoga Tablet' => 'lenovo-yoga', - 'Les Paul' => 'les-paul', - 'Levi's' => 'levi', - 'Lexar' => 'lexar', - 'LG' => 'lg', - 'LG G3' => 'lg-g3', - 'LG G5' => 'lg-g5', - 'LG G6' => 'lg-g6', - 'LG G7' => 'lg-g7', - 'LG G8S ThinQ' => 'lg-g8s-thinq', - 'LG OLED TV' => 'lg-oled-tv', - 'LG Smartphone' => 'lg-smartphone', - 'LG TV' => 'lg-tv', - 'LG V30' => 'lg-v30', - 'LG V40 ThinQ' => 'lg-v40-thinq', - 'Life Insurance' => 'life-insurance', - 'Life is Strange' => 'life-is-strange', - 'Light Box' => 'light-box', - 'Lighting' => 'lighting', - 'Lightning Cable' => 'lightning-cable', - 'Lightsaber' => 'lightsaber', - 'Lindor' => 'lindor', - 'Lindt' => 'lindt', - 'Lingerie' => 'lingerie', - 'Linksys' => 'linksys', - 'Linx' => 'linx', - 'Lion King' => 'lion-king', - 'Lipstick' => 'lipstick', - 'Lipsy' => 'lipsy', - 'Little Tikes' => 'little-tikes', - 'Liverpool F. C.' => 'liverpool-fc', - 'Living Room' => 'living-room', - 'Local Traffic' => 'local-traffic', - 'Lodge' => 'lodge', - 'Loft' => 'loft', - 'Logitech' => 'logitech', - 'Logitech G430' => 'logitech-g430', - 'Logitech G703' => 'logitech-g703', - 'Logitech G903' => 'logitech-g903', - 'Logitech Harmony' => 'harmony', - 'Logitech Keyboard' => 'logitech-keyboard', - 'Logitech Mouse' => 'logitech-mouse', - 'Logitech MX Master' => 'logitech-mx-master', - 'Logitech MX Master 2S' => 'logitech-mx-master-2s', - 'London Eye' => 'london-eye', - 'London Zoo' => 'london-zoo', - 'Longleat' => 'longleat', - 'Long Sleeve' => 'long-sleeve', - 'Lord of the Rings' => 'lord-of-the-rings', - 'Lottery' => 'lottery', - 'Lounger' => 'lounger', - 'Lowepro' => 'lowepro', - 'Lucozade' => 'lucozade', - 'Luigi' => 'luigi', - 'Luigi's Mansion' => 'luigis-manison', - 'Luigi's Mansion 3' => 'luigis-mansion-3', - 'Lunch Bag' => 'lunch-bag', - 'Lunch Box' => 'lunch-box', - 'Lurpak' => 'lurpak', - 'Luton' => 'luton', - 'Lyle & Scott' => 'lyle-and-scott', - 'Lynx' => 'lynx', - 'M.2 SSD' => 'm2-ssd', - 'MacBook' => 'macbook', - 'MacBook Air' => 'macbook-air', - 'MacBook Pro' => 'macbook-pro', - 'MacBook Pro 13' => 'macbook-pro-13', - 'MacBook Pro 15' => 'macbook-pro-15', - 'MacBook Pro 16' => 'macbook-pro-16', - 'Maclaren' => 'maclaren', - 'Mac mini' => 'mac-mini', - 'Madame Tussauds' => 'madame-tussauds', - 'Mad Catz' => 'madcatz', - 'Madden NFL' => 'madden', - 'Madden NFL 20' => 'madden-nfl-20', - 'Mad Max' => 'mad-max', - 'Mafia 3' => 'mafia-3', - 'Magazine' => 'magazine', - 'Magimix' => 'magimix', - 'Magners' => 'magners', - 'Magnum' => 'magnum', - 'Make Up' => 'make-up', - 'Makeup Advent Calendar' => 'makeup-advent-calendar', - 'Make Up Brush' => 'make-up-brush', - 'Makita' => 'makita', - 'Makita Drill' => 'makita-drill', - 'Malibu' => 'malibu', - 'Maltesers' => 'maltesers', - 'MAM' => 'mam', - 'Mamas & Papas' => 'mamas-and-papas', - 'Manchester United' => 'manchester-united', - 'Manfrotto' => 'manfrotto', - 'Manga' => 'manga', - 'Manuka Honey' => 'manuka-honey', - 'Marantz' => 'marantz', - 'Marc Jacobs' => 'marc-jacobs', - 'Marc Jacobs Daisy' => 'daisy', - 'Mario & Sonic at the Olympic Games: Tokyo 2020' => 'mario-and-sonic-tokyo-2020', - 'Mario + Rabbids Kingdom Battle' => 'mario-rabbids-kingdom-battle', - 'Mario Kart' => 'mario-kart', - 'Mario Kart 8' => 'mario-kart-8', - 'Mario Kart 8 Deluxe' => 'mario-kart-8-deluxe', - 'Marmite' => 'marmite', - 'Mars' => 'mars', - 'Marshall' => 'marshall', - 'Marshall Headphones' => 'marshall-headphones', - 'Marvel' => 'marvel', - 'Marvel's Spider-Man (PS4)' => 'spider-man-2018', - 'Marvel's Spider-Man: Miles Morales' => 'spiderman-miles-morales', - 'Mascara' => 'mascara', - 'Massage' => 'massage', - 'Mass Effect' => 'mass-effect', - 'Mass Effect: Andromeda' => 'mass-effect-andromeda', - 'Mastercard' => 'mastercard', - 'Masterplug' => 'masterplug', - 'Maternity & Pregnancy' => 'maternity', - 'Mattress' => 'mattress', - 'Mattress Protector' => 'mattress-protector', - 'Mattress Topper' => 'mattress-topper', - 'Mavic' => 'mavic', - 'Max Factor' => 'max-factor', - 'Maxi Cosi' => 'maxi-cosi', - 'Maximuscle' => 'maximuscle', - 'Maxtor' => 'maxtor', - 'Maybelline' => 'maybelline', - 'Mayo' => 'mayo', - 'Mazda' => 'mazda', - 'McAfee' => 'mcafee', - 'Meat & Sausages' => 'meat', - 'Meccano' => 'meccano', - 'Mechanical Keyboard' => 'mechanical-keyboard', - 'Medal of Honor' => 'medal-of-honor', - 'Medela' => 'medela', - 'Media Player' => 'media-player', - 'Medievil' => 'medievil', - 'Medion' => 'medion', - 'Mega Bloks' => 'mega-bloks', - 'Megathread' => 'megathread', - 'Melissa & Doug' => 'melissa', - 'Memory Cards' => 'memory-cards', - 'Memory Foam Mattress' => 'memory-foam', - 'Men's Boots' => 'mens-boots', - 'Men's Fragrance' => 'mens-fragrance', - 'Men's Shoes' => 'mens-shoes', - 'Men's Suit' => 'suit', - 'Mercedes' => 'mercedes', - 'Meridian' => 'meridian', - 'Merlin' => 'merlin', - 'Merrell' => 'merrell', - 'Messenger Bag' => 'messenger-bag', - 'Metal Gear Solid' => 'metal-gear-solid', - 'Metro Exodus' => 'metro-exodus', - 'Metroid' => 'metroid', - 'Metro Series' => 'metro-series', - 'Michael Kors' => 'michael-kors', - 'Michelin' => 'michelin', - 'Microphone' => 'microphone', - 'Micro SD Card' => 'micro-sd', - 'Micro SDHC' => 'micro-sdhc', - 'Micro SDXC' => 'micro-sdxc', - 'Microserver' => 'microserver', - 'Microsoft' => 'microsoft', - 'Microsoft Flight Simulator' => 'microsoft-flight-simulator', - 'Microsoft Office' => 'microsoft-office', - 'Microsoft Points' => 'microsoft-points', - 'Microsoft Software' => 'microsoft-software', - 'Microsoft Surface Book' => 'surface-book', - 'Microsoft Surface Laptop' => 'surface', - 'Microsoft Surface Pro 6' => 'surface-pro-6', - 'Microsoft Surface Pro 7' => 'surface-pro-7', - 'Microsoft Surface Tablet' => 'microsoft-surface-tablet', - 'Microwave' => 'microwave', - 'Middle Earth' => 'middle-earth', - 'Middle Earth: Shadow of Mordor' => 'shadow-of-mordor', - 'Middle Earth: Shadow of War' => 'middle-earth-shadow-of-war', - 'Miele' => 'miele', - 'Miele Vacuum Cleaner' => 'miele-vacuum-cleaner', - 'Milk' => 'milk', - 'Milk Frother' => 'milk-frother', - 'Milk Tray' => 'milk-tray', - 'Milwaukee' => 'milwaukee', - 'Mince' => 'mince', - 'Minecraft Game' => 'minecraft', - 'Mineral Water' => 'mineral-water', - 'Mini Fridge' => 'mini-fridge', - 'Minions' => 'minions', - 'Mini PC' => 'mini-pc', - 'Minky' => 'minky', - 'Mira' => 'mira', - 'Mirror' => 'mirror', - 'Mirror's Edge' => 'mirrors-edge', - 'Misc' => 'misc', - 'Misfit' => 'misfit', - 'Mitre Saw' => 'mitre-saw', - 'Mitsubishi' => 'mitsubishi', - 'Mixer & Blender' => 'mixer-and-blender', - 'Mobile Contracts' => 'mobile-contract', - 'Mobile Phone' => 'mobile-phone', - 'Model Building' => 'model-building', - 'Moët' => 'moet', - 'Molton Brown' => 'molton-brown', - 'Money Saving Tips and Tricks' => 'money-saving-tips', - 'Monitor' => 'monitor', - 'Monopoly' => 'monopoly', - 'Monsoon' => 'monsoon', - 'Monster Energy' => 'monster-energy', - 'Monster High' => 'monster-high', - 'Monster Hunter' => 'monster-hunter', - 'Monster Hunter World' => 'monster-hunter-world', - 'Mont Blanc' => 'mont-blanc', - 'Mop' => 'mop', - 'Morphy Richards' => 'morphy-richards', - 'Mortal Kombat' => 'mortal-kombat', - 'Mortal Kombat 11' => 'mortal-kombat-11', - 'Mortgage' => 'mortgage', - 'Moschino' => 'moschino', - 'Moses Basket' => 'moses-basket', - 'MOT' => 'mot', - 'Motherboard' => 'motherboard', - 'Moto 360' => 'moto-360', - 'Moto E' => 'moto-e', - 'Moto G' => 'moto-g', - 'Moto G4' => 'moto-g4', - 'Moto G5' => 'moto-g5', - 'Moto G6' => 'moto-g6', - 'Moto G7' => 'moto-g7', - 'Motorcycle' => 'motorcycle', - 'Motorcycle Accessories' => 'motorcycle-accessories', - 'Motorcycle Helmet' => 'motorcycle-helmet', - 'Motorola' => 'motorola', - 'Motorola Smartphone' => 'motorola-smartphone', - 'Moto X' => 'moto-x', - 'Moto Z' => 'moto-z', - 'Mountain Bike' => 'mountain-bike', - 'Mouse & Keyboard Bundles' => 'mouse-and-keyboard-bundle', - 'Mouse Mat' => 'mouse-mat', - 'Mouthwash' => 'mouthwash', - 'Movie and TV Box Set' => 'box-set', - 'Movies & Series' => 'movie', - 'MP3 Player' => 'mp3-player', - 'Mr Kipling' => 'mr-kipling', - 'Mr Men' => 'mr-men', - 'MSI' => 'msi', - 'MSI Laptop' => 'msi-laptop', - 'Muc-Off' => 'muc-off', - 'Mug' => 'mug', - 'Muller' => 'muller', - 'Multi-Room Audio System' => 'multi-room-audio-system', - 'Multitool' => 'multitool', - 'Museums' => 'museums', - 'Music' => 'music', - 'Musical Instruments' => 'musical-instrument', - 'Music App' => 'music-app', - 'Music Streaming' => 'music-streaming', - 'My Little Pony' => 'my-little-pony', - 'Nail Gun' => 'nail-gun', - 'Nail Polish' => 'nail-polish', - 'Nails' => 'nails', - 'Nails Inc.' => 'nails-inc', - 'Nakd' => 'nakd', - 'Nando's' => 'nandos', - 'Nappy' => 'nappy', - 'NAS' => 'nas', - 'National Express Ticket' => 'national-express', - 'National Trust' => 'national-trust', - 'Nature Observation' => 'nature-observation', - 'NatWest' => 'natwest', - 'NBA 2K' => 'nba-2k', - 'NBA Live' => 'nba', - 'Necklace' => 'necklace', - 'Need for Speed' => 'need-for-speed', - 'Need for Speed: Payback' => 'need-for-speed-payback', - 'Need for Speed Heat' => 'need-for-speed-heat', - 'Neff' => 'neff', - 'Nerf Guns' => 'nerf', - 'Nescafé Azera' => 'azera', - 'Nescafé Coffee' => 'nescafe', - 'Nespresso' => 'nespresso', - 'Nespresso Coffee Machine' => 'nespresso-coffee-machine', - 'Nest Hello' => 'nest-hello', - 'Nestlé' => 'nestle', - 'Nest Learning Thermostat' => 'nest-learning-thermostat', - 'Nestlé Cheerios' => 'cheerios', - 'Nestlé Shreddies' => 'shreddies', - 'Netatmo' => 'netatmo', - 'Netflix' => 'netflix', - 'Netgear' => 'netgear', - 'Netgear Arlo' => 'arlo', - 'New Balance' => 'new-balance', - 'New Balance Trainers' => 'new-balance-trainers', - 'New Look' => 'new-look', - 'Newspapers' => 'newspapers', - 'Nextbase' => 'nextbase', - 'NFL' => 'nfl', - 'NHL' => 'nhl', - 'NHL 20' => 'nhl-20', - 'NHS' => 'nhs', - 'NieR: Automata' => 'nier', - 'Night Light' => 'night-light', - 'Nike' => 'nike', - 'Nike Air Max' => 'nike-air-max', - 'Nike Air Max 200' => 'nike-air-max-200', - 'Nike Air Max 270' => 'nike-air-max-270', - 'Nike Air Max 720' => 'nike-air-max-720', - 'Nike Free' => 'nike-free', - 'Nike Huarache' => 'nike-huarache', - 'Nike Jordan' => 'jordan', - 'Nike Presto' => 'nike-presto', - 'Nike Roshe' => 'nike-roshe', - 'Nike Trainers' => 'nike-shoes', - 'Nikon' => 'nikon', - 'Nikon Camera' => 'nikon-camera', - 'Nikon Coolpix' => 'nikon-coolpix', - 'Nikon D3400' => 'nikon-d3400', - 'Nikon Lens' => 'nikon-lens', - 'Nilfisk' => 'nilfisk', - 'Ni No Kuni' => 'ni-no-kuni', - 'Ni No Kuni: Wrath of the White Witch' => 'ni-no-kuni-white-witch', - 'Ni No Kuni II: Revenant Kingdom' => 'ni-no-kuni-2', - 'Nintendo' => 'nintendo', - 'Nintendo 2DS' => '2ds', - 'Nintendo 3DS' => '3ds', - 'Nintendo 3DS Game' => '3ds-games', - 'Nintendo 3DS XL' => 'nintendo-3ds-xl', - 'Nintendo Accessories' => 'nintendo-accessories', - 'Nintendo Classic Mini' => 'nintendo-classic-mini', - 'Nintendo DS Game' => 'ds-games', - 'Nintendo Labo' => 'switch-labo', - 'Nintendo Switch' => 'nintendo-switch', - 'Nintendo Switch Accessories' => 'switch-accessories', - 'Nintendo Switch Case' => 'switch-case', - 'Nintendo Switch Controller' => 'switch-controller', - 'Nintendo Switch Game' => 'switch-game', - 'Nintendo Switch Joy-Con' => 'switch-joy-con', - 'Nintendo Switch Lite' => 'nintendo-switch-lite', - 'Nintendo Switch Pro Controller' => 'switch-pro-controller', - 'Nioh' => 'nioh', - 'Nissan' => 'nissan', - 'Nivea' => 'nivea', - 'No7' => 'no7', - 'Noise Cancelling Headphones' => 'noise-cancelling-headphones', - 'Nokia' => 'nokia', - 'Nokia Smartphones' => 'nokia-mobile', - 'No Man's Sky' => 'no-man-s-sky', - 'Noodles' => 'noodles', - 'Norton' => 'norton', - 'Now' => 'now-tv', - 'Numatic' => 'numatic', - 'Nursery' => 'nursery', - 'Nutella' => 'nutella', - 'NutriBullet' => 'nutribullet', - 'Nutri Ninja' => 'nutri-ninja', - 'Nuts' => 'nuts', - 'Nvidia' => 'nvidia', - 'Nvidia GeForce' => 'geforce', - 'Nvidia Shield' => 'nvidia-shield', - 'NYX' => 'nyx', - 'NZXT' => 'nzxt', - 'O2' => 'o2', - 'O2 Refresh' => 'o2-refresh', - 'Oakley' => 'oakley', - 'Octonauts' => 'octonauts', - 'Oculus Game' => 'oculus-game', - 'Oculus Go' => 'oculus-go', - 'Oculus Quest' => 'oculus-quest', - 'Oculus Rift' => 'oculus', - 'Oculus Rift S' => 'oculus-rift-s', - 'Odeon' => 'odeon', - 'Office' => 'office', - 'Office Chair' => 'office-chair', - 'Official Announcements' => 'official-announcements', - 'Olay' => 'olay', - 'OLED TV' => 'oled', - 'Olive Oil' => 'olive-oil', - 'Olympus' => 'olympus', - 'Omega Seamaster' => 'omega-seamaster', - 'Omega Speedmaster' => 'omega-speedmaster', - 'Omega Watches' => 'omega-watch', - 'OnePlus 3' => 'oneplus-3', - 'OnePlus 5' => 'oneplus-5', - 'OnePlus 6' => 'oneplus-6', - 'OnePlus 6T' => 'oneplus-6t', - 'OnePlus 7' => 'oneplus-7', - 'OnePlus 7 Pro' => 'oneplus-7-pro', - 'OnePlus 7T' => 'oneplus-7t', - 'OnePlus 7T Pro' => 'one-plus-7t-pro', - 'OnePlus 8' => 'oneplus-8', - 'OnePlus 8 Pro' => 'oneplus-8-pro', - 'OnePlus 8T' => 'oneplus-8t', - 'OnePlus 9' => 'oneplus-9', - 'OnePlus 9 Pro' => 'oneplus-9-pro', - 'OnePlus Nord' => 'oneplus-nord', - 'OnePlus Nord N10 5G' => 'oneplus-n10', - 'OnePlus Nord N100' => 'oneplus-n100', - 'OnePlus Smartphone' => 'oneplus', - 'Onesie' => 'onesie', - 'Onkyo' => 'onkyo', - 'Online Courses' => 'online-courses', - 'Operating System' => 'operating-system', - 'Oppo Find X2 Lite' => 'oppo-find-x2-lite', - 'Oppo Find X2 Neo' => 'oppo-find-x2-neo', - 'Oppo Find X2 Pro' => 'oppo-find-x2-pro', - 'Oppo Reno' => 'oppo-reno', - 'Oppo Reno4 5G' => 'oppo-reno4', - 'Oppo Reno4 Z 5G' => 'oppo-reno4-z', - 'Oppo Smartphone' => 'oppo-smartphone', - 'Opticians' => 'opticians', - 'Optoma' => 'optoma', - 'Oral-B' => 'oral-b', - 'Oral-B Toothbrush' => 'oral-b-toothbrush', - 'Oreo' => 'oreo', - 'Origin' => 'origin', - 'Original Penguin' => 'penguin', - 'Orla Kiely' => 'orla-kiely', - 'Osprey' => 'osprey', - 'Osram' => 'osram', - 'Other' => 'other-deals', - 'Ottoman' => 'ottoman', - 'Oukitel' => 'oukitel', - 'Outdoor Clothing' => 'outdoor-clothing', - 'Outdoor Lighting' => 'outdoor-lighting', - 'Outdoor Sports & Camping' => 'outdoor', - 'Outdoor Toys' => 'outdoor-toys', - 'Outlast' => 'outlast', - 'Outlet' => 'outlet', - 'Outwell' => 'outwell', - 'Oven' => 'oven', - 'Overcooked' => 'overcooked', - 'Overcooked 2' => 'overcooked-2', - 'Overwatch' => 'overwatch', - 'Oyster Card' => 'oyster', - 'Package Holidays' => 'holiday', - 'Paco Rabanne' => 'paco-rabanne', - 'Paco Rabanne 1 Million' => 'paco-rabanne-1-million', - 'Paco Rabanne Lady Million' => 'lady-million', - 'Paddling Pool' => 'paddling-pool', - 'Padlock' => 'padlock', - 'Paint' => 'paint', - 'Paint Brush' => 'paint-brush', - 'Pampers' => 'pampers', - 'Panasonic' => 'panasonic', - 'Panasonic Camera' => 'panasonic-camera', - 'Panasonic Lumix' => 'lumix', - 'Panasonic TV' => 'panasonic-tv', - 'Pandora' => 'pandora', - 'Panini' => 'panini', - 'Panini Stickers' => 'panini-stickers', - 'Papa Johns' => 'papa-johns', - 'Paper Mario' => 'paper-mario', - 'Parasol' => 'parasol', - 'Parcel and Delivery Services' => 'parcel', - 'Parka' => 'parka', - 'Parking' => 'parking', - 'Parrot' => 'parrot', - 'Paul Smith' => 'paul-smith', - 'PAW Patrol' => 'paw-patrol', - 'Payday' => 'payday', - 'Payday 2' => 'payday-2', - 'PAYG' => 'payg', - 'Pay Monthly' => 'pay-monthly', - 'PC' => 'pc', - 'PC Case' => 'pc-case', - 'PC Game' => 'pc-game', - 'PC Gaming Accessories' => 'pc-gaming-accessories', - 'PC Gaming Systems' => 'pc-gaming-systems', - 'PC Mouse' => 'mouse', - 'PC Parts' => 'pc-parts', - 'Peanut Butter' => 'peanut-butter', - 'Peanuts' => 'peanuts', - 'Pedometer' => 'pedometer', - 'Pentax' => 'pentax', - 'Peppa Pig' => 'peppa-pig', - 'PepperBonus' => 'pepperbonus', - 'Pepsi' => 'pepsi', - 'Perfume' => 'perfume', - 'Persil' => 'persil', - 'Persona' => 'persona', - 'Persona 5' => 'persona-5', - 'Personal Care & Hygiene' => 'personal-care-hygiene', - 'Petrol and Diesel' => 'petrol', - 'Pet Supplies' => 'pets', - 'Peugeot' => 'peugeot', - 'PG Tips' => 'pg-tips', - 'Philips' => 'philips', - 'Philips Alarm Clock' => 'philips-alarm-clock', - 'Philips Avent' => 'avent', - 'Philips Hue' => 'philips-hue', - 'Philips Lumea' => 'lumea', - 'Philips OneBlade' => 'philips-one-blade', - 'Philips Senseo' => 'philips-senseo', - 'Philips Senseo Coffee Machine' => 'philips-senseo-coffee-machine', - 'Philips Shaver' => 'philips-shaver', - 'Philips Sonicare' => 'sonicare', - 'Philips TV' => 'philips-tv', - 'Phone Holder' => 'phone-holder', - 'Phones & Accessories' => 'phone', - 'Photo & Cameras' => 'photo-video', - 'Photo & Video App' => 'photo-video-app', - 'Photo Editing' => 'photo-editing', - 'Photo Frame' => 'photo-frame', - 'Photo Paper' => 'photo-paper', - 'Piano' => 'piano', - 'Picnic & Outdoor Cooking' => 'picnic', - 'Pikmin 3 Deluxe' => 'pikmin-3-deluxe', - 'Pillow' => 'pillow', - 'Pimm's' => 'pimms', - 'Pioneer' => 'pioneer', - 'Pirate Toys' => 'pirates', - 'PIR Lights' => 'pir', - 'Pixel C' => 'pixel-c', - 'Piz Buin' => 'piz-buin', - 'Pizza' => 'pizza', - 'Pizza Stone' => 'pizza-stone', - 'Planer' => 'planer', - 'Planet Earth' => 'planet-earth', - 'Plant' => 'plant', - 'Plant Pot' => 'plant-pots', - 'Plants vs. Zombies: Battle for Neighborville' => 'battle-for-neighborville', - 'Plants vs Zombies' => 'plants-vs-zombies', - 'Play-Doh' => 'play-doh', - 'PlayerUnknown's Battlegrounds' => 'playerunknown-s-battlegrounds', - 'Playhouse' => 'playhouse', - 'Playing Cards' => 'playing-cards', - 'Playmat' => 'playmat', - 'Playmobil' => 'playmobil', - 'Playmobil Advent Calendar' => 'playmobil-advent-calendar', - 'PlayStation' => 'playstation', - 'PlayStation 5 DualSense Controller' => 'ps5-controller', - 'PlayStation Accessories' => 'playstation-accessories', - 'PlayStation Classic' => 'playstation-classic', - 'PlayStation Move' => 'playstation-move', - 'PlayStation Now' => 'playstation-now', - 'PlayStation Plus' => 'playstation-plus', - 'PlayStation VR' => 'playstation-vr', - 'PlayStation VR Aim Controller' => 'aim-controller-ps4', - 'Pliers' => 'pliers', - 'Plumbing & Fittings' => 'plumbing-and-fitting', - 'Plus Size' => 'plus-size', - 'PNY' => 'pny', - 'POCO F2 Pro' => 'poco-f2-pro', - 'POCO F3' => 'poco-f3', - 'Poco M3' => 'poco-m3', - 'POCO X3' => 'poco-x3', - 'POCO X3 Pro' => 'poco-x3-pro', - 'Pokémon' => 'pokemon', - 'Pokémon: Let's Go' => 'pokemon-lets-go', - 'Pokémon Go' => 'pokemon-go', - 'Pokemon Sword and Shield' => 'pokemon-sword-and-shield', - 'Pokémon Ultra Sun and Ultra Moon' => 'pokemon-ultra-sun-ultra-moon', - 'Poker' => 'poker', - 'Pokken Tournament' => 'pokken-tournament', - 'Polaroid' => 'polaroid', - 'Police Toys' => 'police', - 'Polo Shirt' => 'polo-shirt', - 'Pool' => 'pool', - 'Pool & Snooker' => 'pool-table', - 'Popcorn' => 'popcorn', - 'Pork' => 'pork', - 'Porridge & Oats' => 'porridge-and-oats', - 'Portable Wireless Speaker' => 'wireless-speaker', - 'Poster' => 'poster', - 'Pots and Pans' => 'pan', - 'Potty' => 'potty', - 'Power Bank' => 'power-bank', - 'Powerbeats Pro' => 'powerbeats-pro', - 'Power Dental Flosser' => 'floss', - 'Powerline' => 'powerline', - 'Power Rangers' => 'power-rangers', - 'Power Tool' => 'power-tool', - 'Prada' => 'prada', - 'Pram' => 'pram', - 'Pregnancy' => 'pregnancy', - 'Prescription Glasses' => 'prescription-glasses', - 'Pressure Cooker' => 'pressure-cooker', - 'Pressure Washer' => 'pressure-washer', - 'Price Glitch' => 'price-glitch', - 'Prime Gaming' => 'twitch', - 'Pringles' => 'pringles', - 'Printer & Printer Supplies' => 'printer', - 'Printer Supplies' => 'printer-supplies', - 'Productivity App' => 'productivity-app', - 'Pro Evolution Soccer' => 'pro-evolution-soccer', - 'Pro Evolution Soccer 2018' => 'pro-evolution-soccer-2018', - 'Pro Evolution Soccer 2019' => 'pro-evolution-soccer-2019', - 'Pro Evolution Soccer 2020' => 'pes-2020', - 'Project Cars' => 'project-cars', - 'Project Cars 2' => 'project-cars-2', - 'Projector' => 'projector', - 'Protein' => 'protein', - 'Protein Bars' => 'protein-bars', - 'Protein Shaker' => 'shaker', - 'PS4' => 'ps4-slim', - 'PS4 Camera' => 'ps4-camera', - 'PS4 Controller' => 'ps4-controller', - 'PS4 Games' => 'ps4-games', - 'PS4 Headset' => 'ps4-headset', - 'PS4 Pro' => 'ps4-pro', - 'PS5' => 'ps5', - 'PS5 Games' => 'ps5-game', - 'PSU' => 'psu', - 'Public Transport' => 'public-transport', - 'Pukka' => 'pukka', - 'Pulse Light Epilator' => 'pulse-light-epilator', - 'Puma' => 'puma', - 'Puma Trainers' => 'puma-trainers', - 'Puppy Supplies' => 'puppy', - 'Purse' => 'purse', - 'Pushchair' => 'pushchair', - 'Pushchairs and Strollers' => 'baby-transport', - 'Puzzle' => 'puzzle', - 'PVR' => 'pvr', - 'Pyjamas' => 'pyjamas', - 'Pyrex' => 'pyrex', - 'Q Acoustics' => 'q-acoustics', - 'QNAP' => 'qnap', - 'Qualcast' => 'qualcast', - 'Quality Street' => 'quality-street', - 'Quantum Break' => 'quantum-break', - 'Quechua' => 'quechua', - 'Quick Charge' => 'quick-charge', - 'Quiksilver' => 'quiksilver', - 'Quinny' => 'quinny', - 'Quorn' => 'quorn', - 'Rab' => 'rab', - 'Radeon RX 480' => 'rx-480', - 'Radeon RX 5700' => 'radeon-rx-5700', - 'Radeon RX 5700 XT' => 'radeon-rx-5700-xt', - 'Radeon RX 6800' => 'radeon-rx-6800', - 'Radeon RX 6800 XT' => 'radeon-rx-6800-xt', - 'Radeon RX 6900 XT' => 'radeon-rx-6900-xt', - 'Radiator' => 'radiator', - 'Radio' => 'radio', - 'Radley' => 'radley', - 'Rage 2' => 'rage-2', - 'Railcard' => 'railcard', - 'Rainbow Six' => 'rainbow-six', - 'Rake' => 'rake', - 'Ralph Lauren' => 'ralph-lauren', - 'RAM' => 'ram', - 'Raspberry Pi' => 'raspberry-pi', - 'Ratchet' => 'ratchet', - 'Ratchet and Clank' => 'ratchet-and-clank', - 'Rattan Garden Furniture' => 'rattan', - 'RAVPower' => 'ravpower', - 'Ray Ban' => 'ray-ban', - 'Razer' => 'razer', - 'Razor' => 'razor', - 'Razor Blade' => 'razor-blade', - 'Real Madrid' => 'real-madrid', - 'Realme Smartphones' => 'realme-smartphone', - 'Real Techniques' => 'real-techniques', - 'Recliner' => 'recliner', - 'ReCore' => 'recore', - 'Recreational Sports' => 'recreational-sports', - 'Red Bull' => 'red-bull', - 'Red Dead Redemption' => 'red-dead-redemption', - 'Red Dead Redemption 2' => 'red-dead-redemption-2', - 'Redex' => 'redex', - 'Red Kite' => 'red-kite', - 'Reebok' => 'reebok', - 'Reese's' => 'reeses', - 'Regatta' => 'regatta', - 'Regina' => 'regina', - 'Remington' => 'remington', - 'Remote Control Car' => 'remote-control-car', - 'Renault' => 'renault', - 'Resident Evil' => 'resident-evil', - 'Resident Evil 2' => 'resident-evil-2', - 'Resident Evil 7' => 'resident-evil-7', - 'Restaurant, Café & Pub' => 'restaurant', - 'Retailer Offers and Issues' => 'retailer-offers-and-issues', - 'Ribena' => 'ribena', - 'Rice' => 'rice', - 'Rice Cooker' => 'rice-cooker', - 'Rick and Morty' => 'rick-and-morty', - 'Ricoh' => 'ricoh', - 'Ride On' => 'ride-on', - 'Ring' => 'ring', - 'Ring Door View Cam' => 'ring-door-view-cam', - 'Ring Fit Adventures' => 'ring-fit-adventures', - 'Ring Stick Up Cam' => 'ring-stick-up-cam', - 'Ring Video Doorbell' => 'ring-video-doorbell', - 'Ring Video Doorbell 2' => 'ring-video-doorbell-2', - 'Ring Video Doorbell 3' => 'ring-video-doorbell-3', - 'Ring Video Doorbell Pro' => 'ring-video-doorbell-pro', - 'Road Bike' => 'road-bike', - 'Roaming' => 'roaming', - 'Robinsons' => 'robinsons', - 'Robotic Lawnmower' => 'robotic-lawnmower', - 'Robot Vacuum Cleaner' => 'robot-vacuum-cleaner', - 'Rock Band' => 'rock-band', - 'Rocket League' => 'rocket-league', - 'Rocking Horse' => 'rocking-horse', - 'Rogue One: A Star Wars Story' => 'rogue-one', - 'Roku' => 'roku', - 'Rolex' => 'rolex', - 'Rollerskates' => 'skate', - 'Ronseal' => 'ronseal', - 'Roof Box' => 'roof-box', - 'Roses' => 'roses', - 'Rotary' => 'rotary', - 'Router' => 'router', - 'Rowenta' => 'rowenta', - 'RTX 2060' => 'rtx-2060', - 'RTX 2070' => 'rtx-2070', - 'RTX 2080' => 'rtx-2080', - 'RTX 2080 Ti' => 'rtx-2080-ti', - 'RTX 3070' => 'rtx-3070', - 'RTX 3080' => 'rtx-3080', - 'RTX 3090' => 'rtx-3090', - 'Rug' => 'rug', - 'Rugby' => 'rugby', - 'Rum' => 'rum', - 'Running' => 'running', - 'Running Shoes' => 'running-shoes', - 'Russell Hobbs' => 'russell-hobbs', - 'RX 570' => 'rx-570', - 'RX 580' => 'rx-580', - 'RX 590' => 'rx-590', - 'RX Vega 56' => 'rx-vega-56', - 'RX Vega 64' => 'rx-vega-64', - 'Ryanair' => 'ryanair', - 'Ryobi' => 'ryobi', - 'Safari' => 'safari', - 'Safety Boots' => 'safety-boots', - 'Sage by Heston Blumenthal' => 'sage', - 'Saints Row' => 'saints-row', - 'Saitek' => 'saitek', - 'Sale' => 'sale', - 'Salmon' => 'salmon', - 'Salomon' => 'salomon', - 'Salter' => 'salter', - 'Samsonite' => 'samsonite', - 'Samsung' => 'samsung', - 'Samsung Ecobubble' => 'ecobubble', - 'Samsung Fridge' => 'samsung-fridge', - 'Samsung Galaxy' => 'samsung-galaxy', - 'Samsung Galaxy A10' => 'samsung-galaxy-a10', - 'Samsung Galaxy A20e' => 'samsung-galaxy-a20e', - 'Samsung Galaxy A40' => 'samsung-galaxy-a40', - 'Samsung Galaxy A42 5G' => 'samsung-galaxy-a42-5g', - 'Samsung Galaxy A50' => 'samsung-galaxy-a50', - 'Samsung Galaxy A51' => 'samsung-galaxy-a51', - 'Samsung Galaxy A52 5G' => 'samsung-galaxy-a52', - 'Samsung Galaxy A60' => 'samsung-galaxy-a60', - 'Samsung Galaxy A70' => 'samsung-galaxy-a70', - 'Samsung Galaxy A71' => 'samsung-galaxy-a71', - 'Samsung Galaxy A72' => 'samsung-galaxy-a72', - 'Samsung Galaxy A80' => 'samsung-galaxy-a80', - 'Samsung Galaxy A90' => 'samsung-galaxy-a90', - 'Samsung Galaxy Buds' => 'samsung-galaxy-buds', - 'Samsung Galaxy Buds+' => 'samsung-galaxy-buds-plus', - 'Samsung Galaxy Buds Live' => 'samsung-galaxy-buds-live', - 'Samsung Galaxy Buds Pro' => 'samsung-galaxy-buds-pro', - 'Samsung Galaxy Fold' => 'samsung-galaxy-fold', - 'Samsung Galaxy J5' => 'galaxy-j5', - 'Samsung Galaxy Note' => 'samsung-galaxy-note', - 'Samsung Galaxy Note 8' => 'samsung-galaxy-note-8', - 'Samsung Galaxy Note 9' => 'samsung-galaxy-note-9', - 'Samsung Galaxy Note 10' => 'samsung-galaxy-note-10', - 'Samsung Galaxy Note 10+' => 'samsung-galaxy-note-10-plus', - 'Samsung Galaxy Note20' => 'samsung-galaxy-note20', - 'Samsung Galaxy Note20 Ultra' => 'samsung-galaxy-note20-ultra', - 'Samsung Galaxy S6' => 'samsung-galaxy-s6', - 'Samsung Galaxy S7' => 'samsung-galaxy-s7', - 'Samsung Galaxy S7 Edge' => 'samsung-galaxy-s7-edge', - 'Samsung Galaxy S8' => 'samsung-galaxy-s8', - 'Samsung Galaxy S8+' => 'samsung-s8-plus', - 'Samsung Galaxy S9' => 'samsung-galaxy-s9', - 'Samsung Galaxy S9 Plus' => 'samsung-s9-plus', - 'Samsung Galaxy S10' => 'samsung-galaxy-s10', - 'Samsung Galaxy S10 Lite' => 'samsung-galaxy-s10-lite', - 'Samsung Galaxy S10 Plus' => 'samsung-galaxy-s10-plus', - 'Samsung Galaxy S10e' => 'samsung-galaxy-s10e', - 'Samsung Galaxy S20' => 'samsung-galaxy-s20', - 'Samsung Galaxy S20 FE' => 'samsung-galaxy-s20-fe', - 'Samsung Galaxy S20 Ultra' => 'samsung-galaxy-s20-ultra', - 'Samsung Galaxy S20+' => 'samsung-galaxy-s20-plus', - 'Samsung Galaxy S21 5G' => 'samsung-galaxy-s21-5g', - 'Samsung Galaxy S21 Ultra 5G' => 'samsung-galaxy-s21-ultra-5g', - 'Samsung Galaxy S21+ 5G' => 'samsung-galaxy-s21-plus-5g', - 'Samsung Galaxy Tab' => 'samsung-galaxy-tab', - 'Samsung Galaxy Tab A' => 'samsung-galaxy-tab-a', - 'Samsung Galaxy Tab A7' => 'samsung-galaxy-tab-a7', - 'Samsung Galaxy Tab S' => 'samsung-galaxy-tab-s', - 'Samsung Galaxy Tab S4' => 'samsung-galaxy-tab-s4', - 'Samsung Galaxy Tab S5e' => 'samsung-galaxy-tab-s5e', - 'Samsung Galaxy Tab S6' => 'samsung-galaxy-tab-s6', - 'Samsung Galaxy Watch' => 'samsung-galaxy-watch', - 'Samsung Galaxy Watch3' => 'samsung-galaxy-watch3', - 'Samsung Galaxy Watch Active2' => 'samsung-galaxy-watch-active-2', - 'Samsung Gear' => 'samsung-gear', - 'Samsung Gear S3' => 'gear-s3', - 'Samsung Gear VR' => 'samsung-gear-vr', - 'Samsung Headphones' => 'samsung-headphones', - 'Samsung Monitor' => 'samsung-monitor', - 'Samsung QLED TVs' => 'samsung-qled-tv', - 'Samsung Smartphone' => 'samsung-smartphone', - 'Samsung SSD' => 'samsung-ssd', - 'Samsung The Frame TV' => 'samsung-the-frame', - 'Samsung TV' => 'samsung-tv', - 'Samsung Washing Machine' => 'samsung-washing-machine', - 'Samsung Watch' => 'samsung-watch', - 'Sandals' => 'sandals', - 'Sander' => 'sander', - 'SanDisk' => 'sandisk', - 'SanDisk SSD' => 'sandisk-ssd', - 'Sand Pit' => 'sand-pit', - 'Sandwich Maker' => 'sandwich', - 'San Miguel' => 'san-miguel', - 'Santander' => 'santander', - 'Satchel' => 'satchel', - 'Sat Nav' => 'sat-nav', - 'Sauce' => 'sauce', - 'Saw' => 'saw', - 'Scalextric' => 'scalextric', - 'Scanner' => 'scanner', - 'School Bag' => 'school-bag', - 'School Supplies' => 'school', - 'School Uniform' => 'school-uniform', - 'Schwalbe' => 'schwalbe', - 'Scooby Doo' => 'scooby-doo', - 'Scooter' => 'scooter', - 'Scotch Whisky' => 'scotch', - 'Scrabble' => 'scrabble', - 'Screen Protector' => 'screen-protector', - 'Screenwash' => 'screenwash', - 'Screwdriver' => 'screwdriver', - 'Screws' => 'screws', - 'SD Cards' => 'sd-card', - 'SDHC' => 'sdhc', - 'SDXC' => 'sdxc', - 'Seagate' => 'seagate', - 'Sea Life' => 'sea-life', - 'Sea of Thieves' => 'sea-of-thieves', - 'Season Pass' => 'season-pass', - 'Seaworld' => 'seaworld', - 'Security Camera' => 'security-camera', - 'Seeds & Bulbs' => 'seeds-and-bulbs', - 'Sega' => 'sega', - 'SEGA Mega Drive Mini' => 'sega-mega-drive-mini', - 'Segway' => 'segway', - 'Seiko' => 'seiko', - 'Sekiro: Shadows Die Twice' => 'sekiro', - 'Sekonda' => 'sekonda', - 'Selfie Stick' => 'selfie-stick', - 'Sennheiser' => 'sennheiser', - 'Sennheiser Headphones' => 'sennheiser-headphones', - 'Sensodyne' => 'sensodyne', - 'Server' => 'server', - 'Services & Contracts' => 'services-contracts', - 'Services and Subscriptions' => 'service-contract', - 'Sewing' => 'sewing', - 'Sewing Machine' => 'sewing-machine', - 'Sex Toys' => 'sex-toys', - 'Shadow of the Tomb Raider' => 'shadow-of-the-tomb-raider', - 'Shampoo' => 'shampoo', - 'Shark' => 'shark', - 'Shark DuoClean' => 'shark-duoclean', - 'Shark Vacuum Cleaner' => 'shark-vacuum-cleaner', - 'Sharp' => 'sharp', - 'Sharpener' => 'sharpener', - 'Sharpie' => 'sharpie', - 'Shaver' => 'shaver', - 'Shaving & Beard Care' => 'shaving', - 'Shaving, Trimming, & Hair Removal' => 'hair-removal', - 'Shaving Foam' => 'shaving-foam', - 'Shears' => 'shears', - 'Sheba' => 'sheba', - 'Shed' => 'shed', - 'Shelter' => 'shelter', - 'Shelves' => 'shelves', - 'Shenmue I & II' => 'shenmue-one-and-two', - 'Shenmue III' => 'shenmue-3', - 'Shenmue Series' => 'shenmue-series', - 'Shimano' => 'shimano', - 'Shirt' => 'shirt', - 'Shoe Rack' => 'shoe-rack', - 'Shoes' => 'shoe', - 'Shopkins' => 'shopkins', - 'Shortbread' => 'shortbread', - 'Shorts' => 'shorts', - 'Short Trip' => 'break', - 'Shoulder Bag' => 'shoulder-bag', - 'Shovel' => 'shovel', - 'Shower Curtain' => 'shower-curtain', - 'Shower Enclosure' => 'shower-enclosure', - 'Shower Fittings' => 'shower', - 'Shower Gel' => 'shower-gel', - 'Shower Head' => 'shower-head', - 'Shredder' => 'shredder', - 'Side-by-Side-Fridge' => 'side-by-side-fridge', - 'Sideboard' => 'sideboard', - 'Sid Meier's Civilization VI' => 'civilization-vi', - 'Siemens' => 'siemens', - 'Siemens Washing Machine' => 'siemens-washing-machine', - 'Sigma' => 'sigma', - 'Silentnight' => 'silentnight', - 'Silvercrest' => 'silvercrest', - 'Silver Cross' => 'silver-cross', - 'Sim Free' => 'sim-free', - 'Sim Only' => 'sim-only', - 'Simplehuman' => 'simplehuman', - 'Simpsons' => 'simpsons', - 'Single Malt' => 'single-malt', - 'Sink' => 'sink', - 'Sistema' => 'sistema', - 'Skateboard' => 'skateboard', - 'Skating' => 'skating', - 'Skechers' => 'skechers', - 'Skiing' => 'ski', - 'Skin Care' => 'skincare', - 'Skittles' => 'skittles', - 'Skoda' => 'skoda', - 'Skullcandy' => 'skullcandy', - 'Sky' => 'sky', - 'Sky Cinema' => 'sky-cinema', - 'Skylanders' => 'skylanders', - 'Skylanders Battlecast' => 'skylanders-battlecast', - 'Skylanders Imaginators' => 'skylanders-imaginators', - 'Sleeping Bag' => 'sleeping-bag', - 'Sleeping Dogs' => 'sleeping-dogs', - 'Sleepwear' => 'sleepwear', - 'Slide' => 'slide', - 'Slimming World' => 'slimming-world', - 'Slippers' => 'slippers', - 'Slow Cooker' => 'slow-cooker', - 'Smart Clock' => 'clock', - 'Smart Doorbells' => 'smart-doorbell', - 'Smart Home' => 'smart-home', - 'Smart Light' => 'smart-light', - 'Smart Lock' => 'smart-lock', - 'Smartphone Accessories' => 'smartphone-accessories', - 'Smartphone Case' => 'smartphone-case', - 'Smartphone under £200' => 'smartphone-under-200-pounds', - 'Smartphone under £400' => 'smartphone-under-400-pounds', - 'Smart Plugs' => 'smart-plugs', - 'Smart Speaker' => 'smart-speaker', - 'Smart Tech & Gadgets' => 'smart-tech', - 'Smart Thermostat' => 'thermostat', - 'SmartThings' => 'smartthings', - 'Smart TV' => 'smart-tv', - 'Smart Watch' => 'smartwatch', - 'Smeg' => 'smeg', - 'Smirnoff' => 'smirnoff', - 'Smoke Alarm' => 'smoke-alarm', - 'Smoothie' => 'smoothie', - 'Smoothie Maker' => 'smoothie-maker', - 'Snacks' => 'snacks', - 'Sneakers' => 'sneakers', - 'SNES Nintendo Classic Mini' => 'snes-nintendo-classic', - 'Snickers' => 'snickers', - 'Sniper Elite' => 'sniper-elite', - 'Snowboard' => 'snowboard', - 'Snow Boots' => 'snow-boots', - 'Soap' => 'soap', - 'Soap and Glory' => 'soap-and-glory', - 'Socket Set' => 'socket-set', - 'Socks' => 'socks', - 'SodaStream' => 'soda-stream', - 'Sofa' => 'sofa', - 'Soft Drinks' => 'soft-drinks', - 'Soft Toy' => 'soft-toy', - 'Software' => 'software', - 'Software & Apps' => 'software-apps', - 'Solar Lights' => 'solar-lights', - 'Soldering Iron' => 'soldering', - 'Sonic' => 'sonic', - 'Sonos' => 'sonos', - 'Sonos Beam' => 'sonos-beam', - 'Sonos Move' => 'sonos-move', - 'Sonos One' => 'sonos-one', - 'Sonos PLAY:1' => 'sonos-play-1', - 'Sonos PLAY:3' => 'sonos-play-3', - 'Sonos PLAY:5' => 'sonos-play-5', - 'Sonos PLAYBAR' => 'sonos-playbar', - 'Sonos PLAYBASE' => 'sonos-playbase', - 'Sony' => 'sony', - 'Sony Camera' => 'sony-camera', - 'Sony Headphones' => 'sony-headphones', - 'Sony Pulse 3D Wireless Headset' => 'pulse-3d-wireless-headsets', - 'Sony TV' => 'sony-tv', - 'Sony WF-1000XM3' => 'sony-wf1000xm3', - 'Sony WH-1000XM3' => 'sony-wh-1000xm3', - 'Sony WH-1000XM4' => 'sony-wh1000xm4', - 'Sony Xperia' => 'xperia', - 'Sony Xperia 5' => 'sony-xperia-5', - 'Sony Xperia 10' => 'sony-xperia-10', - 'Sony Xperia Xa' => 'sony-xperia-xa', - 'Sony Xperia Z3' => 'xperia-z3', - 'Sony Xperia Z5' => 'xperia-z5', - 'Soulcalibur' => 'soulcalibur', - 'Soundbar' => 'soundbar', - 'Soundbase' => 'soundbase', - 'Sound Card' => 'sound-card', - 'Soundmagic' => 'soundmagic', - 'Soup' => 'soup', - 'Soup Maker' => 'soup-maker', - 'Sous-Vide' => 'sousvide', - 'Southern Comfort' => 'southern-comfort', - 'South Park' => 'south-park', - 'Spa' => 'spa', - 'Spade' => 'spade', - 'Spanner' => 'spanner', - 'Speaker' => 'speakers', - 'Specialized' => 'specialized', - 'Speedo' => 'speedo', - 'Sphero' => 'sphero', - 'Spice Rack' => 'spice-rack', - 'Spiderman' => 'spiderman', - 'Spiralizer' => 'spiralizer', - 'Spirit & Liqueur' => 'spirits', - 'Spirit Level' => 'spirit-level', - 'Splatoon' => 'splatoon', - 'Sports & Outdoors' => 'sports-fitness', - 'Sports Events' => 'sports-events', - 'Sports Nutrition' => 'nutrition', - 'Spreads' => 'spreads', - 'Spyro Reignited Trilogy' => 'spyro-reignited-trilogy', - 'SSD' => 'ssd', - 'SSHD' => 'sshd', - 'Staedtler' => 'staedtler', - 'Stair Gate' => 'stair-gate', - 'Stanley' => 'stanley', - 'Stapler' => 'stapler', - 'Starbucks' => 'starbucks', - 'Starlink: Battle for Atlas' => 'starlink-battle-for-atlas', - 'Star Ocean' => 'star-ocean', - 'Star Trek' => 'star-trek', - 'Star Wars' => 'star-wars', - 'Star Wars: Battlefront' => 'star-wars-battlefront', - 'Star Wars: Battlefront II' => 'star-wars-battlefront-2', - 'Star Wars: Squadrons' => 'star-wars-squadrons', - 'Star Wars Jedi: Fallen Order' => 'star-wars-jedi-fallen-order', - 'Stationery' => 'stationery', - 'Stationery & Office Supplies' => 'stationery-office-supplies', - 'Staycation' => 'staycation', - 'Steak' => 'steak', - 'Steam Cleaner' => 'steam-cleaner', - 'Steam Controller' => 'steam-controller', - 'Steamer' => 'steamer', - 'Steam Gaming' => 'steam', - 'Steam Iron' => 'steam-iron', - 'Steam Link' => 'steam-link', - 'Steam Mop' => 'steam-mop', - 'SteelSeries' => 'steelseries', - 'Steering Wheel' => 'steering-wheel', - 'Stella' => 'stella', - 'Stool' => 'stool', - 'Storage Box' => 'storage-box', - 'Stormtrooper' => 'stormtrooper', - 'Straightener' => 'straightener', - 'Streaming' => 'streaming', - 'Street Fighter' => 'street-fighter', - 'Street Fighter V' => 'street-fighter-v', - 'Streetwear' => 'streetwear', - 'Strimmer' => 'strimmer', - 'Strongbow' => 'strongbow', - 'Student Discount' => 'student-discount', - 'Subwoofer' => 'subwoofer', - 'Suitcase' => 'suitcase', - 'Suncare' => 'suncare', - 'Sun Cream' => 'sun-cream', - 'Sunglasses' => 'sunglasses', - 'Superdry' => 'superdry', - 'Superfast Broadband' => 'superfast-broadband', - 'Superking' => 'superking', - 'Super Mario' => 'mario', - 'Super Mario 3D All-Stars' => 'super-mario-3d-all-stars', - 'Super Mario 3D World' => 'super-mario-3d-world', - 'Super Mario Maker 2' => 'super-mario-maker-2', - 'Super Mario Odyssey' => 'super-mario-odyssey', - 'Super Mario Party' => 'mario-party', - 'Supermarket' => 'supermarket', - 'Super Smash Bros.' => 'super-smash-bros', - 'Surf' => 'surf', - 'Swarovski' => 'swarovski', - 'Sweets' => 'sweets', - 'Swimming' => 'swimming', - 'Swimming Goggles' => 'goggles', - 'Swimwear' => 'swimwear', - 'Swing' => 'swing', - 'Swingball' => 'swingball', - 'Syberia' => 'syberia', - 'Sylvanian' => 'sylvanian', - 'Synology' => 'synology', - 'T-Mobile' => 't-mobile', - 'T-Shirt' => 't-shirt', - 'Table Lamp' => 'table-lamp', - 'Tablet' => 'tablet', - 'Tablet Accessories' => 'tablet-accessories', - 'Table Tennis' => 'table-tennis', - 'Tableware' => 'tableware', - 'Tacx' => 'tacx', - 'Tado' => 'tado', - 'Tag Heuer' => 'tag-heuer', - 'Takeaway and Food Delivery' => 'takeaway', - 'Tales of Vesperia: Definitive Edition' => 'tales-of-vesperia-definitive-edition', - 'Talisker' => 'talisker', - 'Talkmobile' => 'talkmobile', - 'Tamron' => 'tamron', - 'Tangle Teezer' => 'tangle-teezer', - 'Tank Top' => 'tank-top', - 'Tannoy' => 'tannoy', - 'Tanqueray' => 'tanqueray', - 'Tape' => 'tape', - 'Tassimo' => 'tassimo', - 'Tassimo Coffee Machine' => 'tassimo-coffee-machine', - 'tastecard' => 'tastecard', - 'Taxi' => 'taxi', - 'Tea' => 'tea', - 'Team Sonic Racing' => 'team-sonic-racing', - 'Team Sports' => 'team-sports', - 'Teapot' => 'teapot', - 'Technika' => 'technika', - 'Techwood' => 'techwood', - 'Ted Baker' => 'ted-baker', - 'Teddy Bear' => 'teddy-bear', - 'Teenage Mutant Ninja Turtles' => 'turtle', - 'Teeth Care' => 'teeth-care', - 'Teeth Whitening' => 'teeth-whitening', - 'Tefal' => 'tefal', - 'Tefal Actifry' => 'actifry', - 'Tefal Pan' => 'tefal-pan', - 'Tekken' => 'tekken', - 'Tekken 7' => 'tekken-7', - 'Telegraph' => 'telegraph', - 'Telescope' => 'telescope', - 'Telltale' => 'telltale', - 'Tennis' => 'tennis', - 'Tent' => 'tent', - 'Tequila' => 'tequila', - 'Tesco Clothing' => 'tesco-clothing', - 'Tesla' => 'tesla', - 'Tetris' => 'tetris', - 'Tetris 99' => 'tetris-99', - 'Theatre & Musical' => 'theatre', - 'The Beatles' => 'beatles', - 'The Big Bang Theory' => 'big-bang-theory', - 'The Crew' => 'the-crew', - 'The Dark Pictures: Anthology Man of Medan' => 'the-dark-pictures-anthology-man-of-medan', - 'The Elder Scrolls' => 'elder-scrolls', - 'The Elder Scrolls V: Skyrim' => 'skyrim', - 'The Evil Within' => 'the-evil-within', - 'The Evil Within 2' => 'the-evil-within-2', - 'The Last Guardian' => 'the-last-guardian', - 'The Last of Us' => 'the-last-of-us', - 'The Last of Us Part II' => 'the-last-of-us-part-2', - 'The Legend of Zelda' => 'zelda', - 'The Legend of Zelda: Breath of the Wild' => 'zelda-breath-of-the-wild', - 'The Legend of Zelda: Link's Awakening' => 'the-legend-of-zelda-links-awakening', - 'The Legend of Zelda: Skyward Sword HD' => 'the-legend-of-zelda-skyward-sword-hd', - 'Theme Park' => 'theme-park', - 'The North Face' => 'north-face', - 'The Outer Worlds' => 'the-outer-worlds', - 'Thermos Storage' => 'thermos', - 'The Sims' => 'sims', - 'The Sims 4' => 'the-sims-4', - 'The Sinking City' => 'the-sinking-city', - 'The Sun' => 'the-sun', - 'The Sunday Times' => 'sunday-times', - 'The Walking Dead' => 'walking-dead', - 'The Witcher' => 'witcher', - 'The Witcher 3' => 'the-witcher-3', - 'Thierry Mugler' => 'thierry-mugler', - 'Thomas Sabo' => 'thomas-sabo', - 'Thomas The Tank Engine' => 'thomas-the-tank', - 'Thornton's' => 'thorntons', - 'Thorpe Park' => 'thorpe-park', - 'Throw' => 'throw', - 'Thrustmaster' => 'thrustmaster', - 'Thule' => 'thule', - 'Tickets & Shows' => 'tickets-shows', - 'Tie' => 'tie', - 'Tights' => 'tights', - 'TIGI' => 'tigi', - 'Tilda' => 'tilda', - 'Tile' => 'tile', - 'Timberland' => 'timberland', - 'Timex' => 'timex', - 'Tissot' => 'tissot', - 'Tissues' => 'tissues', - 'Titanfall' => 'titanfall', - 'Titanfall 2' => 'titanfall-2', - 'Toaster' => 'toaster', - 'Toblerone' => 'toblerone', - 'Toddler Bed' => 'toddler-bed', - 'Toilet Brush' => 'brush', - 'Toilet Cleaner' => 'toilet', - 'Toilet Roll' => 'toilet-roll', - 'Toilet Seat' => 'toilet-seat', - 'Tokyo Laundry' => 'tokyo-laundry', - 'Tomb Raider' => 'tomb-raider', - 'Tom Clancy's' => 'tom-clancy', - 'Tom Clancy's: Ghost Recon' => 'ghost-recon', - 'Tom Clancy's Ghost Recon: Wildlands' => 'ghost-recon-wildlands', - 'Tom Clancy's Ghost Recon Breakpoint' => 'tom-clancys-ghost-recon-breakpoint', - 'Tom Clancy's The Division' => 'tom-clancy-the-division', - 'Tom Clancy's The Division 2' => 'tom-clancy-the-division-2', - 'Tom Ford' => 'tom-ford', - 'Tommee Tippee' => 'tommee-tippee', - 'Tommy Hilfiger' => 'tommy-hilfiger', - 'Toms' => 'toms', - 'TomTom' => 'tomtom', - 'Tonic Water' => 'tonic-water', - 'Tony Hawk's Pro Skater 1 + 2' => 'tony-hawks-pro-skater-1-2', - 'Tools' => 'tool', - 'Toothbrush' => 'toothbrush', - 'Toothpaste' => 'toothpaste', - 'Torch' => 'torch', - 'Torque Wrench' => 'torque-wrench', - 'Toshiba' => 'toshiba', - 'Toshiba Laptop' => 'toshiba-laptop', - 'Toshiba TV' => 'toshiba-tv', - 'Total War' => 'total-war', - 'Tottenham Hotspur F. C.' => 'tottenham', - 'Towel' => 'towel', - 'Toy Box' => 'toy-box', - 'Toy Cars' => 'toy-cars', - 'Toy Castle' => 'castle', - 'Toy Digger' => 'digger', - 'Toy Helicopter' => 'helicopter', - 'Toy Kitchen' => 'toy-kitchen', - 'Toy Mask' => 'mask', - 'Toyota' => 'toyota', - 'Toys' => 'toy', - 'Toy Story' => 'toy-story', - 'Toy Tractor' => 'tractor', - 'Toy Train' => 'train', - 'TP-Link' => 'tp-link', - 'TP-Link Archer' => 'archer', - 'TP-Link Router' => 'tp-link-router', - 'Tracksuit' => 'tracksuit', - 'Trainers' => 'trainers', - 'Trains & Buses' => 'train-and-bus-ticket', - 'Train Ticket' => 'train-ticket', - 'Trampoline' => 'trampoline', - 'Transcend' => 'transcend', - 'Transformers' => 'transformers', - 'Travel' => 'travel', - 'Travel App' => 'travel-app', - 'Travel Insurance' => 'travel-insurance', - 'Travelodge' => 'travelodge', - 'Travel System' => 'travel-system', - 'Treadmill' => 'treadmill', - 'TRESemmé' => 'tresemme', - 'Trespass' => 'trespass', - 'Triathlon' => 'triathlon', - 'Trike' => 'trike', - 'Trine 4' => 'trine-4', - 'Tripod' => 'tripod', - 'Tripp' => 'tripp', - 'Triton Shower' => 'triton', - 'Trolley Bag' => 'trolley', - 'Tropico 5' => 'tropico-5', - 'Tropico 6' => 'tropico-6', - 'Tropico Series' => 'tropico-deals', - 'Trousers' => 'trousers', - 'True Wireless Earbuds' => 'wireless-earphones', - 'Trunki' => 'trunki', - 'Tumble Dryer' => 'tumble-dryer', - 'Tuna' => 'tuna', - 'Turbo Trainer' => 'turbo-trainer', - 'Turntable' => 'turntable', - 'Turtle Beach' => 'turtle-beach', - 'TV' => 'tv', - 'TV & Video' => 'tv-video', - 'TV Accessories' => 'tv-accessories', - 'TV Mount' => 'tv-mount', - 'TV Series' => 'tv-series', - 'TV Stand' => 'tv-stand', - 'Twinings' => 'twinings', - 'Twin Peaks' => 'twin-peaks', - 'Twix' => 'twix', - 'Typhoo' => 'typhoo', - 'Tyres' => 'tyres', - 'Ubisoft' => 'ubisoft', - 'UE BOOM' => 'ue-boom', - 'UE Boom 2' => 'ue-boom-2', - 'UEFA' => 'uefa', - 'UE Megablast' => 'ue-megablast', - 'UE Megaboom' => 'ue-megaboom', - 'UGG' => 'ugg', - 'Ulefone' => 'ulefone', - 'Ultrabook' => 'ultrabook', - 'Ultrawide Monitor' => 'ultrawide', - 'Umbrella' => 'umbrella', - 'UMI' => 'umidigi', - 'Uncharted' => 'uncharted', - 'Uncharted 4: A Thief's End' => 'uncharted-4', - 'Uncharted: The Lost Legacy' => 'uncharted-the-lost-legacy', - 'Under Armour' => 'under-armour', - 'Underwear' => 'underwear', - 'Unicorn' => 'unicorn', - 'UNiDAYS' => 'unidays', - 'Universal Remote' => 'universal-remote', - 'Uno' => 'uno', - 'Uplay' => 'uplay', - 'Urban Decay' => 'urban-decay', - 'Urban Sports' => 'urban-sports', - 'USB Cable' => 'usb-cable', - 'USB Hub' => 'usb-hub', - 'USB Memory Stick' => 'flash-drive', - 'USB Type C' => 'usb-type-c', - 'USN' => 'usn', - 'Vacuum Cleaner' => 'vacuum-cleaners', - 'Vacuum Flask' => 'flask', - 'Valkyria Chronicles' => 'valkyria-chronicles', - 'Valkyria Chronicles 4' => 'valkyria-chronicles-4', - 'Vango' => 'vango', - 'Vanish' => 'vanish', - 'Vans' => 'vans', - 'Vans Old Skool' => 'vans-old-skool', - 'Vans Shoes' => 'vans-shoes', - 'Vase' => 'vase', - 'Vaseline' => 'vaseline', - 'Vauxhall' => 'vauxhall', - 'VAX' => 'vax', - 'Vax Blade' => 'vax-blade', - 'Vax Vacuum Cleaner' => 'vax-vacuum', - 'Veet' => 'veet', - 'Vega 7' => 'vega-7', - 'Vegetables' => 'vegetables', - 'Vegetarian' => 'vegetarian', - 'Vehicles' => 'vehicles', - 'Velvet Comfort' => 'velvet', - 'Vera Wang' => 'vera-wang', - 'Verbatim' => 'verbatim', - 'Versace' => 'versace', - 'Vibrator' => 'vibrator', - 'Victorinox' => 'victorinox', - 'Video Games' => 'videogame', - 'Video Streaming' => 'video-streaming', - 'Viktor & Rolf Spicebomb' => 'spicebomb', - 'Vileda' => 'vileda', - 'Villeroy & Boch' => 'villeroy-boch', - 'Viners' => 'viners', - 'Vinyl' => 'vinyl', - 'Virgin' => 'virgin', - 'Vitamins & Supplements' => 'vitamins', - 'Vitamix' => 'vitamix', - 'Vodafone' => 'vodafone', - 'Vodka' => 'vodka', - 'Volvo' => 'volvo', - 'VPN' => 'vpn', - 'VR Headset' => 'vr-headset', - 'VTech' => 'vtech', - 'VTech Toot Toot' => 'toot-toot', - 'Vue' => 'vue', - 'VW' => 'vw', - 'Wacom' => 'wacom', - 'Waffle Maker' => 'waffle-maker', - 'Wahl' => 'wahl', - 'Walkers' => 'walkers', - 'Walking Boots' => 'walking-boots', - 'Wall Art' => 'wall-art', - 'Wallet' => 'wallet', - 'Wallpaper' => 'wallpaper', - 'Wardrobe' => 'wardrobe', - 'Warhammer' => 'warhammer', - 'Washbag' => 'washbag', - 'Washer Dryer' => 'washer-dryer', - 'Washing Machine' => 'washing-machine', - 'Washing Powder' => 'washing-powder', - 'Watch' => 'watch', - 'Watch Dogs' => 'watch-dogs', - 'Watch Dogs 2' => 'watch-dogs-2', - 'Watch Dogs: Legion' => 'watch-dogs-legion', - 'Water Bottle' => 'water-bottle', - 'Water Butt' => 'water-butt', - 'Water Dispenser' => 'water-dispenser', - 'Water Filter' => 'water-filter', - 'Water Gun' => 'water-gun', - 'Waterproof Camera' => 'waterproof-camera', - 'Waterproof Jacket' => 'waterproof-jacket', - 'Watersports' => 'watersport', - 'Water Toys' => 'water-toys', - 'Wayfarer' => 'wayfarer', - 'WD40' => 'wd40', - 'Wearable' => 'wearable', - 'Weather Station' => 'weather-station', - 'Webcam' => 'webcam', - 'Weber' => 'weber', - 'Web Hosting' => 'web-hosting', - 'Wedding' => 'wedding', - 'Weed Killer' => 'weed', - 'Weekend Break' => 'weekend-break', - 'Weetabix' => 'weetabix', - 'Weightlifting' => 'weightlifting', - 'Weight Watchers' => 'weight-watchers', - 'Wellies' => 'wellies', - 'Wellness and Health' => 'wellness-and-health', - 'Wenger' => 'wenger', - 'Western Digital' => 'western-digital', - 'Wetsuit' => 'wetsuit', - 'Wheelbarrow' => 'wheelbarrow', - 'Wheelchair' => 'wheelchair', - 'Whey' => 'whey', - 'Whiskas' => 'whiskas', - 'Whisky' => 'whisky', - 'Whole Home Mesh Wi-Fi System' => 'whole-home-mesh-wifi-system', - 'Wi-Fi Camera' => 'wifi-camera', - 'Wi-Fi Dongle' => 'dongle', - 'Wi-Fi Extender' => 'wifi-extender', - 'Wii' => 'wii', - 'Wii Game' => 'wii-games', - 'Wii U Game' => 'wii-u-game', - 'Wii U Pro Controller' => 'wii-u-pro-controller', - 'Wild Turkey' => 'wild-turkey', - 'Wileyfox' => 'wileyfox', - 'Wilkinson Sword Hydro 5' => 'hydro-5', - 'Wilkinson Sword Razor' => 'wilkinson-sword', - 'Wimbledon Tennis' => 'wimbledon', - 'Window Cleaner' => 'window-cleaner', - 'Windows' => 'windows', - 'Windows 8' => 'windows-8', - 'Windows 10' => 'windows-10', - 'Wine' => 'wine', - 'Wine Advent Calendar' => 'wine-advent-calendar', - 'Wine Glasses' => 'wine-glasses', - 'Winter Jacket' => 'winter-jacket', - 'Wiper Blades' => 'wiper-blades', - 'Wireless Adapter' => 'wireless-adapter', - 'Wireless Charger' => 'wireless-charger', - 'Wireless Controller' => 'wireless-controller', - 'Wireless Headphones' => 'wireless-headphones', - 'Wireless Headset' => 'wireless-headset', - 'Wireless Keyboard' => 'wireless-keyboard', - 'Wireless Mouse' => 'wireless-mouse', - 'Wok' => 'wok', - 'Wolfenstein' => 'wolfenstein', - 'Wolfenstein 2: The New Colossus' => 'wolfenstein-2', - 'Women's Boots' => 'womens-boots', - 'Women's Fragrance' => 'womens-fragrance', - 'Women's Shoes' => 'womens-shoes', - 'Workbench' => 'workbench', - 'World of Warcraft' => 'world-of-warcraft', - 'World War Z' => 'world-war-z', - 'WORX' => 'worx', - 'Wreckfest' => 'wreckfest', - 'Wuaki' => 'wuaki', - 'WWE 2K' => 'wwe', - 'Xbox' => 'xbox', - 'Xbox 360 Game' => 'xbox-360-game', - 'Xbox Accessories' => 'xbox-accessories', - 'Xbox Controller' => 'xbox-controller', - 'Xbox Game Pass' => 'xbox-game-pass', - 'Xbox Gift Card' => 'xbox-gift-card', - 'Xbox Headset' => 'xbox-headset', - 'Xbox Kinect' => 'kinect', - 'Xbox Live' => 'xbox-live', - 'Xbox One Controller' => 'xbox-one-controller', - 'Xbox One Elite Controller' => 'xbox-one-elite-controller', - 'Xbox One Games' => 'xbox-one-games', - 'Xbox One S' => 'xbox-one-s', - 'Xbox One X' => 'xbox-one-x', - 'Xbox Series S' => 'xbox-series-s', - 'Xbox Series X' => 'xbox-series-x', - 'Xbox Series X Controller' => 'xbox-series-x-controller', - 'Xbox Series X Games' => 'xbox-series-x-game', - 'Xbox Wireless Adapter' => 'xbox-wireless-adapter', - 'Xbox Wireless Headset' => 'xbox-wireless-headset', - 'XCOM' => 'xcom', - 'XCOM 2' => 'xcom-2', - 'Xenoblade Chronicles' => 'xenoblade-chronicles', - 'XFX' => 'xfx', - 'Xiaomi' => 'xiaomi', - 'Xiaomi AirDots' => 'xiaomi-airdots', - 'Xiaomi Black Shark' => 'xiaomi-black-shark', - 'Xiaomi Black Shark 2' => 'xiaomi-black-shark-2', - 'Xiaomi Headphones' => 'xiaomi-headphones', - 'Xiaomi Laptop' => 'xiaomi-laptop', - 'Xiaomi Mi 5' => 'xiaomi-mi-5', - 'Xiaomi Mi 6' => 'xiaomi-mi-6', - 'Xiaomi Mi 8' => 'xiaomi-mi-8', - 'Xiaomi Mi 8 Lite' => 'xiaomi-mi-8-lite', - 'Xiaomi Mi 8 Pro' => 'xiaomi-mi-8-pro', - 'Xiaomi Mi 9' => 'xiaomi-mi-9', - 'Xiaomi Mi 9 Lite' => 'xiaomi-mi-9-lite', - 'Xiaomi Mi 9 SE' => 'xiaomi-mi-9-se', - 'Xiaomi Mi 9T' => 'xiaomi-mi-9t', - 'Xiaomi Mi 9T Pro' => 'xiaomi-mi-9t-pro', - 'Xiaomi Mi 10' => 'xiaomi-mi-10', - 'Xiaomi Mi 10 Lite' => 'xiaomi-mi-10-lite', - 'Xiaomi Mi 10T' => 'xiaomi-mi-10t', - 'Xiaomi Mi 10T Lite' => 'xiaomi-mi-10t-lite', - 'Xiaomi Mi 10T Pro' => 'xiaomi-mi-10t-pro', - 'Xiaomi Mi 11' => 'xiaomi-mi-11', - 'Xiaomi Mi 11 Lite 4G' => 'xiaomi-mi-11-lite-4g', - 'Xiaomi Mi 11 Lite 5G' => 'xiaomi-mi-11-lite-5g', - 'Xiaomi Mi 11 Pro' => 'xiaomi-mi-11-pro', - 'Xiaomi Mi 11 Ultra' => 'xiaomi-mi-11-ultra', - 'Xiaomi Mi 11i' => 'xiaomi-mi-11i', - 'Xiaomi Mi A1' => 'xiaomi-mi-a1', - 'Xiaomi Mi A2' => 'mi-a2', - 'Xiaomi Mi A3' => 'xiaomi-mi-a3', - 'Xiaomi Mi Band' => 'xiaomi-mi-band', - 'Xiaomi Mi Band 3' => 'xiaomi-mi-band-3', - 'Xiaomi Mi Band 4' => 'xiaomi-mi-band-4', - 'Xiaomi Mi Band 5' => 'xiaomi-mi-band-5', - 'Xiaomi Mi Box' => 'xiaomi-mi-box', - 'Xiaomi Mi Max 3' => 'xiaomi-mi-max3', - 'Xiaomi Mi Mix' => 'xiaomi-mi-mix', - 'Xiaomi Mi Mix 2' => 'xiaomi-mi-mix-2', - 'Xiaomi Mi Mix 2S' => 'xiaomi-mi-mix-2s', - 'Xiaomi Mi Mix 3' => 'xiaomi-mi-mix-3', - 'Xiaomi Mi Note' => 'xiaomi-mi-note', - 'Xiaomi Mi Note 10' => 'mi-note-10', - 'Xiaomi Mi Pad 4' => 'xiaomi-mi-pad-4', - 'Xiaomi Pocophone F1' => 'pocophone-f1', - 'Xiaomi Redmi' => 'redmi', - 'Xiaomi Redmi 4' => 'xiaomi-redmi-4', - 'Xiaomi Redmi 5' => 'redmi-5', - 'Xiaomi Redmi 6' => 'redmi-6', - 'Xiaomi Redmi 8' => 'redmi-8', - 'Xiaomi Redmi Note 4' => 'note-4', - 'Xiaomi Redmi Note 5' => 'redmi-note-5', - 'Xiaomi Redmi Note 6' => 'redmi-note-6', - 'Xiaomi Redmi Note 6 Pro' => 'xiaomi-redmi-note-6-pro', - 'Xiaomi Redmi Note 7' => 'redmi-note-7', - 'Xiaomi Redmi Note 8' => 'xiaomi-redmi-note-8', - 'Xiaomi Redmi Note 8 Pro' => 'xiaomi-redmi-note-8-pro', - 'Xiaomi Redmi Note 8T' => 'redmi-note-8t', - 'Xiaomi Redmi Note 9' => 'xiaomi-redmi-note-9', - 'Xiaomi Redmi Note 9 Pro' => 'xiaomi-redmi-note-9-pro', - 'Xiaomi Redmi Note 9S' => 'xiaomi-redmi-note-9s', - 'Xiaomi Roborock' => 'xiaomi-roborock', - 'Xiaomi Roborock S5' => 'xiaomi-roborock-s5', - 'Xiaomi Scooter' => 'xiaomi-scooter', - 'Xiaomi Smartphones' => 'xiaomi-smartphone', - 'Xiaomi Tablets' => 'xiaomi-tablet', - 'Yakuza' => 'yakuza', - 'Yale' => 'yale', - 'Yale Smart Lock' => 'yale-smart-lock', - 'Yamaha' => 'yamaha', - 'Yankee Candle' => 'yankee-candle', - 'Yeelight' => 'xiaomi-yeelight', - 'Yoga' => 'yoga', - 'Yoghurt' => 'yoghurt', - 'Yoshi' => 'yoshi', - 'Yoshi's Crafted World' => 'yoshis-crafted-world', - 'YouView' => 'youview', - 'Yves Saint Laurent' => 'yves-saint-laurent', - 'Zanussi' => 'zanussi', - 'Zippo' => 'zippo', - 'Zizzi' => 'zizzi', - 'Zoo' => 'zoo', - 'Zoostorm' => 'zoostorm', - 'ZOTAC' => 'zotac', - 'ZTE' => 'zte', - 'ZTE Smartphone' => 'zte-smartphone', - 'ZyXEL' => 'zyxel', - ] + 'type' => 'text', + 'exampleValue' => 'broadband', + 'title' => 'Group name in the URL : The group name that must be entered is present after "https://www.hotukdeals.com/tag/" and before any "?". +Example: If the URL of the group displayed in the browser is : +https://www.hotukdeals.com/tag/broadband?sortBy=temp +Then enter : +broadband', ], 'order' => [ 'name' => 'Order by', @@ -3287,5 +98,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'title-talk' => 'Discussion Monitoring', 'deal-type' => 'Deal Type', 'localdeal' => 'Local deal', + 'context-hot' => '-hot', + 'context-new' => '-new', ]; } diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 7b23f263..6be2adfb 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -40,1952 +40,14 @@ class MydealsBridge extends PepperBridgeAbstract 'Deals pro Gruppen' => [ 'group' => [ 'name' => 'Gruppen', - 'type' => 'list', - 'title' => 'Gruppe, deren Deals angezeigt werden müssen', - 'values' => [ - '1Password' => '1password', - '3D Drucker' => '3d-drucker', - '4K Fernseher' => '4k-fernseher', - '4K Monitore' => '4k-monitor', - '4K Ultra HD Blu-ray' => 'ultra-hd-blu-ray', - '8K Fernseher' => '8k-fernseher', - '32 Zoll Fernseher' => '32-zoll-fernseher', - '55 Zoll Fernseher' => '55-zoll-fernseher', - '65 Zoll Fernseher' => '65-zoll-fernseher', - '75 Zoll Fernseher' => '75-zoll-fernseher', - '1151 Mainboard' => '1151-mainboard', - 'Abus' => 'abus', - 'ABUS Fahrradschlösser' => 'abus-fahrradschloss', - 'Accessoires' => 'accessoires', - 'Acer' => 'acer', - 'Acer Aspire' => 'acer-aspire', - 'Acer Laptops' => 'acer-laptop', - 'Acer Monitore' => 'acer-monitor', - 'Acer Predator' => 'acer-predator', - 'Action Cameras' => 'actioncam', - 'Actionfiguren' => 'actionfiguren', - 'adidas' => 'adidas', - 'adidas Essentials' => 'adidas-neo', - 'adidas Iniki' => 'adidas-iniki', - 'adidas NMD' => 'adidas-nmd', - 'adidas Originals' => 'adidas-originals', - 'adidas Schuhe' => 'adidas-schuhe', - 'adidas Superstar' => 'adidas-superstar', - 'adidas Ultraboost' => 'adidas-ultraboost', - 'adidas ZX Flux' => 'adidas-zx-flux', - 'Adventskalender' => 'adventskalender', - 'AEG' => 'aeg', - 'AEG Waschmaschinen' => 'aeg-waschmaschine', - 'Age of Empires' => 'age-of-empires', - 'AiO Wasserkühlung' => 'aio-wasserkuehlung', - 'AKG' => 'akg', - 'Akkus' => 'akkus', - 'Akkuschrauber' => 'akkuschrauber', - 'Alfa Romeo' => 'alfa-romeo', - 'Alienware' => 'alienware', - 'Alkohol' => 'alkohol', - 'All Inclusive Reisen' => 'all-inclusive', - 'All in One PCs' => 'all-in-one-pcs', - 'AM4 Mainboard' => 'am4-mainboard', - 'Amazfit' => 'xiaomi-amazfit', - 'Amazfit Bip' => 'amazfit-bip', - 'Amazfit GTS' => 'amazfit-gts', - 'Amazon Echo' => 'amazon-echo', - 'Amazon Echo Dot' => 'amazon-echo-dot', - 'Amazon Echo Plus' => 'amazon-echo-plus', - 'Amazon Echo Show' => 'amazon-echo-show', - 'Amazon Echo Show 5' => 'amazon-echo-show-5', - 'Amazon Echo Show 8' => 'amazon-echo-show-8', - 'Amazon Echo Spot' => 'amazon-echo-spot', - 'Amazon Fire TV Cube' => 'fire-tv-cube', - 'Amazon Fire TV Stick' => 'fire-tv', - 'Amazon Fire TV Stick 4K' => 'fire-tv-stick-4k', - 'Amazon Tablets' => 'amazon-tablet', - 'Amazon Warehouse Deals' => 'amazon-warehouse-deals', - 'AMD' => 'amd', - 'AMD Radeon' => 'amd-radeon', - 'AMD Radeon VII' => 'vega-7', - 'AMD RX Vega' => 'amd-vega', - 'AMD Ryzen' => 'amd-ryzen', - 'AMD Ryzen 9 5900X' => 'amd-ryzen-9-5900x', - 'American Express' => 'american-express', - 'amiibo' => 'amiibo', - 'Analoguhren' => 'analoguhren', - 'Android Apps' => 'android-apps', - 'Android Smartphones' => 'android-smartphones', - 'Angelzubehör' => 'angelsport', - 'Animal Crossing' => 'animal-crossing', - 'Animal Crossing: New Horizons' => 'animal-crossing-new-horizons', - 'Anime' => 'anime', - 'Ankündigungen' => 'ankundigungen', - 'Anno 1800' => 'anno-1800', - 'Anthem' => 'anthem', - 'Anzug' => 'anzug', - 'AOC' => 'aoc', - 'Apex Legends' => 'apex-legends', - 'Apotheke' => 'apotheke', - 'Apple' => 'apple', - 'Apple AirPods' => 'airpods', - 'Apple AirPods 2' => 'airpods-2', - 'Apple AirPods Max' => 'airpods-max', - 'Apple AirPods Pro' => 'airpods-pro', - 'Apple EarPods' => 'apple-earpods', - 'Apple HomePod' => 'homepod', - 'Apple HomePod mini' => 'apple-homepod-mini', - 'Apple Kopfhörer' => 'apple-kopfhoerer', - 'Apple Magic Mouse 2' => 'apple-magic-mouse-2', - 'Apple Pencil' => 'apple-pencil', - 'Apple Pencil 2' => 'apple-pencil-2', - 'Apple TV' => 'apple-tv', - 'Apple Watch' => 'apple-watch', - 'Apple Watch 3' => 'apple-watch-3', - 'Apple Watch 4' => 'apple-watch-4', - 'Apple Watch 5' => 'apple-watch-5', - 'Apple Watch 6' => 'apple-watch-6', - 'Apple Watch SE' => 'apple-watch-se', - 'Apps' => 'apps', - 'Aquaristik' => 'aquaristik', - 'Arbeitsspeicher' => 'arbeitsspeicher', - 'Arbeitszimmermöbel' => 'arbeitszimmer', - 'ASICS' => 'asics', - 'Assassin's Creed' => 'assassins-creed', - 'Assassin's Creed: Valhalla' => 'assassins-creed-valhalla', - 'Assassin's Creed Odyssey' => 'assassins-creed-odyssey', - 'Assassin's Creed Origins' => 'assassins-creed-origins', - 'ASTRO Gaming A50' => 'astro-gaming-a50', - 'ASUS' => 'asus', - 'ASUS Laptops' => 'asus-laptop', - 'Asus Mainboard' => 'asus-mainboard', - 'Asus Monitore' => 'asus-monitor', - 'ASUS ROG' => 'asus-rog', - 'ASUS Smartphones' => 'asus-smartphones', - 'Asus ZenBook' => 'asus-zenbook', - 'ASUS ZenFone 5' => 'asus-zenfone-5', - 'ASUS ZenFone 5Z' => 'asus-zenfone-5z', - 'Audi' => 'audi', - 'Audio & HiFi' => 'audio-hifi', - 'Audioverstärker' => 'audioverstaerker', - 'Audio Zubehör' => 'audio-zubehoer', - 'Aukey' => 'aukey', - 'Außenleuchten' => 'aussenleuchten', - 'Auto & Motorrad' => 'auto-motorrad', - 'Auto Bild' => 'auto-bild', - 'Auto Leasing' => 'auto-leasing', - 'Auto Leasing Gewerbe' => 'gewerbe-leasing', - 'Auto Leasing Privat' => 'privat-leasing', - 'Automatikuhren' => 'automatikuhr', - 'auto motor und sport' => 'auto-motor-sport', - 'Autoradio' => 'autoradio', - 'Auto Teile' => 'autoteile', - 'Autowäsche' => 'autowaesche', - 'Auto Zubehör' => 'auto', - 'AVM FRITZ!Box' => 'avm-fritz-box', - 'AVM FRITZ!Box 7490' => 'avm-fritz-box-7490', - 'AVM FRITZ!Box 7530' => 'avm-fritz-box-7530', - 'AVM FRITZ!Box 7580' => 'avm-fritz-box-7580', - 'AVM FRITZ!Box 7590' => 'avm-fritz-box-7590', - 'AVM FRITZ! DECT 301' => 'avm-fritz-dect-301', - 'AV Receiver' => 'av-receiver', - 'Baby & Kind' => 'kinder', - 'Baby-Erstausstattung' => 'baby-erstausstattung', - 'Babybetten' => 'babybetten', - 'Baby Born' => 'baby-born', - 'Babykleidung' => 'babybekleidung', - 'Babynahrung' => 'babynahrung', - 'Babyphone' => 'babyphone', - 'Backofen & Herd' => 'backofen-herd', - 'Backwaren' => 'backwaren', - 'Backzubehör' => 'backzubehoer', - 'Bademode' => 'bademode', - 'Badmöbel' => 'badezimmer', - 'Bahn-Tickets' => 'bahntickets', - 'Bahncard' => 'bahncard', - 'Balkonmöbel' => 'balkonmoebel', - 'Ballerinas' => 'ballerinas', - 'Bang & Olufsen' => 'bang-olufsen', - 'Bank' => 'bank', - 'Barbie' => 'barbie', - 'Barclaycard' => 'barclaycard', - 'Bartschneider' => 'bartschneider', - 'Batterien' => 'batterien', - 'Battle.net' => 'battle-net', - 'Battlefield' => 'battlefield', - 'Battlefield 1' => 'battlefield-1', - 'Battlefield 5' => 'battlefield-5', - 'Bauknecht' => 'bauknecht', - 'Bauknecht Waschmaschinen' => 'bauknecht-waschmaschine', - 'Baumarkt' => 'baumarkt', - 'Bayonetta' => 'bayonetta', - 'Bayonetta 2' => 'bayonetta-2', - 'Beamer' => 'beamer', - 'Beamer Leinwand' => 'beamer-leinwand', - 'Beats by Dre' => 'beats-by-dre', - 'Beats Solo3' => 'beats-solo3', - 'Beats Solo Pro' => 'beats-solo-pro', - 'Beats Studio3' => 'beats-studio3', - 'Beauty & Gesundheit' => 'beauty', - 'Beko' => 'beko', - 'Beleuchtung' => 'beleuchtung', - 'Belkin' => 'belkin', - 'Ben & Jerry's' => 'ben-jerrys', - 'Bench' => 'bench', - 'BenQ' => 'benq', - 'BenQ Monitore' => 'benq-monitor', - 'be quiet!' => 'be-quiet', - 'be quiet! Netzteile' => 'be-quiet-netzteil', - 'Besteck' => 'besteck', - 'Bethesda' => 'bethesda', - 'Betten' => 'betten', - 'Bettwäsche' => 'bettwaesche', - 'beyerdynamic' => 'beyerdynamic', - 'Beyerdynamic MMX 300' => 'beyerdynamic-mmx-300', - 'BHs' => 'bhs', - 'Bier' => 'bier', - 'Biking & Urban Sport' => 'biking-urban-sport', - 'Bildbearbeitungsprogramme' => 'bildbearbeitungsprogramme', - 'Birkenstock' => 'birkenstock', - 'Black & Decker' => 'black-and-decker', - 'Blackberry Smartphones' => 'blackberry', - 'Black Desert Online' => 'black-desert-online', - 'Blazer' => 'blazer', - 'Blood & Truth' => 'blood-truth', - 'Blu-ray' => 'blu-ray', - 'Blu-ray Player' => 'blu-ray-player', - 'Bluetooth Kopfhörer' => 'bluetooth-kopfhoerer', - 'Bluetooth Lautsprecher' => 'bluetooth-lautsprecher', - 'Blumen' => 'blumen', - 'Blusen' => 'blusen', - 'BMW' => 'bmw', - 'Bodenbelag' => 'bodenbelag', - 'Boho-Chic wohnen' => 'boho-chich-wohnen', - 'Bohrer' => 'bohrer', - 'Bohrhämmer' => 'bohrhaemmer', - 'Bohrmaschinen' => 'bohrmaschinen', - 'Bollerwagen' => 'bollerwagen', - 'Bombay Gin' => 'bombay', - 'Borderlands' => 'borderlands', - 'Borderlands 3' => 'borderlands-3', - 'Bosch' => 'bosch', - 'Bosch Akkuschrauber' => 'bosch-akkuschrauber', - 'Bosch Geschirrspüler' => 'bosch-geschirrspueler', - 'Bosch Kühlschränke' => 'bosch-kuehlschrank', - 'Bosch Waschmaschinen' => 'bosch-waschmaschine', - 'Bose' => 'bose', - 'Bose Headphones 700' => 'bose-headphones-700', - 'Bose Home Speaker 500' => 'bose-home-speaker-500', - 'Bose Kopfhörer' => 'bose-kopfhoerer', - 'Bose QuietComfort' => 'bose-quietcomfort', - 'Bose QuietComfort 35 II' => 'bose-quiet-comfort-35-ii', - 'Bose Solo 5' => 'bose-solo-5', - 'Bose SoundLink' => 'bose-soundlink', - 'Bose SoundTouch' => 'bose-soundtouch', - 'BOSS' => 'boss', - 'Bourbon' => 'bourbon', - 'Bowers & Wilkins' => 'bowers-wilkins', - 'Boxershorts' => 'boxershorts', - 'Boxspringbetten' => 'boxspringbetten', - 'Braun' => 'braun', - 'Braun Rasierer' => 'braun-rasierer', - 'Braun Series 3' => 'braun-series-3', - 'Braun Series 5' => 'braun-series-5', - 'Braun Series 7' => 'braun-series-7', - 'Braun Series 9' => 'braun-series-9', - 'Bridgekameras' => 'bridgekamera', - 'Brigitte' => 'brigitte', - 'Brillen & Kontaktlinsen' => 'brillen', - 'Brita' => 'brita', - 'Britax Römer' => 'britax-roemer', - 'Brotaufstrich' => 'brotaufstrich', - 'Brother Drucker' => 'brother-drucker', - 'Bücher' => 'buecher', - 'Bücher, Magazine & Zeitschriften' => 'buecher-zeitschriften', - 'bugatti' => 'bugatti', - 'Bügeleisen' => 'buegeleisen', - 'Bügeln' => 'buegeln', - 'Buggy' => 'buggy', - 'Burger' => 'burger', - 'BURNHARD' => 'burnhard', - 'Bürobedarf' => 'buerobedarf', - 'Bürostühle' => 'buerostuhl', - 'Bus & Bahn' => 'bus-bahn', - 'Business Mode' => 'business-mode', - 'c't – Magazin für Computertechnik' => 'ct-magazin-computertechnik', - 'Cafissimo' => 'cafissimo', - 'Call of Duty' => 'call-of-duty', - 'Call of Duty: Black Ops 4' => 'call-of-duty-black-ops-4', - 'Call of Duty: Black Ops Cold War' => 'call-of-duty-black-ops-cold-war', - 'Call of Duty: Infinite Warfare' => 'call-of-duty-infinite-warfare', - 'Call of Duty: Modern Warfare' => 'call-of-duty-modern-warfare', - 'Call of Duty: Warzone' => 'call-of-duty-warzone', - 'Call of Duty: WW2' => 'call-of-duty-ww2', - 'Calvin Klein' => 'calvin-klein', - 'Camcorder' => 'camcorder', - 'Campen' => 'campen', - 'Canon' => 'canon', - 'Canon Drucker' => 'canon-drucker', - 'Canon EOS' => 'canon-eos', - 'Canon Kameras' => 'canon-kameras', - 'Canon PowerShot' => 'canon-powershot', - 'CANTON' => 'canton', - 'Caps' => 'caps', - 'Captain Toad: Treasure Tracker' => 'captain-toad-treasure-tracker', - 'Capture One' => 'capture-one', - 'Carhartt' => 'carhartt', - 'Carsharing' => 'carsharing', - 'Casio' => 'casio', - 'Cheap Monday' => 'cheapmonday', - 'Chevrolet' => 'chevrolet', - 'China Handys' => 'china-handys', - 'Chip (Magazin)' => 'chip-magazin', - 'Chips' => 'chips', - 'Christbaumschmuck' => 'christbaumschmuck', - 'Christbaumständer' => 'christbaumstaender', - 'Chromebook' => 'chromebook', - 'Chronographen' => 'chronograph', - 'Chucks' => 'chucks', - 'Citroen' => 'citroen', - 'Coca-Cola' => 'coca-cola', - 'Comics' => 'comics', - 'Computer' => 'computer', - 'Computer & Tablets' => 'computer-tablet', - 'Computer Bild' => 'computer-bild', - 'Controller' => 'controller', - 'Converse' => 'converse', - 'Convertibles' => 'convertibles', - 'Corsair' => 'corsair', - 'Corsair VOID PRO' => 'corsair-void-pro', - 'Couchtische' => 'couchtische', - 'Coupons' => 'coupons', - 'CPU-Kühler' => 'cpu-kuehler', - 'Craghoppers' => 'craghoppers', - 'Crocs' => 'crocs', - 'Crucial' => 'crucial', - 'Cupra' => 'cupra', - 'Cyberpunk 2077' => 'cyberpunk-2077', - 'cybex' => 'cybex', - 'D-Link' => 'd-link', - 'DAB Radios' => 'dab-radios', - 'Dacia' => 'dacia', - 'Damenbekleidung' => 'fashion-frauen', - 'Damenschuhe' => 'damenschuhe', - 'Dampfbügelstation' => 'dampfbuegelstation', - 'Dampfgarer' => 'dampfgarer', - 'Dampfreiniger' => 'dampfreiniger', - 'Dark Souls' => 'dark-souls', - 'Dashcam' => 'dashcam', - 'Datentarif' => 'datentarif', - 'Daypack' => 'daypack', - 'Days Gone' => 'days-gone', - 'DC Shoes' => 'dc-shoes', - 'DDR3 RAM' => 'ddr3-ram', - 'DDR4 RAM' => 'ddr4-ram', - 'De'Longhi' => 'delonghi', - 'Death Stranding' => 'death-stranding', - 'Deckenlampen' => 'deckenlampen', - 'DECT Telefone' => 'telefone', - 'Dekoration' => 'dekoration', - 'Dell' => 'dell', - 'Dell Laptops' => 'dell-laptop', - 'Dell Monitore' => 'dell-monitor', - 'Dell XPS' => 'dell-xps', - 'Denon' => 'denon', - 'Deo' => 'deo', - 'Depot' => 'depot', - 'DER SPIEGEL' => 'der-spiegel', - 'Designermöbel' => 'designermoebel', - 'Desigual' => 'desigual', - 'Desinfektionsmittel' => 'desinfektionsmittel', - 'Desktop PCs' => 'desktop-pc', - 'Dessous' => 'dessous', - 'Destiny' => 'destiny', - 'Destiny 2' => 'destiny-2', - 'Deus Ex' => 'deus-ex', - 'Deus Ex: Mankind' => 'deus-ex-mankind', - 'Deuter' => 'deuter', - 'DeutschlandCard' => 'deutschlandcard', - 'devolo' => 'devolo', - 'DeWalt' => 'dewalt', - 'Die drei Fragezeichen' => 'die-drei-fragezeichen', - 'Die Eiskönigin' => 'die-eiskoenigin', - 'Dienstleistungen & Verträge' => 'dienstleistungen-vertraege', - 'Dies & Das' => 'dies-das', - 'Diesel' => 'diesel', - 'Die Sims' => 'die-sims', - 'Die Sims 4' => 'die-sims-4', - 'Die Zeit' => 'die-zeit', - 'Digitalreceiver' => 'digitalreceiver', - 'Digitaluhren' => 'digitaluhr', - 'Direktflüge' => 'direktfluege', - 'Dirt Devil' => 'dirt-devil', - 'Dishonored' => 'dishonored', - 'Dishonored 2: Das Vermächtnis der Maske' => 'dishonored-2', - 'Disney' => 'disney', - 'Disney+' => 'disney-plus', - 'DJI' => 'dji', - 'DJI Osmo Pocket' => 'dji-osmo-pocket', - 'Dockers' => 'dockers', - 'Dolce Gusto' => 'dolce-gusto', - 'DOOM Eternal' => 'doom-eternal', - 'Douglas Adventskalender' => 'douglas-adventskalender', - 'Dr. Martens' => 'dr-martens', - 'Dragon Ball' => 'dragon-ball', - 'Dragon Ball FighterZ' => 'dragon-ball-fighterz', - 'Dragon Ball Z: Kakarot' => 'dragon-ball-z-kakarot', - 'Dragon Quest Builders' => 'dragon-quest-builders', - 'Dragon Quest Builders 2' => 'dragon-quest-builders-2', - 'Dreame Staubsauger' => 'xiaomi-staubsauger', - 'Dreame T20' => 'dreame-t20', - 'Dreame V9' => 'xiaomi-dreame-v9', - 'Dreame V10' => 'xiaomi-dreame-v10', - 'Dreame V11' => 'xiaomi-dreame-v11', - 'Drohnen' => 'drohnen', - 'Drucker' => 'drucker', - 'Druckerpatronen' => 'druckerpatronen', - 'Druckerzubehör' => 'druckerzubehoer', - 'DSL & Kabel' => 'dsl', - 'Dunstabzugshauben' => 'dunstabzugshauben', - 'Durex' => 'durex', - 'Duscharmaturen' => 'duscharmaturen', - 'Duschgel' => 'duschgel', - 'Duschköpfe' => 'duschkoepfe', - 'DVD' => 'dvd', - 'Dyson' => 'dyson', - 'Dyson Staubsauger' => 'dyson-staubsauger', - 'Dyson V6' => 'dyson-v6', - 'Dyson V7' => 'dyson-v7', - 'Dyson V8' => 'dyson-v8', - 'Dyson V10' => 'dyson-v10', - 'Dyson V11' => 'dyson-v11', - 'Dyson V11 Absolute' => 'dyson-v11-absolute', - 'Dyson V11 Animal' => 'dyson-v11-animal', - 'E-Bikes' => 'e-bikes', - 'E-Scooter' => 'e-scooter', - 'E-Scooter Sharing' => 'e-scooter-sharing', - 'E-Zigaretten' => 'e-zigaretten', - 'Eastpak' => 'eastpak', - 'eBook Reader' => 'ebook-reader', - 'eBooks' => 'ebooks', - 'Ecovacs' => 'ecovacs', - 'Ecovacs Deebot 900' => 'ecovacs-deebot-900', - 'Ecovacs Deebot OZMO 930' => 'ecovacs-deebot-ozmo-930', - 'Edifier' => 'edifier', - 'Edifier R1280DB' => 'edifier-r1280db', - 'Edifier R1280T' => 'edifier-r1280t', - 'Einhell' => 'einhell', - 'Eis' => 'eis', - 'Elektrische Zahnbürsten' => 'elektrische-zahnbuersten', - 'Elektrogrills' => 'elektrogrill', - 'Elektroheizungen' => 'elektroheizungen', - 'Elektronik' => 'elektronik', - 'Elektronik Zubehör' => 'elektronikzubehoer', - 'Elektrorasierer' => 'elektrorasierer', - 'Elektroroller' => 'elektroroller', - 'Elektrowerkzeuge' => 'elektrowerkzeug', - 'Elephone' => 'elephone', - 'ELLE' => 'elle', - 'Emsa' => 'emsa', - 'Energy Drinks' => 'energy-drinks', - 'Entsafter' => 'entsafter', - 'Epilierer' => 'epilierer', - 'Epson' => 'epson', - 'Epson Drucker' => 'epson-drucker', - 'Erotik' => 'erotik', - 'Error Fare' => 'error-fare', - 'Espressomaschinen' => 'espressomaschinen', - 'Esprit' => 'esprit', - 'Esstische' => 'esstisch', - 'Esszimmer' => 'esszimmer', - 'Eterna' => 'eterna', - 'EUROtronic Comet DECT' => 'eurotronic-comet-dect', - 'Externe Festplatten' => 'externe-festplatten', - 'F1 2017' => 'f1-2017', - 'F1 2019' => 'f1-2019', - 'F1 2020' => 'f1-2020', - 'Fahrräder' => 'fahrraeder', - 'Fahrradhelme' => 'fahrradhelme', - 'Fahrradrucksäcke' => 'fahrradrucksack', - 'Fahrradschlösser' => 'fahrradschloss', - 'Fahrradteile' => 'fahrradteile', - 'Fahrradträger' => 'fahrradtraeger', - 'Fahrradzubehör' => 'fahrradzubehoer', - 'Fahrzeuge' => 'fahrzeuge', - 'Falke' => 'falke', - 'Fallout' => 'fallout', - 'Fallout 4' => 'fallout-4', - 'Fallout 76' => 'fallout-76', - 'Family & Kids' => 'family-kids', - 'Far Cry' => 'far-cry', - 'Far Cry 5' => 'far-cry-5', - 'Far Cry New Dawn' => 'far-cry-new-dawn', - 'Fashion & Accessoires' => 'fashion-accessoires', - 'Fast Food' => 'fast-food', - 'Felgen' => 'felgen', - 'Fenstersauger' => 'fenstersauger', - 'Fernbus-Tickets' => 'fernbus', - 'Fernseher' => 'fernseher', - 'Fertiggerichte' => 'fertiggerichte', - 'Festplatten' => 'festplatten', - 'Festplattengehäuse' => 'festplattengehaeuse', - 'FFP2 Masken' => 'ffp2-masken', - 'Fiat' => 'fiat', - 'FIFA' => 'fifa', - 'FIFA 17' => 'fifa-17', - 'FIFA 18' => 'fifa-18', - 'FIFA 19' => 'fifa-19', - 'FIFA 20' => 'fifa-20', - 'FIFA 21' => 'fifa-21', - 'FILA' => 'fila', - 'Filme & Serien' => 'filme-serien', - 'Filterkaffeemaschinen' => 'filterkaffeemaschinen', - 'Final Fantasy' => 'final-fantasy', - 'Final Fantasy 7' => 'final-fantasy-7', - 'Finanzen- und Steuersoftware' => 'finanzen-und-steuersoftware', - 'Finish' => 'finish', - 'Fisch & Meeresfrüchte' => 'fisch-meeresfruechte', - 'Fischertechnik' => 'fischertechnik', - 'Fisher-Price' => 'fisher-price', - 'Fiskars' => 'fiskars', - 'Fissler' => 'fissler', - 'fitbit' => 'fitbit', - 'Fitness & Running' => 'fitness', - 'Fitness Apps' => 'fitness-apps', - 'Fitnessstudio' => 'fitnessstudio', - 'Fitnesstracker' => 'fitnesstracker', - 'Fjällräven' => 'fjaellraeven', - 'Fleisch & Wurst' => 'fleisch-wurst', - 'Fliesenschneider' => 'fliesenschneider', - 'Flüge' => 'fluege', - 'Flurmöbel' => 'flurmoebel', - 'FOCUS' => 'focus', - 'Ford' => 'ford', - 'For Honor' => 'for-honor', - 'Formel 1 Games' => 'formel-1', - 'Fortnite' => 'fortnite', - 'Forza' => 'forza', - 'Forza Horizon' => 'forza-horizon', - 'Forza Horizon 4' => 'forza-horizon-4', - 'Forza Motorsport' => 'forza-motorsport', - 'Forza Motorsport 7' => 'forza-7', - 'Fossil' => 'fossil', - 'Foto & Kamera' => 'foto-video', - 'Foto Apps' => 'foto-apps', - 'Fotobücher' => 'fotobuecher', - 'Fototapete' => 'fototapete', - 'Fragen & Gesuche' => 'gesuche', - 'Frankfurter Allgemeine Zeitung (F.A.Z.)' => 'frankfurter-allgemeine-zeitung', - 'FreeSync Monitore' => 'freesync-monitor', - 'Freizeitpark-Tickets' => 'freizeitpark', - 'Freizeitsport' => 'freizeitsport', - 'Fritteusen' => 'fritteusen', - 'Frontlader' => 'frontlader', - 'Frühlingsdeko' => 'fruehlingsdeko', - 'Frühstücksflocken' => 'fruehstuecksflocken', - 'Fruit of the Loom' => 'fruit-of-the-loom', - 'Fujifilm' => 'fujifilm', - 'Füller' => 'fueller', - 'Full HD-Beamer' => 'full-hd-beamer', - 'Fun Factory' => 'fun-factory', - 'FurReal Friends' => 'furreal-friends', - 'Fußball' => 'fussball', - 'Fußball-Trikots' => 'fussball-trikots', - 'Fußballschuhe' => 'fussballschuhe', - 'G-Star' => 'g-star', - 'G-Sync Monitore' => 'g-sync-monitor', - 'Game of Thrones' => 'game-of-thrones', - 'Gaming' => 'gaming', - 'Gaming Headsets' => 'gaming-headset', - 'Gaming Laptops' => 'gaming-laptop', - 'Gaming Mäuse' => 'gaming-maus', - 'Gaming Monitore' => 'gaming-monitor', - 'Gaming PCs' => 'gaming-pc', - 'Gaming Stühle' => 'gaming-stuhl', - 'Gaming Tastaturen' => 'gaming-tastatur', - 'Gaming Zubehör' => 'spielekonsolen-zubehoer', - 'Ganzjahresreifen' => 'ganzjahresreifen', - 'GAP' => 'gap', - 'Gardena' => 'gardena', - 'Garderobe' => 'garderobe', - 'Garmin' => 'garmin', - 'Garmin Fenix' => 'garmin-fenix', - 'Garten' => 'garten', - 'Garten & Baumarkt' => 'garten-baumarkt', - 'Gartenarbeit' => 'gartenarbeit', - 'Gartenbank' => 'gartenbank', - 'Gartenliegen' => 'sonnenliegen', - 'Gartenmöbel' => 'gartenmoebel', - 'Gartenstühle' => 'gartenstuehle', - 'Gartentische' => 'gartentische', - 'Gasgrills' => 'gasgrill', - 'Gastarif' => 'gastarif', - 'Gears 5' => 'gears-5', - 'Gears of War' => 'gears-of-war', - 'Gefrierschränke' => 'gefrierschrank', - 'Geld-zurück-Aktionen' => 'geld-zurueck', - 'Geldbörsen' => 'geldboersen', - 'Gemüse' => 'gemuese', - 'Geox' => 'geox', - 'Geschirr' => 'geschirr', - 'Geschirrspüler' => 'geschirrspueler', - 'Gesellschaftsspiele' => 'gesellschaftsspiele', - 'Gesichtspflege' => 'gesichtspflege', - 'Gesundheit' => 'gesundheit', - 'Getränke' => 'getraenke', - 'Gewinnspiele' => 'gewinnspiele', - 'GHD' => 'ghd', - 'Ghost of Tsushima' => 'ghost-of-tsushima', - 'GIGABYTE' => 'gigabyte', - 'Gigaset' => 'gigaset', - 'Gillette' => 'gillette', - 'Gillette Rasierer' => 'gillette-rasierer', - 'Gin' => 'gin', - 'Girokonto' => 'konto', - 'Glamour' => 'glamour', - 'Glamourös wohnen' => 'glamouroes-wohnen', - 'Gläser' => 'glaeser', - 'Glätteisen' => 'glaetteisen', - 'Gleitgel' => 'gleitgel', - 'Glühwein' => 'gluehwein', - 'God of War' => 'god-of-war', - 'Google Chromecast' => 'chromecast', - 'Google Chromecast mit Google TV' => 'chromecast-mit-google-tv', - 'Google Chromecast Ultra' => 'chromecast-ultra', - 'Google Home' => 'google-home', - 'Google Home Max' => 'google-home-max', - 'Google Home Mini' => 'google-home-mini', - 'Google Nest Hub' => 'google-nest-hub', - 'Google Pixel' => 'google-pixel', - 'Google Pixel 2' => 'google-pixel-2', - 'Google Pixel 3' => 'google-pixel-3', - 'Google Pixel 4' => 'google-pixel-4', - 'Google Pixel 4 XL' => 'google-pixel-4xl', - 'Google Pixel 4a' => 'google-pixel-4a', - 'Google Pixel 4a 5G' => 'google-pixel-4a-5g', - 'Google Pixel 5' => 'google-pixel-5', - 'Google Smartphones' => 'google-smartphones', - 'Google Stadia Konsolen' => 'google-stadia', - 'GoPro Action Cameras' => 'gopro', - 'GoPro HERO 7' => 'gopro-hero-7', - 'GoPro HERO 8' => 'gopro-hero-8', - 'GoPro HERO 9' => 'gopro-hero-9', - 'Gorenje' => 'gorenje', - 'Grafikkarten' => 'grafikkarten', - 'Gran Turismo' => 'gran-turismo', - 'Gran Turismo Sport' => 'gran-turismo-sport', - 'Grazia' => 'grazia', - 'Grills' => 'grill', - 'Grillzubehör' => 'grillzubehoer', - 'Grundig' => 'grundig', - 'GTA' => 'gta', - 'GTA V' => 'gta-v', - 'GTX 1060' => 'gtx-1060', - 'GTX 1070' => 'gtx-1070', - 'GTX 1080' => 'gtx-1080', - 'GTX 1080 Ti' => 'gtx-1080-ti', - 'GTX 1660' => 'gtx-1660', - 'GTX 1660 Ti' => 'gtx-1660-ti', - 'Gucci' => 'gucci', - 'Gummistiefel' => 'gummistiefel', - 'Gürtel' => 'guertel', - 'Gutscheinfehler' => 'gutscheinfehler', - 'Haarentfernung' => 'haarentfernung', - 'Haargel' => 'haargel', - 'Haarpflege' => 'haarpflege', - 'Haarschneidemaschinen' => 'haarschneidemaschinen', - 'Haarspray' => 'haarspray', - 'Haartrockner' => 'haartrockner', - 'Haftpflichtversicherung' => 'haftpflichtversicherung', - 'Hama' => 'hama', - 'Handelsblatt' => 'handelsblatt', - 'Handmixer' => 'handmixer', - 'Handtaschen' => 'handtaschen', - 'Handtücher' => 'handtuecher', - 'Handwerkzeuge' => 'handwerkzeug', - 'Handy & Smartphone Zubehör' => 'smartphone-zubehoer', - 'Handyhalterung' => 'handyhalterung', - 'Handyhüllen' => 'handyhuelle', - 'Handys mit Vertrag' => 'handys-mit-vertrag', - 'Handys ohne Vertrag' => 'handys-ohne-vertrag', - 'Handyversicherung' => 'handyversicherung', - 'Handyverträge' => 'handyvertraege', - 'Handyverträge 3 Monate Kündigungsfrist' => 'handyvertraege-3-monate-kuendigungsfrist', - 'Handyverträge monatlich kündbar' => 'handyvertraege-monatlich-kuendbar', - 'Hängematten' => 'haengematten', - 'Hanteln' => 'hanteln', - 'Haribo' => 'haribo', - 'Harman Kardon' => 'harman-kardon', - 'Harry Potter' => 'harry-potter', - 'Hasbro' => 'hasbro', - 'Haushaltsartikel' => 'haushaltsartikel', - 'Haushaltsgeräte' => 'haushaltsgeraete', - 'Haushaltswaren' => 'haushaltswaren', - 'Hausratversicherung' => 'hausratsversicherung', - 'Hausschuhe' => 'hausschuhe', - 'Haustier' => 'haustier', - 'Hautpflege' => 'hautpflege', - 'Head & Shoulders' => 'head-and-shoulders', - 'Heckenscheren' => 'heckenschere', - 'Heimkino' => 'heimkino', - 'Heimtextilien' => 'heimtextilien', - 'Heißluftfritteusen' => 'heissluftfriteuse', - 'Heizkörperthermostat' => 'heizkoerperthermostat', - 'Heizungen' => 'heizungen', - 'Hemden' => 'hemden', - 'Hendrick's Gin' => 'hendricks-gin', - 'Herbstdeko' => 'herbstdeko', - 'Herrenbekleidung' => 'fashion-maenner', - 'Herrenschuhe' => 'herrenschuhe', - 'HiPP' => 'hipp', - 'Hisense' => 'hisense', - 'Hochbetten' => 'hochbetten', - 'Hochdruckreiniger' => 'hochdruckreiniger', - 'Hochstuhl' => 'hochstuhl', - 'Hollywoodschaukel' => 'hollywoodschaukel', - 'Home & Living' => 'home-living', - 'homee' => 'homee', - 'Honda' => 'honda', - 'Honor' => 'honor', - 'Honor 5' => 'honor-5', - 'Honor 6' => 'honor-6', - 'Honor 7X' => 'honor-7', - 'Honor 8' => 'honor-8', - 'Honor 9' => 'honor-9', - 'Honor 20' => 'honor-20', - 'Honor 20 Lite' => 'honor-20-lite', - 'Honor Band 4' => 'honor-band-4', - 'Honor Band 5' => 'honor-band-5', - 'Honor Play' => 'honor-play', - 'Honor Smartphones' => 'honor-smartphones', - 'Honor View 10' => 'honor-view-10', - 'Honor View 20' => 'honor-view-20', - 'Hoodies' => 'hoodies', - 'Hörbücher' => 'hoerbuecher', - 'Horizon Zero Dawn' => 'horizon-zero-dawn', - 'Hörspiele' => 'hoerspiele', - 'Hörzu' => 'hoerzu', - 'Hosen' => 'hosen', - 'Hotels & Unterkünfte' => 'hotel', - 'Hot Wheels' => 'hot-wheels', - 'Hoverboards' => 'hoverboards', - 'HP' => 'hp', - 'HP Drucker' => 'hp-drucker', - 'HP Laptops' => 'hp-laptop', - 'HP OMEN' => 'hp-omen', - 'HP Pavilion' => 'hp-pavilion', - 'HTC 10' => 'htc-10', - 'HTC Desire 12' => 'htc-desire', - 'HTC Smartphones' => 'htc-smartphones', - 'HTC U11' => 'htc-u11', - 'HTC Vive' => 'htc-vive', - 'Huawei' => 'huawei', - 'Huawei Kopfhörer' => 'huawei-kopfhoerer', - 'Huawei Mate 9' => 'huawei-mate-9', - 'Huawei Mate 10' => 'huawei-mate-10', - 'Huawei Mate 20' => 'huawei-mate-20', - 'Huawei Mate 20 Lite' => 'huawei-mate-20-lite', - 'Huawei Mate 20 Pro' => 'huawei-mate-20-pro', - 'Huawei Mate 30 Pro' => 'huawei-mate-30-pro', - 'Huawei MateBook' => 'huawei-matebook', - 'Huawei P10' => 'huawei-p10', - 'Huawei P20' => 'huawei-p20', - 'Huawei P30' => 'huawei-p30', - 'Huawei P30 Lite' => 'huawei-p30-lite', - 'Huawei P30 Pro' => 'huawei-p30-pro', - 'Huawei P40' => 'huawei-p40', - 'Huawei P40 Lite' => 'huawei-p40-lite', - 'Huawei P40 Pro' => 'huawei-p40-pro', - 'Huawei P Smart' => 'huawei-p-smart', - 'Huawei Smartphones' => 'huawei-smartphones', - 'Huawei Tablets' => 'huawei-mediapad', - 'Huawei Watch GT2' => 'huawei-watch-gt2', - 'Huawei Y7' => 'huawei-y7', - 'Hunde' => 'hunde', - 'Hundefutter' => 'hundefutter', - 'Hüte & Mützen' => 'huete-muetzen', - 'Hyrule Warriors' => 'hyrule-warriors', - 'Hyrule Warriors: Zeit der Verheerung' => 'hyrule-warriors-zeit-der-verheerung', - 'Hyundai' => 'hyundai', - 'iMac' => 'imac', - 'Immortals Fenyx Rising' => 'immortals-fenyx-rising', - 'In-Ear Kopfhörer' => 'in-ear-kopfhoerer', - 'Industrial Style' => 'industrial-style', - 'Inline Skates' => 'inline-skates', - 'Instax Mini' => 'instax-mini', - 'Intel Core i9-9900K' => 'intel-core-i9-9900k', - 'Intel i3' => 'intel-i3', - 'Intel i5' => 'intel-i5', - 'Intel i7' => 'intel-i7', - 'Intel i9' => 'intel-i9', - 'Intenso' => 'intenso', - 'Internet Security' => 'internet-security', - 'Intimpflege' => 'intimpflege', - 'iOS Apps' => 'ios-apps', - 'iPad' => 'ipad', - 'iPad 2019' => 'ipad-2019', - 'iPad 2020' => 'ipad-2020', - 'iPad Air' => 'ipad-air-2', - 'iPad Air 2019' => 'ipad-air-2019', - 'iPad Air 2020' => 'ipad-air-2020', - 'iPad mini' => 'ipad-mini', - 'iPad Pro' => 'ipad-pro', - 'iPad Pro 11' => 'ipad-pro-11', - 'iPad Pro 12.9' => 'ipad-pro-12-9', - 'iPad Pro 2020' => 'ipad-pro-2020', - 'iPhone' => 'iphone', - 'iPhone 6' => 'iphone-6', - 'iPhone 6 Plus' => 'iphone-6-plus', - 'iPhone 6s' => 'iphone-6s', - 'iPhone 6s Plus' => 'iphone-6s-plus', - 'iPhone 7' => 'iphone-7', - 'iPhone 7 Plus' => 'iphone-7-plus', - 'iPhone 8' => 'iphone-8', - 'iPhone 8 Plus' => 'iphone-8-plus', - 'iPhone 11' => 'iphone-11', - 'iPhone 11 Pro' => 'iphone-11-pro', - 'iPhone 11 Pro Max' => 'iphone-11-pro-max', - 'iPhone 12' => 'iphone-12', - 'iPhone 12 mini' => 'iphone-12-mini', - 'iPhone 12 Pro' => 'iphone-12-pro', - 'iPhone 12 Pro Max' => 'iphone-12-pro-max', - 'iPhone SE' => 'iphone-se', - 'iPhone X' => 'iphone-x', - 'iPhone Xr' => 'iphone-xr', - 'iPhone Xs' => 'iphone-xs', - 'iPhone Xs Max' => 'iphone-xs-max', - 'iPhone Zubehör' => 'iphone-zubehoer', - 'Irish Whiskey' => 'irish-whiskey', - 'iRobot' => 'irobot', - 'iRobot Roomba' => 'irobot-roomba', - 'iRobot Roomba 980' => 'irobot-roomba-980', - 'iRobot Roomba i7' => 'irobot-roomba-i7', - 'Isomatten' => 'isomatten', - 'iTunes Guthaben' => 'itunes-guthaben', - 'Jabra Elite 75t' => 'jabra-elite-75t', - 'Jabra Elite 85h' => 'jabra-elite-85h', - 'Jabra Elite 85t' => 'jabra-elite-85t', - 'Jabra Elite Active 75t' => 'jabra-elite-active-75t', - 'Jabra Kopfhörer' => 'jabra-kopfhoerer', - 'JACK & JONES' => 'jack-jones', - 'Jacken' => 'jacken', - 'JACK WOLFSKIN' => 'jack-wolfskin', - 'Jagdzubehör' => 'jagdzubehoer', - 'JBL' => 'jbl', - 'JBL Charge 4' => 'jbl-charge-4', - 'JBL Flip' => 'jbl-flip', - 'JBL GO' => 'jbl-go', - 'Jeans' => 'jeans', - 'Jim Beam' => 'jim-beam', - 'Jogginghosen' => 'jogginghosen', - 'Joghurt' => 'joghurt', - 'Johnnie Walker' => 'johnnie-walker', - 'Jura Kaffeemaschinen' => 'jura', - 'Just Cause' => 'just-cause', - 'Just Cause 4' => 'just-cause-4', - 'Kaffee' => 'kaffee', - 'Kaffeekapseln' => 'kaffeekapseln', - 'Kaffeemaschinen' => 'kaffeemaschinen', - 'Kaffeemühlen' => 'kaffeemuehlen', - 'Kaffeepadmaschinen' => 'kaffeepadmaschinen', - 'Kaffeepads' => 'kaffeepads', - 'Kaffeevollautomaten' => 'kaffeevollautomaten', - 'Kameras' => 'kamera', - 'Kamera Zubehör' => 'kamerazubehoer', - 'Kamine' => 'kamine', - 'Kapselmaschinen' => 'kapselmaschinen', - 'Kärcher' => 'kaercher', - 'Kärcher Fenstersauger' => 'kaercher-fenstersauger', - 'Kärcher Hochdruckreiniger' => 'kaercher-hochdruckreiniger', - 'Kartenspiele' => 'kartenspiel', - 'Käse' => 'kaese', - 'Katzen' => 'katzen', - 'Katzenfutter' => 'katzenfutter', - 'Kaufen im Ausland' => 'kaufen-ausland', - 'Ketchup' => 'ketchup', - 'KFZ Versicherung' => 'kfz-versicherung', - 'KIA' => 'kia', - 'kiddy' => 'kiddy', - 'Kinder Adventskalender' => 'kinder-adventskalender', - 'Kinderbekleidung' => 'kinderkleidung', - 'Kinderbetten' => 'kinderbett', - 'Kinderfahrräder' => 'kinderfahrrad', - 'Kinderschuhe' => 'kinderschuhe', - 'Kindersitz' => 'kindersitz', - 'Kinderwagen' => 'kinderwagen', - 'Kinderwagen & Autositze' => 'baby-transport', - 'Kinderzimmermöbel' => 'kinderzimmer', - 'Kindle' => 'kindle', - 'Kindle Oasis' => 'kindle-oasis', - 'Kindle Paperwhite' => 'kindle-paperwhite', - 'Kingdom Come: Deliverance' => 'kingdom-come-deliverance', - 'Kingdom Hearts' => 'kingdom-hearts', - 'Kingdom Hearts 3' => 'kingdom-hearts-3', - 'Kingston HyperX Cloud Flight' => 'kingston-hyperx-cloud-flight', - 'Kingston HyperX Cloud II' => 'hyperx-cloud-ii', - 'Kino' => 'kino', - 'KitchenAid' => 'kitchenaid', - 'Kleider' => 'kleider', - 'Kleiderschränke' => 'kleiderschraenke', - 'Kleidung' => 'kleidung', - 'Klemmbausteine' => 'klemmbausteine', - 'Klimaanlagen' => 'klimaanlagen', - 'Klimatechnik' => 'klimatechnik', - 'Klipsch' => 'klipsch', - 'Kochgeräte' => 'kochgeraete', - 'Kodak' => 'kodak', - 'Koffer' => 'koffer', - 'Kohlenmonoxidmelder' => 'kohlenmonoxidmelder', - 'Kolonialstil' => 'kolonialstil', - 'Kommoden & Sideboards' => 'kommoden-sideboards', - 'Kondome' => 'kondome', - 'König der Löwen Musical' => 'koenig-der-loewen-musical', - 'Kontaktgrills' => 'kontaktgrill', - 'Konto & Kreditkarten' => 'konto-kreditkarten', - 'Konzert-Tickets' => 'konzerte', - 'Kopfhörer' => 'kopfhoerer', - 'Körperpflege & Hygiene' => 'koerperpflege', - 'Kosmetik' => 'kosmetik', - 'Kostüme' => 'kostuem', - 'Kraftstoffe & Betriebsstoffe' => 'kraftstoffe-betriebsstoffe', - 'Krafttraining' => 'krafttraining', - 'Kredit' => 'kredit', - 'Kreditkarten' => 'kreditkarten', - 'Kreissägen' => 'kreissaegen', - 'Kreuzfahrten' => 'kreuzfahrten', - 'Krups' => 'krups', - 'Küche' => 'kueche', - 'Küchengeräte' => 'kuechengeraete', - 'Küchenhelfer' => 'kuechenhelfer', - 'Küchenmaschinen' => 'kuechenmaschinen', - 'Küchenmesser' => 'messer', - 'Küchenutensilien' => 'kuechenutensilien', - 'Kugelschreiber' => 'kugelschreiber', - 'Kühl-Gefrierkombinationen' => 'kuehl-gefrierkombination', - 'Kühlboxen' => 'kuehlboxen', - 'Kühlschränke' => 'kuehlschrank', - 'Kultur & Freizeit' => 'kultur-freizeit', - 'Kunst & Hobby' => 'hobby', - 'Kurse & Trainings' => 'kurse-trainings', - 'Lacoste' => 'lacoste', - 'Ladegeräte' => 'ladegeraete', - 'Lampen' => 'lampen', - 'Landhausstil' => 'landhausstil', - 'Landwirtschafts-Simulator' => 'landwirtschafts-simulator', - 'Laptops' => 'laptop', - 'Laserdrucker' => 'laserdrucker', - 'Last Minute Reisen' => 'last-minute', - 'Lattenroste' => 'lattenroste', - 'Laubsauger' => 'laubsauger', - 'Laufräder' => 'laufraeder', - 'Laufschuhe' => 'laufschuhe', - 'Laufsport' => 'laufsport', - 'Lautsprecher' => 'lautsprecher', - 'Lavazza' => 'lavazza', - 'Lay-Z-Spa Whirlpools' => 'lay-z-spa-whirlpools', - 'Lebensmittel' => 'lebensmittel', - 'Lebensmittel & Haushalt' => 'food', - 'LED Lampen' => 'led-lampen', - 'LEGO' => 'lego', - 'LEGO Adventskalender' => 'lego-adventskalender', - 'LEGO Architecture' => 'lego-architecture', - 'LEGO Batman' => 'lego-batman', - 'LEGO City' => 'lego-city', - 'LEGO Creator' => 'lego-creator', - 'LEGO Dimensions' => 'lego-dimensions', - 'LEGO DUPLO' => 'lego-duplo', - 'LEGO Friends' => 'lego-friends', - 'LEGO Harry Potter' => 'lego-harry-potter', - 'LEGO Marvel Super Heroes' => 'lego-marvel-super-heroes', - 'LEGO Nexo Knights' => 'lego-nexo-knights', - 'LEGO NINJAGO' => 'lego-ninjago', - 'LEGO Star Wars' => 'lego-star-wars', - 'LEGO Star Wars Millennium Falcon' => 'lego-star-wars-millennium-falcon', - 'LEGO Super Mario' => 'lego-super-mario', - 'LEGO Technic' => 'lego-technic', - 'LEGO The Simpsons' => 'lego-simpsons', - 'Leifheit' => 'leifheit', - 'Lenovo' => 'lenovo', - 'Lenovo Laptops' => 'lenovo-laptop', - 'Lenovo Tablets' => 'lenovo-tablet', - 'Lenovo ThinkPad' => 'lenovo-thinkpad', - 'Lenovo Yoga' => 'lenovo-yoga', - 'Leonardo' => 'leonardo', - 'Leuchtmittel' => 'leuchten', - 'Levi's' => 'levis', - 'Lexar' => 'lexar', - 'Lexmark' => 'lexmark', - 'LG' => 'lg', - 'LG Fernseher' => 'lg-fernsher', - 'LG G5' => 'lg-g5', - 'LG G6' => 'lg-g6', - 'LG G7 ThinQ' => 'lg-g7-thinq', - 'LG OLED Fernseher' => 'lg-oled-tv', - 'LG Smartphones' => 'lg-smartphones', - 'LG V30' => 'lg-v30', - 'Lichterketten' => 'lichterketten', - 'Liebeskind' => 'liebeskind', - 'Lieferservice' => 'lieferservice', - 'Lindt' => 'lindt', - 'Lindt Adventskalender' => 'lindt-adventskalender', - 'Logitech' => 'logitech', - 'Logitech G413' => 'logitech-g413', - 'Logitech G430' => 'logitech-g430', - 'Logitech G502 Proteus Spectrum' => 'logitech-g502', - 'Logitech G513' => 'logitech-g513', - 'Logitech G533' => 'logitech-g533', - 'Logitech G633 Artemis Spectrum' => 'logitech-g633', - 'Logitech G703' => 'logitech-g703', - 'Logitech G903' => 'logitech-g903', - 'Logitech G910 Orion Spectrum' => 'logitech-g910', - 'Logitech G915' => 'logitech-g915', - 'Logitech G933 Artemis Spectrum' => 'logitech-g933', - 'Logitech Harmony' => 'logitech-harmony', - 'Logitech Mäuse' => 'logitech-maeuse', - 'Logitech MX Master' => 'logitech-mx-master', - 'Logitech MX Master 2S' => 'logitech-mx-master-2s', - 'Logitech Tastaturen' => 'logitech-tastaturen', - 'Logitech Z333' => 'logitech-z333', - 'Logitech Z337' => 'logitech-z337', - 'Logitech Z906' => 'logitech-z906', - 'Luftbefeuchter' => 'luftbefeuchter', - 'Luftentfeuchter' => 'luftentfeuchter', - 'Luftmatratzen' => 'luftmatratzen', - 'Luftreiniger' => 'luftreiniger', - 'Luigi's Mansion' => 'luigis-mansion', - 'Luigi's Mansion 3' => 'luigis-mansion-3', - 'Lustiges Taschenbuch' => 'lustiges-taschenbuch', - 'M.2 SSD' => 'm2-ssd', - 'MacBook' => 'macbook', - 'MacBook Air' => 'macbook-air', - 'MacBook Pro' => 'macbook-pro', - 'MacBook Pro 13' => 'macbook-pro-13', - 'MacBook Pro 15' => 'macbook-pro-15', - 'MacBook Pro 16' => 'macbook-pro-16', - 'Mac mini' => 'mac-mini', - 'Mac Software' => 'mac-software', - 'Madden NFL' => 'madden-nfl', - 'Magazine' => 'magazine', - 'Magnat' => 'magnat', - 'Magnum Eis' => 'magnum-eis', - 'Mähroboter' => 'maehroboter', - 'Mainboards' => 'mainboards', - 'Make Up Adventskalender' => 'make-up-adventskalender', - 'Makita' => 'makita', - 'Makita Akkuschrauber' => 'makita-akkuschrauber', - 'Malerwerkzeuge' => 'malerpinsel', - 'Mangas' => 'mangas', - 'Marantz' => 'marantz', - 'Mario Kart' => 'mario-kart', - 'Mario Kart 8 Deluxe' => 'mario-kart-8-deluxe', - 'Marken' => 'marken', - 'Marvel' => 'marvel', - 'Marvel's Spider-Man: Miles Morales' => 'marvels-spider-man-miles-morales', - 'Mass Effect' => 'mass-effect', - 'Mass Effect: Andromeda' => 'mass-effect-andromeda', - 'Massivholzmöbel' => 'massivholzmoebel', - 'Mastercard' => 'mastercard', - 'Matratzen' => 'matratzen', - 'Maxi Cosi' => 'maxi-cosi', - 'Mazda' => 'mazda', - 'Medion' => 'medion', - 'Mercedes-Benz' => 'mercedes-benz', - 'Mesh WLAN Router' => 'mesh-wlan-router', - 'Metabo' => 'metabo', - 'Metro (Spiel)' => 'metro', - 'Metro Exodus' => 'metro-exodus', - 'Michael Kors' => 'michael-kors', - 'microSD' => 'microsd', - 'microSDHC' => 'microsdhc', - 'microSDXC' => 'microsdxc', - 'Microsoft Flight Simulator' => 'microsoft-flight-simulator', - 'Microsoft Software' => 'microsoft-software', - 'Microsoft Surface Notebooks' => 'microsoft-surface-notebooks', - 'Microsoft Surface Pro 4' => 'surface-pro-4', - 'Microsoft Surface Pro 6' => 'surface-pro-6', - 'Microsoft Surface Pro 7' => 'microsoft-surface-pro-7', - 'Microsoft Surface Tablets' => 'microsoft-surface', - 'Miele' => 'miele', - 'Miele Geschirrspüler' => 'miele-geschirrspueler', - 'Miele Staubsauger' => 'miele-staubsauger', - 'Miele Waschmaschinen' => 'miele-waschmaschine', - 'Mietwagen' => 'mietwagen', - 'Mikrofone' => 'mikrofone', - 'Mikrowellen' => 'mikrowelle', - 'Milchaufschäumer' => 'milchaufschaeumer', - 'Milka' => 'milka', - 'Minecraft' => 'minecraft', - 'Mineralwasser' => 'mineralwasser', - 'Minions' => 'minions', - 'Mini PCs' => 'mini-pc', - 'Mitsubishi' => 'mitsubishi', - 'Mittelerde' => 'middle-earth', - 'Mittelerde: Mordors Schatten' => 'mittelerde-mordors-schatten', - 'Mittelerde: Schatten des Krieges' => 'mittelerde-schatten-des-krieges', - 'Mixer & Rührer' => 'mixer', - 'Möbel' => 'moebel-deko', - 'Modellbau' => 'modellbau', - 'Modern wohnen' => 'modern-wohnen', - 'Monitore' => 'monitor', - 'Monkey 47' => 'monkey-47', - 'Monopoly' => 'monopoly', - 'Monster Hunter' => 'monster-hunter', - 'Monster Hunter: World' => 'monster-hunter-world', - 'Mortal Kombat' => 'mortal-kombat', - 'Mortal Kombat 11' => 'mortal-kombat-11', - 'Motorola' => 'motorola', - 'Motorola Smartphones' => 'motorola-smartphones', - 'Motorradbekleidung' => 'motorradbekleidung', - 'Motorradhelm' => 'motorradhelm', - 'Motorrad Zubehör' => 'motorrad', - 'Moto Z' => 'moto-z', - 'Mountainbikes' => 'mountainbikes', - 'MSI' => 'msi', - 'Mülleimer' => 'muelleimer', - 'Multifunktionsdrucker' => 'multifunktionsdrucker', - 'Multiroom Speaker' => 'multiroom', - 'Mund- & Zahnpflege' => 'mund-zahnpflege', - 'Mundschutzmasken' => 'mundschutzmasken', - 'Museums-Tickets' => 'museum', - 'Musical Tickets' => 'musical', - 'Musik' => 'musik', - 'Musik Apps' => 'musik-apps', - 'Musikinstrumente' => 'musikinstrumente', - 'Musik Streaming' => 'musik-streaming', - 'Müsli' => 'muesli', - 'Mustang' => 'mustang', - 'Mützen' => 'muetzen', - 'Nachtwäsche' => 'nachtwaesche', - 'Nähbedarf' => 'naehen', - 'Nähmaschinen' => 'naehmaschine', - 'Nahrungsergänzungsmittel' => 'nahrungsergaenzungsmittel', - 'Nahverkehr' => 'nahverkehr', - 'Naketano' => 'naketano', - 'NAS' => 'nas', - 'Nassrasierer' => 'rasierer', - 'Navigationsgeräte' => 'navigationsgeraete', - 'Neato' => 'neato', - 'Neato Robotics Botvac D7 Connected' => 'neato-botvac-d7', - 'Need for Speed' => 'need-for-speed', - 'Need for Speed Heat' => 'need-for-speed-heat', - 'Need for Speed Payback' => 'need-for-speed-payback', - 'Nerf' => 'nerf', - 'Nescafé' => 'nescafe', - 'Nespresso' => 'nespresso', - 'Nespresso Kaffeemaschinen' => 'nespresso-kaffeemaschinen', - 'Netflix' => 'netflix', - 'NETGEAR' => 'netgear', - 'NETGEAR Nighthawk' => 'netgear-nighthawk', - 'NETGEAR Orbi' => 'netgear-orbi', - 'NETGEAR Router' => 'netgear-router', - 'Netzteile' => 'netzteile', - 'Netzwerk' => 'netzwerk', - 'New Balance' => 'new-balance', - 'Nike' => 'nike', - 'Nike Air Force 1' => 'nike-air-force', - 'Nike Air Max' => 'nike-air-max', - 'Nike Air Max 270' => 'nike-air-max-270', - 'Nike Air Max 720' => 'nike-air-max-720', - 'Nike Air Max Thea' => 'nike-air-max-thea', - 'Nike Air Presto' => 'nike-presto', - 'Nike Free' => 'nike-free', - 'Nike Huarache' => 'nike-huarache', - 'Nike Roshe Run' => 'nike-roshe-run', - 'Nike Schuhe' => 'nike-schuhe', - 'Nikon' => 'nikon', - 'Nikon DSLR' => 'nikon-dslr', - 'Ni No Kuni' => 'ni-no-kuni', - 'Ni No Kuni: Der Fluch der Weißen Königin' => 'ni-no-kuni-der-fluch-der-weissen-koenigin', - 'Ni No Kuni II: Revenant Kingdom' => 'ni-no-kuni-ii', - 'Nintendo' => 'nintendo', - 'Nintendo 2DS Konsolen' => 'nintendo-2ds', - 'Nintendo 3DS Konsolen' => 'nintendo-3ds', - 'Nintendo 3DS Spiele' => 'nintendo-3ds-spiele', - 'Nintendo 3DS Zubehör' => 'nintendo-3ds-zubehoer', - 'Nintendo Classic Mini NES Konsolen' => 'nintendo-classic-mini-nes', - 'Nintendo Classic Mini SNES Konsolen' => 'nintendo-classic-mini-snes', - 'Nintendo eShop Guthaben' => 'nintendo-eshop-guthaben', - 'Nintendo Switch Controller' => 'nintendo-switch-controller', - 'Nintendo Switch Konsolen' => 'nintendo-switch', - 'Nintendo Switch Lite Konsolen' => 'nintendo-switch-lite', - 'Nintendo Switch Pro Controller' => 'nintendo-switch-pro-controller', - 'Nintendo Switch Spiele' => 'nintendo-switch-spiele', - 'Nintendo Switch Zubehör' => 'nintendo-switch-zubehoer', - 'Nintendo Zubehör' => 'nintendo-zubehoer', - 'Nissan' => 'nissan', - 'Nivea' => 'nivea', - 'Nokia' => 'nokia', - 'Nokia Handys' => 'nokia-handys', - 'Nudeln' => 'nudeln', - 'Nuki Smart Locks' => 'nuki-smart-lock', - 'Nüsse' => 'nuesse', - 'Nutella' => 'nutella', - 'Nvidia' => 'nvidia', - 'Nvidia GeForce' => 'nvidia-geforce', - 'Nvidia SHIELD TV' => 'nvidia-shield', - 'o2' => 'o2-netz', - 'Objektive' => 'objektiv', - 'Obst' => 'obst', - 'Obst & Gemüse' => 'obst-gemuese', - 'Oculus Quest' => 'oculus-quest', - 'Oculus Rift' => 'oculus-rift', - 'Office Programme' => 'office-programme', - 'OLED Fernseher' => 'oled-fernseher', - 'Olympus' => 'olympus', - 'On-Ear Kopfhörer' => 'on-ear-kopfhoerer', - 'OnePlus 3' => 'oneplus-3', - 'OnePlus 5' => 'oneplus-5', - 'OnePlus 6' => 'oneplus-6', - 'OnePlus 7' => 'oneplus-7', - 'OnePlus 7 Pro' => 'oneplus-7-pro', - 'OnePlus 7T' => 'oneplus-7t', - 'OnePlus 7T Pro' => 'oneplus-7t-pro', - 'OnePlus 8' => 'oneplus-8', - 'OnePlus 8 Pro' => 'one-plus-8-pro', - 'OnePlus 8T' => 'oneplus-8t', - 'OnePlus Nord' => 'oneplus-nord', - 'OnePlus Smartphones' => 'oneplus-smartphones', - 'Onkyo' => 'onkyo', - 'Opel' => 'opel', - 'OPPO Find X2 Lite' => 'oppo-find-x2-lite', - 'OPPO Find X2 Neo' => 'oppo-find-x2-neo', - 'OPPO Find X2 Pro' => 'oppo-find-x2-pro', - 'OPPO Reno2' => 'oppo-reno2', - 'OPPO Reno2 Z' => 'oppo-reno2-z', - 'OPPO Reno4 5G' => 'oppo-reno4-5g', - 'OPPO Reno4 Pro 5G' => 'oppo-reno4-pro-5g', - 'OPPO Reno4 Z 5G' => 'oppo-reno4-z-5g', - 'OPPO Smartphones' => 'oppo-smartphones', - 'Oral-B' => 'oral-b', - 'Oral-B Elektrische Zahnbürsten' => 'oral-b-elektrische-zahnbuersten', - 'Origin' => 'origin', - 'Osram' => 'osram', - 'Osram Smart+' => 'osram-smart-plus', - 'Osterdeko' => 'osterdeko', - 'Outdoor & Camping' => 'outdoor', - 'Outdoorbekleidung' => 'outdoorbekleidung', - 'Outdoorjacken' => 'outdoorjacken', - 'Outdoor Spielzeuge' => 'outdoor-spielzeug', - 'Over-Ear Kopfhörer' => 'over-ear-kopfhoerer', - 'Pampers' => 'pampers', - 'Panama Jack' => 'panama-jack', - 'Panasonic' => 'panasonic', - 'Panasonic Fernseher' => 'panasonic-fernseher', - 'Panasonic Kameras' => 'panasonic-kameras', - 'Panasonic Lumix' => 'panasonic-lumix', - 'Paper Mario: The Origami King' => 'paper-mario-the-origami-king', - 'Papiertapete' => 'papiertapete', - 'Parfum' => 'parfum', - 'Parfum Damen' => 'parfum-damen', - 'Parfum Herren' => 'parfum-herren', - 'Pauschalreisen' => 'pauschalreise', - 'Pavillons' => 'pavillons', - 'Paw Patrol' => 'paw-patrol', - 'PAYBACK' => 'payback', - 'Payday' => 'payday', - 'Payday 2' => 'payday-2', - 'paydirekt' => 'paydirekt', - 'PC Gaming Systeme' => 'pc-gaming-systeme', - 'PC Gaming Zubehör' => 'pc-gaming-zubehoer', - 'PC Gehäuse' => 'pc-gehaeuse', - 'PC Komponenten' => 'hardware', - 'PC Lautsprecher' => 'pc-lautsprecher', - 'PC Mäuse' => 'pc-maus', - 'PC Spiele' => 'pc-spiele', - 'PC Zubehör' => 'pc-zubehoer', - 'Pendelleuchten' => 'pendelleuchten', - 'Pentax' => 'pentax', - 'Pepe Jeans' => 'pepe-jeans', - 'Peppa Wutz' => 'peppa-wutz', - 'PepperBonus' => 'pepperbonus', - 'Pestos' => 'pestos', - 'Peugeot' => 'peugeot', - 'Pfannen' => 'pfannen', - 'Pflanzen' => 'pflanzen', - 'Philips' => 'philips', - 'Philips Fernseher' => 'philips-fernseher', - 'Philips Hue' => 'philips-hue', - 'Philips Hue E14' => 'philips-hue-e14', - 'Philips Hue E27' => 'philips-hue-e27', - 'Philips Hue Go' => 'philips-hue-go', - 'Philips Hue GU10' => 'philips-hue-gu10', - 'Philips Hue LightStrip' => 'philips-hue-lightstrip', - 'Philips Hue Play Gradient LightStrip' => 'philips-hue-play-gradient-lightstrip', - 'Philips Hue Play HDMI Sync Box' => 'philips-hue-play-hdmi-sync-box', - 'Philips Hue Play Lightbar' => 'philips-hue-play', - 'Philips OneBlade' => 'philips-oneblade', - 'Philips Rasierer' => 'philips-rasierer', - 'Philips Sonicare' => 'philips-sonicare', - 'Philips Staubsauger' => 'philips-staubsauger', - 'Philips Wecker' => 'philips-wecker', - 'Photoshop' => 'photoshop', - 'Pioneer' => 'pioneer', - 'Pizza' => 'pizza', - 'Plattenspieler' => 'plattenspieler', - 'Playboy' => 'playboy', - 'Playerunknown's Battlegrounds' => 'playerunknowns-battlegrounds', - 'PLAYMOBIL' => 'playmobil', - 'PLAYMOBIL Adventskalender' => 'playmobil-adventskalender', - 'PlayStation' => 'playstation', - 'PlayStation 4 Controller' => 'playstation-4-controller', - 'PlayStation 4 Konsolen' => 'playstation-4', - 'PlayStation 4 Pro Konsolen' => 'playstation-4-pro', - 'PlayStation 4 Spiele' => 'playstation-4-spiele', - 'PlayStation 5 Konsolen' => 'playstation-5', - 'PlayStation 5 Spiele' => 'playstation-5-spiele', - 'PlayStation Classic Konsolen' => 'playstation-classic', - 'PlayStation Now' => 'playstation-now', - 'PlayStation Plus' => 'playstation-plus', - 'PlayStation Zubehör' => 'playstation-zubehoer', - 'Plüschtiere' => 'plueschtiere', - 'Plus Size Mode' => 'plus-size-mode', - 'POCO F2 Pro' => 'poco-f2-pro', - 'POCO X3' => 'poco-x3', - 'Pokémon' => 'pokemon', - 'Pokémon: Let's Go' => 'pokemon-lets-go', - 'Pokémon Schwert und Schild' => 'pokemon-schwert-schild', - 'Pokémon Tekken' => 'pokemon-tekken', - 'Pokémon Ultrasonne & Ultramond' => 'pokemon-ultrasonne-ultramond', - 'Poloshirts' => 'poloshirts', - 'Polsterbetten' => 'polsterbetten', - 'Polyrattan Möbel' => 'polyrattan', - 'Pools' => 'pools', - 'Powerbanks' => 'powerbanks', - 'Powerbeats Pro' => 'powerbeats', - 'Preisfehler' => 'preisfehler', - 'Prepaid-Tarife' => 'prepaid-tarife', - 'Prime Gaming' => 'twitch-prime', - 'Pro Evolution Soccer' => 'pro-evolution-soccer', - 'Pro Evolution Soccer 2018' => 'pes-2018', - 'Pro Evolution Soccer 2019' => 'pes-2019', - 'Pro Evolution Soccer 2020' => 'pes-2020', - 'Proteine' => 'whey-proteine', - 'Prozessoren' => 'prozessoren', - 'PSN Guthaben' => 'psn-guthaben', - 'Puky' => 'puky', - 'Pullover' => 'pullover', - 'PUMA' => 'puma', - 'Pumps' => 'pumps', - 'Puppen' => 'puppen', - 'Puppenhäuser' => 'puppenhaeuser', - 'Puzzles' => 'puzzle', - 'Qeridoo' => 'qeridoo', - 'Qeridoo Fahrradanhänger' => 'qeridoo-fahrradanhaenger', - 'Qeridoo KidGoo 2' => 'qeridoo-kidgoo-2', - 'Qeridoo Sportrex 2' => 'qeridoo-sportrex-2', - 'Quiksilver' => 'quiksilver', - 'Raclettes' => 'raclettes', - 'Radios' => 'radios', - 'Radsport' => 'radsport', - 'Rasenmäher' => 'rasenmaeher', - 'Rasentrimmer' => 'rasentrimmer', - 'Rasierklingen' => 'rasierklingen', - 'Raspberry Pi' => 'raspberry-pi', - 'Rasur, Enthaarung & Trimmen' => 'rasur-enthaarung', - 'Rauchmelder' => 'rauchmelder', - 'Ravensburger' => 'ravensburger', - 'Ray-Ban' => 'ray-ban', - 'Razer DeathAdder' => 'razer-deathadder', - 'RC Autos' => 'rc-autos', - 'Red Bull' => 'red-bull', - 'Red Dead Redemption' => 'red-dead-redemption', - 'Red Dead Redemption 2' => 'red-dead-redemption-2', - 'Reebok' => 'reebok', - 'Regale' => 'regale', - 'Reifen' => 'reifen', - 'Reinigungsmittel' => 'reinigungsmittel', - 'Reise Apps' => 'reise-apps', - 'Reisen' => 'reisen', - 'Reiskocher' => 'reiskocher', - 'Remington' => 'remington', - 'Renault' => 'renault', - 'Rennräder' => 'rennraeder', - 'Repeater' => 'repeater', - 'Resident Evil' => 'resident-evil', - 'Resident Evil 2' => 'resident-evil-2', - 'Resident Evil 7' => 'resident-evil-7', - 'Restaurant' => 'restaurant', - 'Retro Stil' => 'retro-stil', - 'Rimowa' => 'rimowa', - 'Ring Fit Adventure' => 'ring-fit-adventure', - 'Rituals' => 'rituals', - 'Rituals Adventskalender' => 'rituals-adventskalender', - 'Roborock' => 'xiaomi-roborock', - 'Roborock S5 Max' => 'roborock-s5-max', - 'Roborock S6' => 'roborock-s6', - 'Roborock S6 MaxV' => 'roborock-s6-maxv', - 'ROCCAT' => 'roccat', - 'ROCCAT Tyon' => 'roccat-tyon', - 'Röcke' => 'roecke', - 'Rocket League' => 'rocket-league', - 'Roidmi Staubsauger' => 'roidmi-staubsauger', - 'Rollei' => 'rollei', - 'Rösle' => 'roesle', - 'Router' => 'router', - 'Roxy' => 'roxy', - 'RTX 2060' => 'rtx-2060', - 'RTX 2070' => 'rtx-2070', - 'RTX 2080' => 'rtx-2080', - 'RTX 2080 Ti' => 'rtx-2080-ti', - 'RTX 3070' => 'rtx-3070', - 'RTX 3080' => 'rtx-3080', - 'RTX 3090' => 'rtx-3090', - 'Rucksäcke' => 'rucksaecke', - 'Russell Hobbs' => 'russell-hobbs', - 'RX 480' => 'rx-480', - 'RX 570' => 'rx-570', - 'RX 580' => 'rx-580', - 'RX 590' => 'rx-590', - 'RX 5700 XT' => 'rx-5700-xt', - 'RX 6800' => 'rx-6800', - 'RX 6800 XT' => 'rx-6800-xt', - 'RX 6900 XT' => 'rx-6900-xt', - 'RX Vega 56' => 'rx-vega-56', - 'RX Vega 64' => 'rx-vega-64', - 'Sägen' => 'saegen', - 'Salomon' => 'salomon', - 'Samsonite' => 'samsonite', - 'Samsung' => 'samsung', - 'Samsung Fernseher' => 'samsung-fernseher', - 'Samsung Galaxy A7' => 'samsung-galaxy-a7', - 'Samsung Galaxy A8' => 'samsung-galaxy-a8', - 'Samsung Galaxy A51' => 'samsung-galaxy-a51', - 'Samsung Galaxy A71' => 'samsung-galaxy-a71', - 'Samsung Galaxy Buds' => 'samsung-galaxy-buds', - 'Samsung Galaxy Buds+' => 'samsung-galaxy-buds-plus', - 'Samsung Galaxy Buds Live' => 'samsung-galaxy-buds-live', - 'Samsung Galaxy Buds Pro' => 'samsung-galaxy-buds-pro', - 'Samsung Galaxy Note9' => 'samsung-galaxy-note-9', - 'Samsung Galaxy Note20' => 'samsung-galaxy-note20', - 'Samsung Galaxy Note20 Ultra' => 'samsung-galaxy-note20-ultra', - 'Samsung Galaxy S7' => 'samsung-galaxy-s7', - 'Samsung Galaxy S7 Edge' => 'samsung-galaxy-s7-edge', - 'Samsung Galaxy S8' => 'samsung-galaxy-s8', - 'Samsung Galaxy S8+' => 'samsung-galaxy-s8-plus', - 'Samsung Galaxy S9' => 'samsung-galaxy-s9', - 'Samsung Galaxy S9+' => 'samsung-galaxy-s9-plus', - 'Samsung Galaxy S10' => 'samsung-galaxy-s10', - 'Samsung Galaxy S10+' => 'samsung-galaxy-s10-plus', - 'Samsung Galaxy S10e' => 'samsung-galaxy-s10e', - 'Samsung Galaxy S20' => 'samsung-galaxy-s20', - 'Samsung Galaxy S20 FE' => 'samsung-galaxy-s20-fe', - 'Samsung Galaxy S20 Ultra' => 'samsung-galaxy-s20-ultra', - 'Samsung Galaxy S20+' => 'samsung-galaxy-s20-plus', - 'Samsung Galaxy S21 5G' => 'samsung-galaxy-s21-5g', - 'Samsung Galaxy S21 Ultra 5G' => 'samsung-galaxy-s21-ultra-5g', - 'Samsung Galaxy S21+ 5G' => 'samsung-galaxy-s21-plus-5g', - 'Samsung Galaxy Tab S4' => 'samsung-galaxy-tab-s4', - 'Samsung Galaxy Tab S6' => 'samsung-galaxy-tab-s6', - 'Samsung Galaxy Watch' => 'samsung-galaxy-watch', - 'Samsung Galaxy Watch Active2' => 'samsung-galaxy-watch-active-2', - 'Samsung Gear' => 'samsung-gear', - 'Samsung Gear S3' => 'samsung-gear-s3', - 'Samsung Gear VR' => 'samsung-gear-vr', - 'Samsung Kopfhörer' => 'samsung-kopfhoerer', - 'Samsung Kühlschränke' => 'samsung-kuehlschrank', - 'Samsung Monitore' => 'samsung-monitor', - 'Samsung QLED Fernseher' => 'samsung-qled-fernseher', - 'Samsung Smartphones' => 'samsung-smartphone', - 'Samsung SSD' => 'samsung-ssd', - 'Samsung Tablets' => 'samsung-tablet', - 'Samsung The Frame Fernseher' => 'samsung-the-frame-fernseher', - 'Samsung Waschmaschinen' => 'samsung-waschmaschine', - 'Sandalen' => 'sandalen', - 'SanDisk' => 'sandisk', - 'SanDisk SSD' => 'sandisk-ssd', - 'Sanitär & Armaturen' => 'sanitaer-armaturen', - 'Saucen' => 'saucen', - 'Saugroboter' => 'saugroboter', - 'Scanner' => 'scanner', - 'Schallplatten' => 'schallplatten', - 'Scheppach' => 'scheppach', - 'Schlafsäcke' => 'schlafsack', - 'Schlafsofas' => 'schlafsofas', - 'Schlafzimmer' => 'schlafzimmer', - 'Schlagschrauber' => 'schlagschrauber', - 'Schlauchboote' => 'schlauchboote', - 'Schleich' => 'schleich', - 'Schlitten' => 'schlitten', - 'Schmuck' => 'schmuck', - 'Schneefräsen' => 'schneefraesen', - 'Schnellkochtöpfe' => 'schnellkochtoepfe', - 'Schnürhalbschuhe' => 'schnuerhalbschuhe', - 'Schokolade' => 'schokolade', - 'Schraubendreher' => 'schraubendreher', - 'Schreibgeräte' => 'schreibgeraete', - 'Schreibtische' => 'schreibtisch', - 'Schuhe' => 'schuhe', - 'Schuhschränke' => 'schuhschraenke', - 'Schulbedarf' => 'schulbedarf', - 'Schulranzen' => 'schulranzen', - 'Schutzfolien' => 'schutzfolien', - 'Schwangerschaft' => 'schwangerschaft', - 'Schwerlastregale' => 'schwerlastregale', - 'Scooter' => 'scooter', - 'Scotch Whisky' => 'scotch-whisky', - 'SDHC Speicherkarten' => 'sdhc-speicherkarten', - 'SD Karten' => 'sd-karten', - 'Seagate' => 'seagate', - 'Sea of Thieves' => 'sea-of-thieves', - 'Seat' => 'seat', - 'Sega Mega Drive Mini Konsolen' => 'sega-mega-drive-mini', - 'Seidensticker' => 'seidensticker', - 'Sekiro: Shadows Die Twice' => 'sekiro', - 'Senf' => 'senf', - 'Sennheiser' => 'sennheiser', - 'Senseo' => 'senseo', - 'Service-Verträge' => 'service-vertraege', - 'Sessel' => 'sessel', - 'Sextoys' => 'sextoys', - 'Shadow of the Tomb Raider' => 'shadow-of-the-tomb-raider', - 'Shampoo' => 'shampoo', - 'Sharkoon' => 'sharkoon', - 'Sharp' => 'sharp', - 'Shenmue' => 'shenmue', - 'Shenmue I & II' => 'shenmue-i-ii', - 'Shenmue III' => 'shenmue-iii', - 'Shishas' => 'shishas', - 'Shishas & Zubehör' => 'shishas-zubehoer', - 'Shoop' => 'shoop', - 'Shops: Erfahrungen' => 'shops', - 'Shorts' => 'shorts', - 'Sicherheitstechnik' => 'sicherheitstechnik', - 'Side-by-Side-Kühlschränke' => 'side-by-side-kuehlschrank', - 'Sid Meier's Civilization VI' => 'sid-meiers-civilization-vi', - 'Sid Meier’s Civilization' => 'sid-meiers-civilization', - 'Siemens' => 'siemens', - 'Siemens Geschirrspüler' => 'siemens-geschirrspueler', - 'Siemens Kühlschränke' => 'siemens-kuehlschrank', - 'Siemens Waschmaschinen' => 'siemens-waschmaschine', - 'Silit' => 'silit', - 'Skandi Stil' => 'skandi-stil', - 'Skateboards' => 'skateboard', - 'Skaten' => 'skaten', - 'Ski & Snowboard' => 'snowboard', - 'Skoda' => 'skoda', - 'Sky' => 'sky', - 'Sky Ticket' => 'sky-ticket', - 'Smarte Beleuchtung' => 'smarte-beleuchtung', - 'Smarte Wecker' => 'smarte-wecker', - 'Smart Home' => 'smart-home', - 'Smart Home Steckdosen' => 'smart-home-steckdosen', - 'Smart Locks' => 'smart-lock', - 'Smartphones' => 'smartphone', - 'Smartphones unter 200€' => 'smartphones-unter-200-euro', - 'Smart Speaker' => 'smart-speaker', - 'Smart Tech & Gadgets' => 'smart-tech', - 'Smartwatches' => 'smartwatch', - 'Smoothie Maker' => 'smoothie-maker', - 'Snacks & Knabberzeug' => 'snacks-knabberzeug', - 'Sneakers' => 'sneaker', - 'Socken' => 'socken', - 'SodaStream' => 'sodastream', - 'Sofas' => 'sofa', - 'Sofortbildkameras' => 'sofortbildkameras', - 'Softdrinks' => 'softdrinks', - 'Software' => 'software', - 'Software & Apps' => 'apps-software', - 'Solarleuchten' => 'solarleuchten', - 'Somat' => 'somat', - 'Sommerreifen' => 'sommerreifen', - 'Sonnenbrillen' => 'sonnenbrillen', - 'Sonnencreme' => 'sonnencreme', - 'Sonnenpflege' => 'sonnenpflege', - 'Sonnenschirme' => 'sonnenschirme', - 'Sonoff' => 'sonoff', - 'Sonos' => 'sonos', - 'Sonos Beam' => 'sonos-beam', - 'Sonos Move' => 'sonos-move', - 'Sonos One' => 'sonos-one', - 'Sonos PLAY:1' => 'sonos-play-1', - 'Sonos PLAY:3' => 'sonos-play-3', - 'Sonos Play:5 (Five)' => 'sonos-play-5', - 'Sonos Playbar' => 'sonos-playbar', - 'Sonos Playbase' => 'sonos-playbase', - 'Sonstiges' => 'diverses', - 'Sony' => 'sony', - 'Sony Alpha 7' => 'sony-alpha-7', - 'Sony Alpha 7 II' => 'sony-alpha-7-ii', - 'Sony Alpha 7 III' => 'sony-alpha-7-iii', - 'Sony Alpha 6000' => 'sony-alpha-6000', - 'Sony Alpha 6300' => 'sony-alpha-6300', - 'Sony Alpha 6400' => 'sony-alpha-6400', - 'Sony Alpha 6500' => 'sony-alpha-6500', - 'Sony DualSense Wireless-Controller' => 'playstation-5-controller', - 'Sony Fernseher' => 'sony-fernseher', - 'Sony Kameras' => 'sony-kameras', - 'Sony Kopfhörer' => 'sony-kopfhoerer', - 'Sony PlayStation VR' => 'sony-playstation-vr', - 'Sony PULSE 3D Wireless Headset' => 'sony-pulse-3d-wireless-headset', - 'Sony WF-1000XM3' => 'sony-wf-1000xm3', - 'Sony WH-1000XM3' => 'sony-wh-1000xm3', - 'Sony WH-1000XM4' => 'sony-wh-1000xm4', - 'Sony Xperia' => 'sony-xperia', - 'Sony Xperia X' => 'sony-xperia-x', - 'Sony Xperia XA' => 'sony-xperia-xa', - 'Sony Xperia XZ' => 'sony-xperia-xz', - 'Soundbar' => 'soundbar', - 'Soundbase' => 'soundbase', - 'Soundkarten' => 'soundkarten', - 'South Park: Die rektakuläre Zerreißprobe' => 'south-park-die-rektakulaere-zerreissprobe', - 'Spartipps' => 'spartipps', - 'Speicherkarten' => 'speicherkarten', - 'Speichermedien' => 'speichermedien', - 'Speiseöle' => 'speiseoele', - 'Spiegelreflexkameras' => 'spiegelreflexkamera', - 'Spiele & Brettspiele' => 'spiele-brettspiele', - 'Spiele Apps' => 'spiele-apps', - 'Spielekonsolen' => 'spielekonsolen', - 'Spielfiguren & Spielsets' => 'spielfiguren-spielsets', - 'Spielzeuge' => 'spielzeug', - 'Spirituosen' => 'spirituosen', - 'Sport & Outdoor' => 'sport', - 'Sportbekleidung' => 'sportbekleidung', - 'Sport Bild' => 'sport-bild', - 'Sportnahrung' => 'sportlernahrung', - 'Sporttasche' => 'sporttasche', - 'Spotify' => 'spotify', - 'Spülmaschinentabs' => 'spuelmaschinentabs', - 'Spyro Reignited Trilogy' => 'spyro-reignited-trilogy', - 'SSD' => 'ssd', - 'Stabmixer' => 'stabmixer', - 'Städtereisen' => 'staedtereise', - 'Standmixer' => 'standmixer', - 'Star Trek' => 'star-trek', - 'Star Wars' => 'star-wars', - 'Star Wars: Battlefront 2' => 'star-wars-battlefront-2', - 'Star Wars: Squadrons' => 'star-wars-squadrons', - 'Star Wars Battlefront' => 'star-wars-battlefront', - 'Star Wars Jedi: Fallen Order' => 'star-wars-jedi-fallen-order', - 'Staubsauger' => 'staubsauger', - 'Staubsaugerbeutel' => 'staubsaugerbeutel', - 'Staubsauger ohne Beutel' => 'staubsauger-ohne-beutel', - 'Steam' => 'steam', - 'Steckschlüssel' => 'steckschluessel', - 'SteelSeries' => 'steelseries', - 'Stehlampen' => 'stehlampen', - 'Steiff' => 'steiff', - 'Stern (Magazin)' => 'stern-magazin', - 'Stichsägen' => 'stichsaegen', - 'Stiefel' => 'stiefel', - 'Stiefeletten' => 'stiefeletten', - 'Stiftung Warentest' => 'stiftung-warentest-magazin', - 'Streaming-Dienste' => 'streaming-dienste', - 'Streaming Lautsprecher' => 'streaming-lautsprecher', - 'Strom & Gas' => 'strom-gas', - 'Stromtarif' => 'stromtarif', - 'Studentenrabatte' => 'studentenrabatte', - 'Stühle' => 'stuehle', - 'Subwoofer' => 'subwoofer', - 'SUP Boards' => 'sup-boards', - 'Superdry' => 'superdry', - 'Super Mario' => 'super-mario', - 'Super Mario 3D All-Stars' => 'super-mario-3d-all-stars', - 'Super Mario Maker 2' => 'super-mario-maker-2', - 'Super Mario Odyssey' => 'super-mario-odyssey', - 'Super Mario Party' => 'super-mario-party', - 'Supermarkt' => 'supermarkt', - 'Super Smash Bros. Ultimate' => 'super-smash-bros-ultimate', - 'Süßigkeiten' => 'suessigkeiten', - 'Synology' => 'synology', - 'Syoss' => 'syoss', - 'Systemkameras' => 'systemkamera', - 'T-Shirts' => 't-shirts', - 'Tablets' => 'tablet', - 'Tablet Zubehör' => 'tablet-zubehoer', - 'tado° Smartes Heizkörper-Thermostat' => 'tado-smartes-thermostat', - 'Tamaris' => 'tamaris', - 'Tangle Teezer' => 'tangle-teezer', - 'Tanqueray' => 'tanqueray', - 'Tapeten' => 'tapeten', - 'Taschen' => 'taschen', - 'Taschenlampen' => 'taschenlampen', - 'Taschentücher' => 'taschentuecher', - 'Tassimo' => 'tassimo', - 'Tassimo Kaffeemaschinen' => 'tassimo-kaffeemaschinen', - 'Tastaturen' => 'tastatur', - 'TCL Fernseher' => 'tcl-fernseher', - 'Team Sonic Racing' => 'team-sonic-racing', - 'Teamsport' => 'teamsport', - 'Tee' => 'tee', - 'Tefal' => 'tefal', - 'Tefal OptiGrills' => 'tefal-optigrill', - 'Tefal Pfannen' => 'tefal-pfannen', - 'Tekken' => 'tekken', - 'Tekken 7' => 'tekken-7', - 'Telefon- & Internet-Verträge' => 'telefon-internet', - 'Telefone & Zubehör' => 'handy-smartphone', - 'Telekom' => 'telekom-net', - 'Telekom Magenta' => 'telekom-magenta', - 'Telekom SmartHome' => 'telekom-smarthome', - 'Teppiche' => 'teppiche', - 'Tesla' => 'tesla', - 'Tetris' => 'tetris', - 'Teufel' => 'teufel', - 'The Elder Scrolls' => 'the-elder-scrolls', - 'The Elder Scrolls V: Skyrim' => 'skyrim', - 'The Evil Within' => 'the-evil-within', - 'The Evil Within 2' => 'the-evil-within-2', - 'The Last of Us' => 'the-last-of-us', - 'The Last of Us Part II' => 'the-last-of-us-part-ii', - 'The Legend of Zelda' => 'the-legend-of-zelda', - 'The Legend of Zelda: Breath of the Wild' => 'zelda-breath-of-the-wild', - 'The Legend of Zelda: Link's Awakening' => 'zelda-links-awakening', - 'The Legend of Zelda: Skyward Sword HD' => 'zelda-skyward-sword-hd', - 'The North Face' => 'the-north-face', - 'The Outer Worlds' => 'the-outer-worlds', - 'Thermosflaschen' => 'thermosflaschen', - 'Thermoskannen' => 'thermoskanne', - 'The Witcher' => 'the-witcher', - 'The Witcher 3' => 'the-witcher-3', - 'Thule' => 'thule', - 'Thule Chariot Fahrradanhänger' => 'thule-chariot-fahrradanhaenger', - 'Thule Dachboxen' => 'thule-dachboxen', - 'Thule Fahrradträger' => 'thule-fahrradtraeger', - 'Tickets & Shows' => 'erlebnisse', - 'Tiefkühlkost' => 'tiefkuehkost', - 'Timberland' => 'timberland', - 'Tintenstrahldrucker' => 'tintenstrahldrucker', - 'Tischlampen' => 'tischlampen', - 'Tischtennis' => 'tischtennis', - 'Tischtennisplatten' => 'tischtennisplatten', - 'Tischtennisschläger' => 'tischtennisschlaeger', - 'Toaster' => 'toaster', - 'Toilettenpapier' => 'toilettenpapier', - 'tolino' => 'tolino', - 'Tomb Raider' => 'tomb-raider', - 'Tom Clancy's' => 'tom-clancys', - 'Tom Clancy's: Ghost Recon Wildlands' => 'tom-clancys-ghost-recon-wildlands', - 'Tom Clancy's Ghost Recon Breakpoint' => 'tom-clancys-ghost-recon-breakpoint', - 'Tom Clancy's The Division 2' => 'tom-clancy-the-division-2', - 'Tommy Hilfiger' => 'tommy-hilfiger', - 'TOM TAILOR' => 'tom-tailor', - 'Toner' => 'toner', - 'Tonic Water' => 'tonic-water', - 'Toniebox' => 'toniebox', - 'Tonies Figuren' => 'tonie-figuren', - 'Töpfe' => 'toepfe', - 'Töpfe & Pfannen' => 'kochen', - 'Toplader' => 'toplader', - 'Toshiba' => 'toshiba', - 'Total War' => 'total-war', - 'Toyota' => 'toyota', - 'TP-Link' => 'tp-link', - 'TP-Link Router' => 'tp-link-router', - 'Trampoline' => 'trampolin', - 'TREKSTOR' => 'trekstor', - 'Trockner' => 'trockner', - 'Tropical Islands' => 'tropical-island', - 'Tropico' => 'tropico', - 'Tropico 5' => 'tropico-5', - 'Tropico 6' => 'tropico-6', - 'TV & Video' => 'tv-video', - 'TV Boxen' => 'tv-box', - 'TV Spielfilm' => 'tv-spielfilm', - 'TV Wandhalterungen' => 'tv-wandhalterung', - 'TV Zubehör' => 'tv-zubehoer', - 'Übergangsjacken' => 'uebergangsjacken', - 'Überwachungskamera' => 'ueberwachungskamera', - 'UE BLAST' => 'ue-blast', - 'UE BOOM' => 'ue-boom', - 'UE BOOM 2' => 'ue-boom-2', - 'UE BOOM 3' => 'ue-boom-3', - 'UE MEGABLAST' => 'ue-megablast', - 'UE MEGABOOM' => 'ue-megaboom', - 'UE MEGABOOM 3' => 'ue-megaboom-3', - 'UE WONDERBOOM' => 'ue-wonderboom', - 'UE WONDERBOOM 2' => 'ue-wonderboom-2', - 'UGG' => 'ugg', - 'Uhren' => 'uhren', - 'Umstandsmode' => 'umstandsmode', - 'Uncharted' => 'uncharted', - 'Uncharted 4' => 'uncharted-4', - 'Uncharted: The Lost Legacy' => 'uncharted-the-lost-legacy', - 'Under Armour' => 'under-armour', - 'Universalfernbedienungen' => 'universalfernbedienungen', - 'Unterwäsche' => 'unterwaesche', - 'Uplay' => 'uplay', - 'Urban Sport' => 'urban-sport', - 'Urlaub' => 'urlaub', - 'USB Sticks' => 'usb-stick', - 'Vakuumierer' => 'vakuumierer', - 'Vans' => 'vans', - 'Vans Old Skool' => 'vans-old-skool', - 'Vans Schuhe' => 'vans-schuhe', - 'Vaude' => 'vaude', - 'Ventilatoren' => 'ventilator', - 'Verbandskästen' => 'verbandskaesten', - 'Versicherung' => 'versicherung', - 'Versicherung & Finanzen' => 'vertraege-finanzen', - 'Videobearbeitungsprogramme' => 'videobearbeitungsprogramme', - 'Video Player' => 'video-player', - 'Videospiele' => 'videospiele', - 'Video Streaming' => 'video-streaming', - 'Vileda' => 'vileda', - 'Villeroy & Boch' => 'villeroy-boch', - 'Virenschutz' => 'virenschutz', - 'VISA' => 'visa', - 'Vliestapeten' => 'vliestapete', - 'Vodafone' => 'vodafone-netz', - 'Vodka' => 'vodka', - 'Volvo' => 'volvo', - 'Vorratsdosen' => 'vorratsdosen', - 'Vorstellungsrunde' => 'vorstellungsrunde', - 'VPN' => 'vpn', - 'VPS' => 'vps', - 'VR Brillen' => 'vr-brille', - 'VR Spiele' => 'vr-spiele', - 'VTech' => 'vtech', - 'VW' => 'vw', - 'Waffeleisen' => 'waffeleisen', - 'Wandbilder' => 'wandtattoos', - 'Wanderrucksäcke' => 'wanderrucksack', - 'Wanderschuhe' => 'wanderschuhe', - 'Wandersport' => 'hiking', - 'Wandfarben' => 'wandfarben', - 'Wandlampen' => 'wandlampen', - 'Wäscheständer' => 'waeschestaender', - 'Waschmaschinen' => 'waschmaschinen', - 'Waschmittel' => 'waschmittel', - 'Waschtrockner' => 'waschtrockner', - 'Wasserfilter' => 'wasserfilter', - 'Wasserkocher' => 'wasserkocher', - 'Wasserkühlung' => 'wasserkuehlung', - 'Wasserspielzeuge' => 'wasserspielzeug', - 'Wassersport' => 'wassersport', - 'Watch Dogs' => 'watch-dogs', - 'Watch Dogs 2' => 'watch-dogs-2', - 'Watch Dogs: Legion' => 'watch-dogs-legion', - 'WC Sitze' => 'wc-sitze', - 'WD-40' => 'wd-40', - 'Wearables' => 'wearable', - 'Webcams' => 'webcam', - 'Weber Gasgrills' => 'weber-gasgrill', - 'Weber Grills' => 'weber-grill', - 'Weihnachtsbäume' => 'weihnachtsbaum', - 'Weihnachtsbeleuchtung' => 'weihnachtsbeleuchtung', - 'Weihnachtsdeko' => 'weihnachtsdeko', - 'Weihnachtspullover' => 'weihnachtspullover', - 'Wein' => 'wein', - 'Wellensteyn' => 'wellensteyn', - 'Wellness & Gesundheit' => 'wellness-massagen', - 'Wera' => 'wera', - 'Werkstatt & Service' => 'werkstatt-service', - 'Werkstatteinrichtungen' => 'werkstatteinrichtungen', - 'Werkzeuge' => 'werkzeug', - 'Werkzeugkoffer' => 'werkzeugkoffer', - 'Wesco Mülleimer' => 'wesco-muelleimer', - 'Western Digital' => 'western-digital', - 'Wetterstationen' => 'wetterstationen', - 'Whirlpools' => 'whirlpools', - 'Whisky' => 'whisky', - 'Wiko' => 'wiko', - 'Wilkinson Sword Rasierer' => 'wilkinson-sword', - 'Windeln' => 'windeln', - 'Winkelschleifer' => 'winkelschleifer', - 'Winterdeko' => 'winterdeko', - 'Winterjacken' => 'winterjacken', - 'Winterreifen' => 'winterreifen', - 'Winterstiefel' => 'winterstiefel', - 'Wireless Charger' => 'wireless-charger', - 'Wirtschaftswoche' => 'wirtschaftswoche', - 'WMF' => 'wmf', - 'WMF Besteck' => 'wmf-besteck', - 'WMF Topfset' => 'wmf-topfset', - 'Wohnzimmermöbel' => 'wohnzimmer', - 'Wolfenstein' => 'wolfenstein', - 'Wolfenstein II: The New Colossus' => 'wolfenstein-2-the-new-colossus', - 'Womanizer' => 'womanizer', - 'World of Warcraft' => 'world-of-warcraft', - 'Wrangler' => 'wrangler', - 'X570 Mainboard' => 'x570-mainboard', - 'Xbox' => 'xbox', - 'Xbox Controller' => 'xbox-controller', - 'Xbox Elite Wireless Controller' => 'xbox-one-elite-controller', - 'Xbox Elite Wireless Controller 2' => 'xbox-one-elite-controller-2', - 'Xbox Game Pass' => 'xbox-game-pass', - 'Xbox Game Pass Ultimate' => 'xbox-game-pass-ultimate', - 'Xbox Guthaben' => 'xbox-guthaben', - 'Xbox Live Gold' => 'xbox-live', - 'Xbox One Controller' => 'xbox-one-controller', - 'Xbox One S Konsolen' => 'xbox-one-s', - 'Xbox One Spiele' => 'xbox-one-spiele', - 'Xbox One X Konsolen' => 'xbox-one-x', - 'Xbox Series S Konsolen' => 'xbox-series-s', - 'Xbox Series X Controller' => 'xbox-series-x-controller', - 'Xbox Series X Konsolen' => 'xbox-series-x', - 'Xbox Series X Spiele' => 'xbox-series-x-spiele', - 'Xbox Wireless Headset' => 'xbox-wireless-headset', - 'Xbox Zubehör' => 'xbox-zubehoer', - 'Xiaomi' => 'xiaomi', - 'Xiaomi Air Laptop' => 'xiaomi-air', - 'Xiaomi E-Scooter' => 'xiaomi-e-scooter', - 'Xiaomi Fernseher' => 'xiaomi-fernseher', - 'Xiaomi Kopfhörer' => 'xiaomi-kopfhoerer', - 'Xiaomi Mi 5S' => 'xiaomi-mi-5', - 'Xiaomi Mi 6' => 'xiaomi-mi-6', - 'Xiaomi Mi 8' => 'xiaomi-mi-8', - 'Xiaomi Mi 8 Lite' => 'xiaomi-mi-8-lite', - 'Xiaomi Mi 8 Pro' => 'xiaomi-mi-8-pro', - 'Xiaomi Mi 9' => 'xiaomi-mi-9', - 'Xiaomi Mi 9 Lite' => 'xiaomi-mi-9-lite', - 'Xiaomi Mi 9 SE' => 'xiaomi-mi-9-se', - 'Xiaomi Mi 9T' => 'xiaomi-mi-9t', - 'Xiaomi Mi 9T Pro' => 'xiaomi-mi-9t-pro', - 'Xiaomi Mi 10' => 'xiaomi-mi-10', - 'Xiaomi Mi 10 Lite' => 'xiaomi-mi-10-lite', - 'Xiaomi Mi 10 Pro' => 'xiaomi-mi-10-pro', - 'Xiaomi Mi 11' => 'xiaomi-mi-11', - 'Xiaomi Mi A1' => 'xiaomi-mi-a1', - 'Xiaomi Mi A2' => 'xiaomi-mi-a2', - 'Xiaomi Mi AirDots' => 'xiaomi-mi-airdots', - 'Xiaomi Mi AirDots Pro' => 'xiaomi-airdots-pro', - 'Xiaomi Mi Band' => 'xiaomi-mi-band', - 'Xiaomi Mi Band 4' => 'xiaomi-mi-band-4', - 'Xiaomi Mi Band 5' => 'xiaomi-mi-band-5', - 'Xiaomi Mi Electric Scooter 1S' => 'xiaomi-mi-scooter-1s', - 'Xiaomi Mi Electric Scooter M365' => 'xiaomi-mi-electric-scooter-m365', - 'Xiaomi Mi Electric Scooter Pro 2' => 'xiaomi-mi-electric-scooter-pro-2', - 'Xiaomi Mi Mix' => 'xiaomi-mi-mix', - 'Xiaomi Mi Mix 3' => 'xiaomi-mi-mix-3', - 'Xiaomi Mi Note' => 'xiaomi-mi-note', - 'Xiaomi Mi Note 10' => 'xiaomi-mi-note-10', - 'Xiaomi Mi Note 10 Lite' => 'xiaomi-mi-note-10-lite', - 'Xiaomi Mi Note 10 Pro' => 'xiaomi-mi-note-10-pro', - 'Xiaomi Mi TV 4S' => 'xiaomi-mi-smart-tv-4s', - 'Xiaomi Mi TV Stick' => 'xiaomi-mi-tv-stick', - 'Xiaomi Pocophone F1' => 'xiaomi-pocophone-f1', - 'Xiaomi Redmi 9' => 'xiaomi-redmi-9', - 'Xiaomi Redmi 9A' => 'xiaomi-redmi-9a', - 'Xiaomi Redmi AirDots' => 'xiaomi-redmi-airdots', - 'Xiaomi Redmi Note 4' => 'xiaomi-redmi-note-4', - 'Xiaomi Redmi Note 5' => 'xiaomi-redmi-note-5', - 'Xiaomi Redmi Note 8' => 'xiaomi-redmi-note-8', - 'Xiaomi Redmi Note 8 Pro' => 'xiaomi-redmi-note-8-pro', - 'Xiaomi Redmi Note 9' => 'xiaomi-redmi-note-9', - 'Xiaomi Redmi Note 9 Pro' => 'xiaomi-redmi-note-9-pro', - 'Xiaomi Redmi Note 9S' => 'xiaomi-redmi-note-9s', - 'Xiaomi Redmi Note 10' => 'xiaomi-redmi-note-10', - 'Xiaomi Redmi Note 10 Pro' => 'xiaomi-redmi-note-10-pro', - 'Xiaomi Smart Home' => 'xiaomi-smart-home', - 'Xiaomi Smartphones' => 'xiaomi-smartphones', - 'Xiaomi YouPin' => 'xiaomi-youpin', - 'XMG' => 'xmg', - 'Yamaha' => 'yamaha', - 'Yeelight' => 'xiaomi-yeelight', - 'Yoga' => 'yoga', - 'Yogamatten' => 'yogamatten', - 'Yoshi's Crafted World' => 'yoshis-crafted-world', - 'Zahnbürsten' => 'zahnbuersten', - 'Zahnpasta' => 'zahnpasta', - 'Zahnzusatzversicherung' => 'zahnzusatzversicherung', - 'Zeitschriften' => 'zeitschriften-magazine', - 'Zelte' => 'zelte', - 'Zirkel' => 'zirkel', - 'Zoo-Tickets' => 'zoo', - 'Zotac' => 'zotac', - 'ZTE Smartphones' => 'zte-smartphones', - 'ZWILLING' => 'zwilling', - 'ZWILLING Besteck' => 'zwilling-besteck', - ] - ], + 'type' => 'text', + 'exampleValue' => 'dsl', + 'title' => 'Gruppenname in der URL: Der einzugebende Gruppenname steht nach "https://www.mydealz.de/gruppe/" und vor einem "?". +Beispiel: Wenn die URL der Gruppe, die im Browser angezeigt wird, : +https://www.mydealz.de/gruppe/dsl?sortBy=temp +Dann geben Sie ein: +dsl', + ], 'order' => [ 'name' => 'sortieren nach', 'type' => 'list', @@ -2034,5 +96,7 @@ class MydealsBridge extends PepperBridgeAbstract 'title-talk' => 'Überwachung Diskussion', 'deal-type' => 'Angebotsart', 'localdeal' => 'Lokales Angebot', + 'context-hot' => '-hot', + 'context-new' => '-new', ]; } diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 4e9ab0b5..c64ad2fa 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -263,11 +263,40 @@ HEREDOC; */ private function getTalkTitle() { - $html = getSimpleHTMLDOMCached($this->getInput('url')); - $title = $html->find('title', 0)->plaintext; + $cacheKey = $this->getInput('url') . 'TITLE'; + $title = $this->loadCacheValue($cacheKey); + // The cache does not contain the title of the bridge, we must get it and save it in the cache + if ($title === null) { + $html = getSimpleHTMLDOMCached($this->getInput('url')); + $title = $html->find('title', 0)->plaintext; + // Save the value in the cache for the next 15 days + $this->saveCacheValue($cacheKey, $title, 86400 * 15); + } return $title; } + /** + * Get the Title from a Group if it exists + * @return string String of the Talk title + */ + private function getGroupTitle() + { + $cacheKey = $this->getInput('group') . 'TITLE'; + $title = $this->loadCacheValue($cacheKey); + // The cache does not contain the title of the bridge, we must get it and save it in the cache + if ($title == null) { + $html = getSimpleHTMLDOMCached($this->getGroupURI()); + // Search the title in the javascript mess + preg_match('/threadGroupName":"([^"]*)","threadGroupUrlName":"' . $this->getInput('group') . '"/m', $html, $matches); + $title = $matches[1]; + // Save the value in the cache for the next 15 days + $this->saveCacheValue($cacheKey, $title, 86400 * 15); + } + + $order = $this->getKey('order'); + return $title . ' - ' . $order; + } + /** * Get the HTML Title code from an item * @return string String of the deal title @@ -429,7 +458,7 @@ HEREDOC; return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-keyword') . ' : ' . $this->getInput('q'); break; case $this->i8n('context-group'): - return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-group') . ' : ' . $this->getKey('group'); + return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-group') . ' : ' . $this->getGroupTitle(); break; case $this->i8n('context-talk'): return $this->i8n('bridge-name') . ' - ' . $this->i8n('title-talk') . ' : ' . $this->getTalkTitle(); @@ -496,8 +525,15 @@ HEREDOC; $group = $this->getInput('group'); $order = $this->getInput('order'); + // This permit to keep the existing Feed to work + if ($order == $this->i8n('context-hot')) { + $sortBy = 'temp'; + } else if ($order == $this->i8n('context-new')) { + $sortBy = 'new'; + } + $url = $this->i8n('bridge-uri') - . $this->i8n('uri-group') . $group . $order; + . $this->i8n('uri-group') . $group . '?sortBy=' . $sortBy; return $url; } From 086ef7f8a7edba3697c976ffc8cef2fcd24578db Mon Sep 17 00:00:00 2001 From: Matt Connell Date: Sat, 23 Nov 2024 18:12:36 +0000 Subject: [PATCH 236/423] feat: add WKYT bridge (#4337) --- bridges/WKYTNewsBridge.php | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 bridges/WKYTNewsBridge.php diff --git a/bridges/WKYTNewsBridge.php b/bridges/WKYTNewsBridge.php new file mode 100644 index 00000000..e3b95f00 --- /dev/null +++ b/bridges/WKYTNewsBridge.php @@ -0,0 +1,27 @@ +find('.card-body'); + + foreach ($articles as $article) { + $item = []; + $url = $article->find('.headline a', 0); + $item['uri'] = $url->href; + $item['title'] = trim($url->plaintext); + $item['author'] = $article->find('.author', 0)->plaintext; + $item['content'] = $article->find('.deck', 0)->plaintext; + $this->items[] = $item; + } + } +} From e3260ff529afde368eaf9503d348d29159bb0fbe Mon Sep 17 00:00:00 2001 From: Sebastian Wolf <117176763+swofl@users.noreply.github.com> Date: Sat, 23 Nov 2024 19:19:20 +0100 Subject: [PATCH 237/423] [NordbayernBridge] fill item categories if available (#4338) --- bridges/NordbayernBridge.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bridges/NordbayernBridge.php b/bridges/NordbayernBridge.php index 48157921..94319e2b 100644 --- a/bridges/NordbayernBridge.php +++ b/bridges/NordbayernBridge.php @@ -163,6 +163,13 @@ class NordbayernBridge extends BridgeAbstract $item['content'] .= $this->getUseFullContent($content); } + $categories = $article->find('[class=themen]', 0); + if ($categories) { + $item['categories'] = []; + foreach ($categories->find('a') as $category) { + $item['categories'][] = $category->innertext; + } + } $article->clear(); return $item; From 628b30208ab8d268ab2fbaae2de7f19443baac27 Mon Sep 17 00:00:00 2001 From: Dag Date: Sat, 23 Nov 2024 22:28:50 +0100 Subject: [PATCH 238/423] fix: dont aquire exclusive locks (#4340) Due to bugs in logging/error-handling there sometimes are deadlocks --- caches/FileCache.php | 2 +- lib/logger.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/caches/FileCache.php b/caches/FileCache.php index dfd295e8..ff939bea 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -53,7 +53,7 @@ class FileCache implements CacheInterface 'value' => $value, ]; $cacheFile = $this->createCacheFile($key); - $bytes = file_put_contents($cacheFile, serialize($item), LOCK_EX); + $bytes = file_put_contents($cacheFile, serialize($item)); // todo: Consider tightening the permissions of the created file. It usually allow others to read, depending on umask if ($bytes === false) { // Consider just logging the error here diff --git a/lib/logger.php b/lib/logger.php index 74a0e713..9e5f6ce9 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -136,7 +136,7 @@ final class StreamHandler $record['message'], $context ); - $bytes = file_put_contents($this->stream, $text, FILE_APPEND | LOCK_EX); + $bytes = file_put_contents($this->stream, $text, FILE_APPEND); } } From 83bc3fd7620c161996f06480dbec7197ae2d95e6 Mon Sep 17 00:00:00 2001 From: User123698745 Date: Sun, 24 Nov 2024 03:57:28 +0100 Subject: [PATCH 239/423] [DRKBlutspendeBridge] add new bridge (#4324) * [DRKBlutspendeBridge] add new bridge * [DRKBlutspendeBridge] move explode_lines into DRKBlutspendeBridge class --- bridges/DRKBlutspendeBridge.php | 107 ++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 bridges/DRKBlutspendeBridge.php diff --git a/bridges/DRKBlutspendeBridge.php b/bridges/DRKBlutspendeBridge.php new file mode 100644 index 00000000..15075898 --- /dev/null +++ b/bridges/DRKBlutspendeBridge.php @@ -0,0 +1,107 @@ + [ + 'term' => [ + 'name' => 'PLZ / Ort', + 'required' => true, + 'exampleValue' => '12555', + ], + 'radius' => [ + 'name' => 'Umkreis in km', + 'type' => 'number', + 'exampleValue' => 10, + ], + 'limit_days' => [ + 'name' => 'Limit von Tagen', + 'title' => 'Nur Termine innerhalb der nächsten x Tagen', + 'type' => 'number', + 'exampleValue' => 28, + ], + 'limit_items' => [ + 'name' => 'Limit von Terminen', + 'title' => 'Nicht mehr als x Termine', + 'type' => 'number', + 'required' => true, + 'defaultValue' => 20, + ] + ] + ]; + + public function collectData() + { + $limitItems = intval($this->getInput('limit_items')); + $this->collectExpandableDatas(self::buildAppointmentsURI(), $limitItems); + } + + protected function parseItem(array $item) + { + $html = getSimpleHTMLDOM($item['uri']); + + $detailsElement = $html->find('.details', 0); + + $dateElement = $detailsElement->find('.datum', 0); + $dateLines = self::explodeLines($dateElement->plaintext); + + $addressElement = $detailsElement->find('.adresse', 0); + $addressLines = self::explodeLines($addressElement->plaintext); + + $infoElement = $detailsElement->find('.angebote > h4 + p', 0); + $info = $infoElement ? $infoElement->innertext : ''; + + $imageElements = $detailsElement->find('.fotos img'); + + $item['title'] = $dateLines[0] . ' ' . $dateLines[1] . ' ' . $addressLines[0] . ' - ' . $addressLines[1]; + + $item['content'] = <<{$dateLines[0]} {$dateLines[1]}

    +

    {$addressElement->innertext}

    +

    {$info}

    + HTML; + + foreach ($imageElements as $imageElement) { + $src = $imageElement->getAttribute('src'); + $item['content'] .= <<

    + HTML; + } + + $item['description'] = null; + + return $item; + } + + public function getURI() + { + if ($this->queriedContext === self::CONTEXT_APPOINTMENTS) { + return str_replace('.rss?', '?', self::buildAppointmentsURI()); + } + return parent::getURI(); + } + + private function buildAppointmentsURI() + { + $term = $this->getInput('term') ?? ''; + $radius = $this->getInput('radius') ?? ''; + $limitDays = intval($this->getInput('limit_days')); + $dateTo = $limitDays > 0 ? date('Y-m-d', time() + (60 * 60 * 24 * $limitDays)) : ''; + return self::BASE_URI . '/blutspendetermine/termine.rss?date_to=' . $dateTo . '&radius=' . $radius . '&term=' . $term; + } + + /** + * Returns an array of strings, each of which is a substring of string formed by splitting it on boundaries formed by line breaks. + */ + private function explodeLines(string $text): array + { + return array_map('trim', preg_split('/(\s*(\r\n|\n|\r)\s*)+/', $text)); + } +} From 74496e23aaa9101ac718a3b12dafa4206921932c Mon Sep 17 00:00:00 2001 From: Sebastian Wolf <117176763+swofl@users.noreply.github.com> Date: Sun, 24 Nov 2024 18:09:59 +0100 Subject: [PATCH 240/423] [MixologyBridge] add new bridge (#4331) * [MixologyBridge] add new bridge * [MixologyBridge] change invalid item property tags to categories * [MixologyBridge] rewrite into FeedExpander * [MixologyBridge] fix code formatting --- bridges/MixologyBridge.php | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 bridges/MixologyBridge.php diff --git a/bridges/MixologyBridge.php b/bridges/MixologyBridge.php new file mode 100644 index 00000000..1246b4db --- /dev/null +++ b/bridges/MixologyBridge.php @@ -0,0 +1,49 @@ + self::LIMIT, + ] ]; + + public function collectData() + { + $feed_url = self::URI . '/feed'; + $limit = $this->getInput('limit') ?? 10; + $this->collectExpandableDatas($feed_url, $limit); + } + + protected function parseItem(array $item) + { + $article = getSimpleHTMLDOMCached($item['uri']); + + $content = ''; + + $headerImage = $article->find('div.edgtf-full-width img.wp-post-image', 0); + + if (is_object($headerImage)) { + $item['enclosures'] = []; + $item['enclosures'][] = $headerImage->src; + $content .= ''; + } + + foreach ($article->find('article .wpb_content_element > .wpb_wrapper') as $element) { + $content .= $element->innertext; + } + + $item['content'] = $content; + + $item['categories'] = []; + + foreach ($article->find('.edgtf-tags > a') as $tag) { + $item['categories'][] = $tag->plaintext; + } + + return $item; + } +} From ec6f98e3c272cd06ae59fafadaaaca77904a522b Mon Sep 17 00:00:00 2001 From: SebLaus <97241865+SebLaus@users.noreply.github.com> Date: Sun, 24 Nov 2024 18:11:57 +0100 Subject: [PATCH 241/423] Added Alternate way to get Price if no buttons available (#4342) --- bridges/IdealoBridge.php | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php index 92bb30d0..55cee467 100644 --- a/bridges/IdealoBridge.php +++ b/bridges/IdealoBridge.php @@ -150,29 +150,34 @@ class IdealoBridge extends BridgeAbstract $ActualNewPrice = $html->find('div[id=oopStage-conditionButton-new]', 0); // Second Button contains the used product price $ActualUsedPrice = $html->find('div[id=oopStage-conditionButton-used]', 0); + // Get the first item of the offers list to have an option if there is no New/Used Button available + $altPrice = $html->find('.productOffers-listItemOfferPrice', 0); if ($ActualNewPrice) { $PriceNew = $ActualNewPrice->find('strong', 0)->plaintext; // Save current price $this->saveCacheValue($KeyNEW, $PriceNew); - } else if ($ActualNewPrice === null && $ActualUsedPrice !== null) { - // In case there is no actual New Price and a Ured Price exists, then delete the previous value in the cache - $this->cache->delete($this->getShortName() . '_' . $KeyNEW); + } else if ($altPrice) { + // Get price from first List item if no New/used Buttons available + $PriceNew = trim($altPrice->plaintext); + $this->saveCacheValue($KeyNEW, $PriceNew); + } else if (($ActualNewPrice === null || $altPrice === null) && $ActualUsedPrice !== null) { + // In case there is no actual New Price and a Used Price exists, then delete the previous value in the cache + $this->cache->delete($this->getShortName() . '_' . $KeyNEW); } - // Second Button contains the used product price if ($ActualUsedPrice) { $PriceUsed = $ActualUsedPrice->find('strong', 0)->plaintext; // Save current price $this->saveCacheValue($KeyUSED, $PriceUsed); - } else if ($ActualUsedPrice === null && $ActualNewPrice !== null) { + } else if ($ActualUsedPrice === null && ($ActualNewPrice !== null || $altPrice !== null)) { // In case there is no actual Used Price and a New Price exists, then delete the previous value in the cache - $this->cache->delete($this->getShortName() . '_' . $KeyUSED); + $this->cache->delete($this->getShortName() . '_' . $KeyUSED); } - // Only continue if a price has changed and there exists a New or Used price (sometimes no new Price _and_ Used Price are shown) - if (!($ActualNewPrice === null && $ActualUsedPrice === null ) && ($PriceNew != $OldPriceNew || $PriceUsed != $OldPriceUsed)) { + // Only continue if a price has changed and there exists a New, Used or Alternative price (sometimes no new Price _and_ Used Price are shown) + if (!($ActualNewPrice === null && $ActualUsedPrice === null && $altPrice === null) && ($PriceNew != $OldPriceNew || $PriceUsed != $OldPriceUsed)) { // Get Product Image $image = $html->find('.datasheet-cover-image', 0)->src; From b42a993176bc1e291f52b4de4ce3397fa307e84b Mon Sep 17 00:00:00 2001 From: thomas-333 <144368654+thomas-333@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:01:37 +0000 Subject: [PATCH 242/423] [Bluesky] New bridge (#4341) * Create BlueskyProfileBridge.php Bridge for Bluesky * Update BlueskyProfileBridge.php Attempt to fix test error * Rename BlueskyProfileBridge.php to BlueskyBridge.php and add list of select data source * Update BlueskyBridge.php to pass lint checks --- bridges/BlueskyBridge.php | 230 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 bridges/BlueskyBridge.php diff --git a/bridges/BlueskyBridge.php b/bridges/BlueskyBridge.php new file mode 100644 index 00000000..8dab82f4 --- /dev/null +++ b/bridges/BlueskyBridge.php @@ -0,0 +1,230 @@ + [ + 'name' => 'Bluesky Data Source', + 'type' => 'list', + 'defaultValue' => 'Profile', + 'values' => [ + 'Profile' => 'getAuthorFeed', + ], + 'title' => 'Select the type of data source to fetch from Bluesky.' + ], + 'handle' => [ + 'name' => 'User Handle', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'jackdodo.bsky.social', + 'title' => 'Handle found in URL' + ], + 'filter' => [ + 'name' => 'Filter', + 'type' => 'list', + 'defaultValue' => 'posts_and_author_threads', + 'values' => [ + 'posts_and_author_threads' => 'posts_and_author_threads', + 'posts_with_replies' => 'posts_with_replies', + 'posts_no_replies' => 'posts_no_replies', + 'posts_with_media' => 'posts_with_media', + ], + 'title' => 'Combinations of post/repost types to include in response.' + ] + ] + ]; + + private $profile; + + public function getName() + { + if (isset($this->profile)) { + return sprintf('%s (@%s) - Bluesky', $this->profile['displayName'], $this->profile['handle']); + } + return parent::getName(); + } + + public function getURI() + { + if (isset($this->profile)) { + return self::URI . '/profile/' . $this->profile['handle']; + } + return parent::getURI(); + } + + public function getIcon() + { + if (isset($this->profile)) { + return $this->profile['avatar']; + } + return parent::getIcon(); + } + + public function getDescription() + { + if (isset($this->profile)) { + return $this->profile['description']; + } + return parent::getDescription(); + } + + private function parseExternal($external, $did) + { + $description = ''; + $externalUri = $external['uri']; + $externalTitle = htmlspecialchars($external['title'], ENT_QUOTES, 'UTF-8'); + $externalDescription = htmlspecialchars($external['description'], ENT_QUOTES, 'UTF-8'); + $thumb = $external['thumb'] ?? null; + + if (preg_match('/youtube\.com\/watch\?v=([^\&\?\/]+)/', $externalUri, $id) || preg_match('/youtu\.be\/([^\&\?\/]+)/', $externalUri, $id)) { + $videoId = $id[1]; + $description .= "

    External Link: $externalTitle

    "; + $description .= ""; + } else { + $description .= "

    External Link: $externalTitle

    "; + $description .= "

    $externalDescription

    "; + + if ($thumb) { + $thumbUrl = 'https://cdn.bsky.app/img/feed_thumbnail/plain/' . $did . '/' . $thumb['ref']['$link'] . '@jpeg'; + $description .= "

    \"External

    "; + } + } + return $description; + } + + private function textToDescription($text) + { + $text = nl2br(htmlspecialchars($text, ENT_QUOTES, 'UTF-8')); + $text = preg_replace('/(https?:\/\/[^\s]+)/i', '$1', $text); + + return $text; + } + + public function collectData() + { + $handle = $this->getInput('handle'); + $filter = $this->getInput('filter') ?: 'posts_and_author_threads'; + + $did = $this->resolveHandle($handle); + $this->profile = $this->getProfile($did); + $authorFeed = $this->getAuthorFeed($did, $filter); + + foreach ($authorFeed['feed'] as $post) { + $item = []; + $item['uri'] = self::URI . '/profile/' . $post['post']['author']['handle'] . '/post/' . explode('app.bsky.feed.post/', $post['post']['uri'])[1]; + $item['title'] = strtok($post['post']['record']['text'], "\n"); + $item['timestamp'] = strtotime($post['post']['record']['createdAt']); + $item['author'] = $this->profile['displayName']; + + $description = $this->textToDescription($post['post']['record']['text']); + + // Retrieve DID for constructing image URLs + $authorDid = $post['post']['author']['did']; + + if (isset($post['post']['record']['embed']['$type']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.external') { + $description .= $this->parseExternal($post['post']['record']['embed']['external'], $authorDid); + } + + if (isset($post['post']['record']['embed']['$type']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.video') { + $thumbnail = $post['post']['embed']['thumbnail'] ?? null; + if ($thumbnail) { + $itemUri = self::URI . '/profile/' . $post['post']['author']['handle'] . '/post/' . explode('app.bsky.feed.post/', $post['post']['uri'])[1]; + $description .= "

    \"Video

    "; + } + } + + if (isset($post['post']['record']['embed']['$type']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.recordWithMedia#view') { + $thumbnail = $post['post']['embed']['media']['thumbnail'] ?? null; + $playlist = $post['post']['embed']['media']['playlist'] ?? null; + if ($thumbnail) { + $description .= "

    '; + } + } + + if (!empty($post['post']['record']['embed']['images'])) { + foreach ($post['post']['record']['embed']['images'] as $image) { + $linkRef = $image['image']['ref']['$link']; + $thumbnailUrl = $this->resolveThumbnailUrl($authorDid, $linkRef); + $fullsizeUrl = $this->resolveFullsizeUrl($authorDid, $linkRef); + $description .= "

    \"Image\""; + } + } + + // Enhanced handling for quote posts with images + if (isset($post['post']['record']['embed']) && $post['post']['record']['embed']['$type'] === 'app.bsky.embed.record') { + $quotedRecord = $post['post']['record']['embed']['record']; + $quotedAuthor = $post['post']['embed']['record']['author']['handle'] ?? null; + $quotedDisplayName = $post['post']['embed']['record']['author']['displayName'] ?? null; + $quotedText = $post['post']['embed']['record']['value']['text'] ?? null; + + if ($quotedAuthor && isset($quotedRecord['uri'])) { + $parts = explode('/', $quotedRecord['uri']); + $quotedPostId = end($parts); + $quotedPostUri = self::URI . '/profile/' . $quotedAuthor . '/post/' . $quotedPostId; + } + + if ($quotedText) { + $description .= '
    Quote from ' . htmlspecialchars($quotedDisplayName) . ' (@ ' . htmlspecialchars($quotedAuthor) . '):
    '; + $description .= $this->textToDescription($quotedText); + if (isset($quotedPostUri)) { + $description .= "

    View original quote post

    "; + } + } + } + + if (isset($post['post']['embed']['record']['value']['embed']['images'])) { + $quotedImages = $post['post']['embed']['record']['value']['embed']['images']; + foreach ($quotedImages as $image) { + $linkRef = $image['image']['ref']['$link'] ?? null; + if ($linkRef) { + $quotedAuthorDid = $post['post']['embed']['record']['author']['did'] ?? null; + $thumbnailUrl = $this->resolveThumbnailUrl($quotedAuthorDid, $linkRef); + $fullsizeUrl = $this->resolveFullsizeUrl($quotedAuthorDid, $linkRef); + $description .= "

    \"Quoted"; + } + } + } + + $item['content'] = $description; + $this->items[] = $item; + } + } + + private function resolveHandle($handle) + { + $uri = 'https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=' . urlencode($handle); + $response = json_decode(getContents($uri), true); + return $response['did']; + } + + private function getProfile($did) + { + $uri = 'https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=' . urlencode($did); + $response = json_decode(getContents($uri), true); + return $response; + } + + private function getAuthorFeed($did, $filter) + { + $uri = 'https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed?actor=' . urlencode($did) . '&filter=' . urlencode($filter) . '&limit=30'; + $response = json_decode(getContents($uri), true); + return $response; + } + + private function resolveThumbnailUrl($authorDid, $linkRef) + { + return 'https://cdn.bsky.app/img/feed_thumbnail/plain/' . $authorDid . '/' . $linkRef . '@jpeg'; + } + + private function resolveFullsizeUrl($authorDid, $linkRef) + { + return 'https://cdn.bsky.app/img/feed_fullsize/plain/' . $authorDid . '/' . $linkRef . '@jpeg'; + } +} From c651e11b0ff3a6ea0b9ff10a6669553b912db434 Mon Sep 17 00:00:00 2001 From: tillcash Date: Mon, 25 Nov 2024 23:33:35 +0530 Subject: [PATCH 243/423] [MaalaimalarBridge] fix new url (#4344) --- bridges/MaalaimalarBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/MaalaimalarBridge.php b/bridges/MaalaimalarBridge.php index e83eafe2..fcdbda48 100644 --- a/bridges/MaalaimalarBridge.php +++ b/bridges/MaalaimalarBridge.php @@ -5,6 +5,7 @@ class MaalaimalarBridge extends BridgeAbstract const NAME = 'Maalaimalar'; const URI = 'https://www.maalaimalar.com'; const DESCRIPTION = 'Retrieve news from maalaimalar.com'; + const CACHE_TIMEOUT = 60 * 5; // 5 minutes const MAINTAINER = 'tillcash'; const PARAMETERS = [ [ @@ -13,7 +14,7 @@ class MaalaimalarBridge extends BridgeAbstract 'type' => 'list', 'values' => [ 'news' => [ - 'tamilnadu' => '/news/state', + 'tamilnadu' => '/news/tamilnadu', 'puducherry' => '/news/puducherry', 'india' => '/news/national', 'world' => '/news/world', From c7f9870ba747abb9f92800e615fbe8ff517b8a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wr=C3=B3bel?= Date: Tue, 26 Nov 2024 03:04:02 +0100 Subject: [PATCH 244/423] [OLXBridge] fix title and shiping info retrieval (#4346) --- bridges/OLXBridge.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bridges/OLXBridge.php b/bridges/OLXBridge.php index 1a8be901..31f05eaa 100644 --- a/bridges/OLXBridge.php +++ b/bridges/OLXBridge.php @@ -103,20 +103,15 @@ EOF; continue; } - $shippingOffered = $post->find('.css-1c0ed4l svg', 0)->outertext ?? false; - if ($this->getInput('shippingOfferedOnly') && !$shippingOffered) { - continue; - } - $negotiable = $post->find('p[data-testid="ad-price"] span.css-e2218f', 0)->plaintext ?? false; if ($negotiable) { $price = trim(str_replace($negotiable, '', $price)); $negotiable = '(' . $negotiable . ')'; } - if ($post->find('h6', 0)->plaintext != '') { + if ($post->find('h4', 0)->plaintext != '') { $item['uri'] = $post->find('a', 0)->href; - $item['title'] = $post->find('h6', 0)->plaintext; + $item['title'] = $post->find('h4', 0)->plaintext; } # ignore the date component, as it is too convoluted — use the deep-crawled one; see below @@ -128,6 +123,12 @@ EOF; # Given that, do deep-crawl *all* the results, which allows to aso obtain the ID, the simplified location # and date strings, as well as the detailed description. $articleHTMLContent = getSimpleHTMLDOMCached($item['uri']); + $articleHTMLContent = defaultLinkTo($articleHTMLContent, $this->getHostname()); + + $shippingOffered = $articleHTMLContent->find('img[alt="Safety Badge"]', 0)->src ?? false; + if ($this->getInput('shippingOfferedOnly') && !$shippingOffered) { + continue; + } # Extract a clean ID without resorting to the convoluted CSS class or sibling selectors. Should be always present. $refreshLink = $articleHTMLContent->find('a[data-testid=refresh-link]', 0)->href ?? false; @@ -195,7 +196,7 @@ EOF;

    $location

    -

    $price $negotiable $shippingOffered

    +

    $price $negotiable