From b3bf95bfddf557c204362553bf3b324e43575039 Mon Sep 17 00:00:00 2001 From: csisoap <33269526+csisoap@users.noreply.github.com> Date: Wed, 12 Jul 2023 04:17:38 +0700 Subject: [PATCH 001/716] Replace token for Twitter. (#3522) --- lib/TwitterClient.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 605c27ff..18e8d02a 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -11,7 +11,7 @@ class TwitterClient public function __construct(CacheInterface $cache) { $this->cache = $cache; - $this->authorization = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'; + $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; $this->data = $cache->loadData() ?? []; } From 69aa751f40960bd49060b7bb4696116b182bdc85 Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 11 Jul 2023 23:26:22 +0200 Subject: [PATCH 002/716] fix(cache): bug in prior refactor (#3525) * fix(cache): bug in prior refactor * yup --- lib/TwitterClient.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 18e8d02a..cdedcbc1 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -12,7 +12,7 @@ class TwitterClient { $this->cache = $cache; $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; - $this->data = $cache->loadData() ?? []; + $this->data = $this->cache->loadData() ?? []; } public function fetchUserTweets(string $screenName): \stdClass @@ -95,6 +95,9 @@ class TwitterClient $response = getContents($url, $this->createHttpHeaders(), [CURLOPT_POST => true]); $guest_token = json_decode($response)->guest_token; $this->data['guest_token'] = $guest_token; + + $this->cache->setScope('twitter'); + $this->cache->setKey(['cache']); $this->cache->saveData($this->data); Logger::info("Fetch new guest token: $guest_token"); } @@ -119,6 +122,9 @@ class TwitterClient } $userInfo = $response->data->user; $this->data[$screenName] = $userInfo; + + $this->cache->setScope('twitter'); + $this->cache->setKey(['cache']); $this->cache->saveData($this->data); return $userInfo; } From 0f2b55fbef10fd3cda7498441c81e27b0518e4f8 Mon Sep 17 00:00:00 2001 From: Fake4d Date: Wed, 12 Jul 2023 11:56:07 +0200 Subject: [PATCH 003/716] Update Configuration.php - Old Version Date in new Release (#3526) Old Version Date in new Release --- lib/Configuration.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Configuration.php b/lib/Configuration.php index b6b2e4d6..57a7db7e 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -19,7 +19,7 @@ */ final class Configuration { - private const VERSION = 'dev.2023-03-22'; + private const VERSION = 'dev.2023-07-11'; private static $config = []; From b9102d7e87726ca42659ff70e62ed6320b2657fb Mon Sep 17 00:00:00 2001 From: Jisagi Date: Thu, 13 Jul 2023 17:23:12 +0200 Subject: [PATCH 004/716] Add Steam Group Announcements Bridge (#3527) * Create SteamGroupAnnouncementsBridge.php * Shorten implementation Maybe this fixes the tests * test --------- Co-authored-by: Dag --- bridges/SteamGroupAnnouncementsBridge.php | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 bridges/SteamGroupAnnouncementsBridge.php diff --git a/bridges/SteamGroupAnnouncementsBridge.php b/bridges/SteamGroupAnnouncementsBridge.php new file mode 100644 index 00000000..2b848850 --- /dev/null +++ b/bridges/SteamGroupAnnouncementsBridge.php @@ -0,0 +1,25 @@ + [ + 'name' => 'Group name', + 'exampleValue' => 'freegamesfinders', + 'required' => true + ] + ] + ]; + + public function collectData() + { + $uri = self::URI . 'groups/' . $this->getInput('g') . '/rss'; + $this->collectExpandableDatas($uri, 10); + } +} From a234392f801451f104edbe940c36fe2749757be6 Mon Sep 17 00:00:00 2001 From: alexvong243f Date: Fri, 14 Jul 2023 03:15:49 +0000 Subject: [PATCH 005/716] docs: improve discoverability of instagram related bridges (#3528) --- bridges/PicnobBridge.php | 2 +- bridges/PicukiBridge.php | 2 +- docs/10_Bridge_Specific/Instagram.md | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bridges/PicnobBridge.php b/bridges/PicnobBridge.php index 68241a8c..1d7d06b4 100644 --- a/bridges/PicnobBridge.php +++ b/bridges/PicnobBridge.php @@ -6,7 +6,7 @@ class PicnobBridge extends BridgeAbstract const NAME = 'Picnob Bridge'; const URI = 'https://www.picnob.com/'; const CACHE_TIMEOUT = 3600; // 1h - const DESCRIPTION = 'Returns Picnob posts by user or by hashtag'; + const DESCRIPTION = 'Returns Picnob (Instagram viewer) posts by user or by hashtag'; const PARAMETERS = [ 'Username' => [ diff --git a/bridges/PicukiBridge.php b/bridges/PicukiBridge.php index e90177ed..7a4f9eb5 100644 --- a/bridges/PicukiBridge.php +++ b/bridges/PicukiBridge.php @@ -6,7 +6,7 @@ class PicukiBridge extends BridgeAbstract const NAME = 'Picuki Bridge'; const URI = 'https://www.picuki.com/'; const CACHE_TIMEOUT = 3600; // 1h - const DESCRIPTION = 'Returns Picuki posts by user and by hashtag'; + const DESCRIPTION = 'Returns Picuki (Instagram viewer) posts by user and by hashtag'; const PARAMETERS = [ 'Username' => [ diff --git a/docs/10_Bridge_Specific/Instagram.md b/docs/10_Bridge_Specific/Instagram.md index 7b9bf6b2..85e7b8a8 100644 --- a/docs/10_Bridge_Specific/Instagram.md +++ b/docs/10_Bridge_Specific/Instagram.md @@ -4,6 +4,8 @@ InstagramBridge To somehow bypass the [rate limiting issue](https://github.com/RSS-Bridge/rss-bridge/issues/1891) it is suggested to deploy a private RSS-Bridge instance that uses a working Instagram account. +**NOTE**: There exists alternative bridges (e.g. PicukiBridge and PicnobBridge) for viewing posts without a working account. + Configuration ------------- From 9efdf24a6ed4c96a03d4aeb8684d5b082627057c Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 14 Jul 2023 22:09:45 +0200 Subject: [PATCH 006/716] Add CustomBridge (#3457) * Add CustomBridge For advanced users. Create RSS feed using HTML selectors. * [CssSelectorBridge] Refactor, Allow Unexpanded Rename bridge to CssSelectorBridge Allow unexpanded feed, i.e. make feed from home page only (1 request) Refactor bridge to put most of the code into protected functions Makes the code more maintainable and allows inheritance for variants * [CssSelectorBridge] Fix linting --- bridges/CssSelectorBridge.php | 208 ++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 bridges/CssSelectorBridge.php diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php new file mode 100644 index 00000000..ae135113 --- /dev/null +++ b/bridges/CssSelectorBridge.php @@ -0,0 +1,208 @@ + [ + 'name' => 'Site URL: Home page with latest articles', + 'exampleValue' => 'https://example.com/blog/', + 'required' => true + ], + 'url_selector' => [ + 'name' => 'Selector for article links or their parent elements', + 'exampleValue' => 'a.article', + 'required' => true + ], + 'url_pattern' => [ + 'name' => '[Optional] Pattern for site URLs to keep in feed', + 'exampleValue' => 'https://example.com/article/.*', + ], + 'content_selector' => [ + 'name' => '[Optional] Selector to extract each article content', + 'exampleValue' => 'article.content', + ], + 'content_cleanup' => [ + 'name' => '[Optional] Content cleanup: List of items to remove', + 'exampleValue' => 'div.ads, div.comments', + ], + 'title_cleanup' => [ + 'name' => '[Optional] Text to remove from expanded article title', + 'exampleValue' => ' | BlogName', + ], + 'limit' => self::LIMIT + ] + ]; + + private $feedName = ''; + + public function getURI() + { + $url = $this->getInput('home_page'); + if (empty($url)) { + $url = parent::getURI(); + } + return $url; + } + + public function getName() + { + if (!empty($this->feedName)) { + return $this->feedName; + } + return parent::getName(); + } + + public function collectData() + { + $url = $this->getInput('home_page'); + $url_selector = $this->getInput('url_selector'); + $url_pattern = $this->getInput('url_pattern'); + $content_selector = $this->getInput('content_selector'); + $content_cleanup = $this->getInput('content_cleanup'); + $title_cleanup = $this->getInput('title_cleanup'); + $limit = $this->getInput('limit') ?? 10; + + $html = defaultLinkTo(getSimpleHTMLDOM($url), $url); + $this->feedName = $this->getPageTitle($html, $title_cleanup); + $items = $this->htmlFindLinks($html, $url_selector, $url_pattern, $limit); + + if (empty($content_selector)) { + $this->items = $items; + } else { + foreach ($items as $item) { + $this->items[] = $this->expandEntryWithSelector( + $item['uri'], + $content_selector, + $content_cleanup, + $title_cleanup + ); + } + } + } + + /** + * Filter a list of URLs using a pattern and limit + * @param array $links List of URLs + * @param string $url_pattern Pattern to look for in URLs + * @param int $limit Optional maximum amount of URLs to return + * @return array Array of URLs + */ + protected function filterUrlList($links, $url_pattern, $limit = 0) + { + if (!empty($url_pattern)) { + $url_pattern = '/' . str_replace('/', '\/', $url_pattern) . '/'; + $links = array_filter($links, function ($url) { + return preg_match($url_pattern, $url) === 1; + }); + } + + if ($limit > 0 && count($links) > $limit) { + $links = array_slice($links, 0, $limit); + } + + return $links; + } + + /** + * Retrieve title from webpage URL or DOM + * @param string|object $page URL or DOM to retrieve title from + * @param string $title_cleanup optional string to remove from webpage title, e.g. " | BlogName" + * @return string Webpage title + */ + protected function getPageTitle($page, $title_cleanup = null) + { + if (is_string($page)) { + $page = getSimpleHTMLDOMCached($page); + } + $title = html_entity_decode($page->find('title', 0)->plaintext); + if (!empty($title)) { + $title = trim(str_replace($title_cleanup, '', $title)); + } + return $title; + } + + /** + * Retrieve first N links from webpage URL or DOM satisfying the specified criteria + * @param string|object $page URL or DOM to retrieve links from + * @param string $url_selector DOM selector for matching links or their parent element + * @param string $url_pattern Optional filter to keep only links matching the pattern + * @param int $limit Optional maximum amount of URLs to return + * @return array of minimal feed items {'uri': entry_url, 'title', entry_title} + */ + protected function htmlFindLinks($page, $url_selector, $url_pattern = '', $limit = 0) + { + $links = $page->find($url_selector); + + if (empty($links)) { + returnClientError('No results for URL selector'); + } + + $link_to_title = []; + foreach ($links as $link) { + if ($link->tag != 'a') { + $link = $link->find('a', 0); + } + $link_to_title[$link->href] = $link->plaintext; + } + + $links = $this->filterUrlList(array_keys($link_to_title), $url_pattern, $limit); + + if (empty($links)) { + returnClientError('No results for URL pattern'); + } + + $items = []; + foreach ($links as $link) { + $item = []; + $item['uri'] = $link; + $item['title'] = $link_to_title[$link]; + $items[] = $item; + } + + return $items; + } + + /** + * Retrieve article content from its URL using content selector and return a feed item + * @param string $entry_url URL to retrieve article from + * @param string $content_selector HTML selector for extracting content, e.g. "article.content" + * @param string $content_cleanup Optional selector for removing elements, e.g. "div.ads, div.comments" + * @param string $title_cleanup Optional string to remove from article title, e.g. " | BlogName" + * @return array Entry data: uri, title, content + */ + protected function expandEntryWithSelector($entry_url, $content_selector, $content_cleanup = null, $title_cleanup = null) + { + if (empty($content_selector)) { + returnClientError('Please specify a content selector'); + } + + $entry_html = getSimpleHTMLDOMCached($entry_url); + $article_content = $entry_html->find($content_selector); + + if (!empty($article_content)) { + $article_content = $article_content[0]; + } else { + returnClientError('Could not find content selector at URL: ' . $entry_url); + } + + if (!empty($content_cleanup)) { + foreach ($article_content->find($content_cleanup) as $item_to_clean) { + $item_to_clean->outertext = ''; + } + } + + $article_content = convertLazyLoading($article_content); + $article_content = defaultLinkTo($article_content, $entry_url); + + $item = []; + $item['uri'] = $entry_url; + $item['title'] = $this->getPageTitle($entry_html, $title_cleanup); + $item['content'] = $article_content; + return $item; + } +} From ea0456ea08b86c3d9bd6b79ddcb48d8faf47b805 Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Fri, 14 Jul 2023 23:58:57 -0400 Subject: [PATCH 007/716] [New Bridge] Qwantz bridge (#3531) * [QwantzBridge] new bridge * [QwantzBridge] title fix * lint fixes --- bridges/QwantzBridge.php | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 bridges/QwantzBridge.php diff --git a/bridges/QwantzBridge.php b/bridges/QwantzBridge.php new file mode 100644 index 00000000..e48e948a --- /dev/null +++ b/bridges/QwantzBridge.php @@ -0,0 +1,37 @@ +find('img')[0]->{'src'}; + $subject = $content->find('a')[1]->{'href'}; + $subject = urldecode(substr($subject, strpos($subject, 'subject') + 8)); + $p = (string)$content->find('P')[0]; + + $item['content'] = "{$subject}

{$title}

{$p}"; + + return $item; + } + + public function collectData() + { + $this->collectExpandableDatas(self::URI . 'rssfeed.php'); + } + + public function getIcon() + { + return self::URI . 'favicon.ico'; + } +} From 73d88dda46b0f6748bfaa795b177aae2a4cbf900 Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Sat, 15 Jul 2023 00:11:00 -0400 Subject: [PATCH 008/716] [New Bridge] Strava bridge (#3533) * [StravaBridge] new bridge * [StravaBridge] gpx link, detect parameters --- bridges/StravaBridge.php | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 bridges/StravaBridge.php diff --git a/bridges/StravaBridge.php b/bridges/StravaBridge.php new file mode 100644 index 00000000..da1739cb --- /dev/null +++ b/bridges/StravaBridge.php @@ -0,0 +1,81 @@ + [ + 'name' => 'athleteID', + 'required' => true + ] + ], + ]; + + public function detectParameters($url) + { + if (preg_match('/strava\.com\/athletes\/([\d]+)/', $url, $matches) > 0) { + return [ + 'athleteID' => $matches[1] + ]; + } + return null; + } + + public function collectData() + { + $athleteID = $this->getInput('athleteID'); + + $dom = getSimpleHTMLDOM(self::URI . '/athletes/' . $athleteID); + $scriptRegex = "/data-react-props='(.*?)'/"; + preg_match($scriptRegex, $dom, $matches) or returnServerError('Could not find json'); + $jsonData = json_decode(html_entity_decode($matches[1])); + $this->feedName = $jsonData->athlete->name . "'s Recent Activities"; + $this->iconURL = $jsonData->athlete->avatarUrl; + foreach ($jsonData->recentActivities as $activity) { + $item = []; + + $item['title'] = $activity->name . ' (' . $activity->detailedType . ')'; + $item['author'] = $jsonData->athlete->name; + $item['uri'] = self::URI . '/activities/' . $activity->id; + $item['timestamp'] = $activity->startDateLocal; + + $content = 'Distance: ' . $activity->distance . + '
Elev Gain: ' . $activity->elevation . + '
Time: ' . $activity->movingTime . '

'; + + foreach ($activity->images as $image) { + $src = $image->squareSrc; + if (empty($src)) { + $src = $image->defaultSrc; + } + $content .= ''; + } + $item['content'] = $content; + + $item['enclosures'][] = $item['uri'] . '/export_gpx'; + + $this->items[] = $item; + } + } + + public function getName() + { + if (empty($this->feedName)) { + return parent::getName(); + } else { + return $this->feedName; + } + } + + public function getIcon() + { + if (empty($this->iconURL)) { + return parent::getIcon(); + } else { + return $this->iconURL; + } + } +} From c8039d483bb9f0690a4f9aef6700483c055aef22 Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Sat, 15 Jul 2023 09:12:11 -0400 Subject: [PATCH 009/716] [TraktBridge] new bridge (#3534) --- bridges/TraktBridge.php | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 bridges/TraktBridge.php diff --git a/bridges/TraktBridge.php b/bridges/TraktBridge.php new file mode 100644 index 00000000..7aa90dfd --- /dev/null +++ b/bridges/TraktBridge.php @@ -0,0 +1,70 @@ + [ + 'name' => 'username', + 'required' => true + ], + 'hide_shows' => [ + 'name' => 'Hide shows', + 'type' => 'checkbox', + 'title' => 'Hide shows', + ], + + ], + ]; + + public function detectParameters($url) + { + if (preg_match('/trakt\.tv\/users\/(.*?)\//', $url, $matches) > 0) { + return [ + 'username' => $matches[1] + ]; + } + return null; + } + + public function collectData() + { + $username = $this->getInput('username'); + $dom = getSimpleHTMLDOMCached(self::URI . '/users/' . $username . '/history'); + $this->feedName = $dom->find('#avatar-wrapper h1 a', 0)->plaintext; + $this->iconURL = $dom->find('img.avatar', 0)->{'src'}; + + foreach ($dom->find('#history-items .posters', 0)->find('div.grid-item') as $div) { + if ($this->getInput('hide_shows') && $div->{'data-type'} != 'movie') { + continue; + } + $item = []; + $item['author'] = $this->feedName; + $item['title'] = $div->find('img.real', 0)->{'title'}; + $item['timestamp'] = $div->find('.format-date', 0)->plaintext; + $item['content'] = ''; + $item['uri'] = self::URI . $div->{'data-url'}; + $this->items[] = $item; + } + } + public function getName() + { + if (empty($this->feedName)) { + return parent::getName(); + } else { + return $this->feedName; + } + } + public function getIcon() + { + if (empty($this->iconURL)) { + return parent::getIcon(); + } else { + return $this->iconURL; + } + } +} From e5729fdaacb7fc18c3590c406cc3c50eaa0222c8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sat, 15 Jul 2023 15:18:09 +0200 Subject: [PATCH 010/716] [YouTubeCommunityTabBridge] Fix PHP warnings for posts w/o text (#3536) Because "contentText" is always present, PHP warnings were previously generated for posts without text. Existence of sub-property "runs" gets checked now to avoid this. --- bridges/YouTubeCommunityTabBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/YouTubeCommunityTabBridge.php b/bridges/YouTubeCommunityTabBridge.php index c9ea5863..32502f61 100644 --- a/bridges/YouTubeCommunityTabBridge.php +++ b/bridges/YouTubeCommunityTabBridge.php @@ -92,7 +92,7 @@ class YouTubeCommunityTabBridge extends BridgeAbstract $item['author'] = $details->authorText->runs[0]->text; $item['content'] = ''; - if (isset($details->contentText)) { + if (isset($details->contentText->runs)) { $text = $this->getText($details->contentText->runs); $this->itemTitle = $this->ellipsisTitle($text); From eaea8e664078db55dd32dd227bad34acb6c46229 Mon Sep 17 00:00:00 2001 From: Dag Date: Sat, 15 Jul 2023 21:16:23 +0200 Subject: [PATCH 011/716] fix: Undefined index: REMOTE_ADDR in non-web sapi' (#3538) --- lib/Debug.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Debug.php b/lib/Debug.php index f6a8d105..48dbb31a 100644 --- a/lib/Debug.php +++ b/lib/Debug.php @@ -7,7 +7,7 @@ class Debug */ public static function isEnabled(): bool { - $ip = $_SERVER['REMOTE_ADDR']; + $ip = $_SERVER['REMOTE_ADDR'] ?? 'x.y.z.1'; $enableDebugMode = Configuration::getConfig('system', 'enable_debug_mode'); $debugModeWhitelist = Configuration::getConfig('system', 'debug_mode_whitelist') ?: []; if ($enableDebugMode && ($debugModeWhitelist === [] || in_array($ip, $debugModeWhitelist))) { From ef8181478d62b7a558dcf9821076882b13443b0f Mon Sep 17 00:00:00 2001 From: Dag Date: Sat, 15 Jul 2023 22:12:16 +0200 Subject: [PATCH 012/716] perf(SqliteCache): add index to updated (#3515) * refactor(SqliteCache) * perf(SqliteCache): add index to updated --- caches/SQLiteCache.php | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index 309b86d1..cb9f33b1 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -1,8 +1,5 @@ - */ class SQLiteCache implements CacheInterface { private \SQLite3 $db; @@ -32,6 +29,7 @@ class SQLiteCache implements CacheInterface $this->db = new \SQLite3($config['file']); $this->db->enableExceptions(true); $this->db->exec("CREATE TABLE storage ('key' BLOB PRIMARY KEY, 'value' BLOB, 'updated' INTEGER)"); + $this->db->exec('CREATE INDEX idx_storage_updated ON storage (updated)'); } $this->db->busyTimeout($config['timeout']); } @@ -42,20 +40,22 @@ class SQLiteCache implements CacheInterface $stmt->bindValue(':key', $this->getCacheKey()); $result = $stmt->execute(); if ($result) { - $data = $result->fetchArray(\SQLITE3_ASSOC); - if (isset($data['value'])) { - return unserialize($data['value']); + $row = $result->fetchArray(\SQLITE3_ASSOC); + $data = unserialize($row['value']); + if ($data !== false) { + return $data; } } - return null; } public function saveData($data): void { + $blob = serialize($data); + $stmt = $this->db->prepare('INSERT OR REPLACE INTO storage (key, value, updated) VALUES (:key, :value, :updated)'); $stmt->bindValue(':key', $this->getCacheKey()); - $stmt->bindValue(':value', serialize($data)); + $stmt->bindValue(':value', $blob); $stmt->bindValue(':updated', time()); $stmt->execute(); } @@ -66,12 +66,9 @@ class SQLiteCache implements CacheInterface $stmt->bindValue(':key', $this->getCacheKey()); $result = $stmt->execute(); if ($result) { - $data = $result->fetchArray(\SQLITE3_ASSOC); - if (isset($data['updated'])) { - return $data['updated']; - } + $row = $result->fetchArray(\SQLITE3_ASSOC); + return $row['updated']; } - return null; } From e8420b9f39fa5db6272d341d31b090a1be22e211 Mon Sep 17 00:00:00 2001 From: Dag Date: Sat, 15 Jul 2023 22:44:26 +0200 Subject: [PATCH 013/716] fix(sqlitecache): log failed unserialization (#3539) --- caches/SQLiteCache.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index cb9f33b1..bc110928 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -41,10 +41,12 @@ class SQLiteCache implements CacheInterface $result = $stmt->execute(); if ($result) { $row = $result->fetchArray(\SQLITE3_ASSOC); - $data = unserialize($row['value']); + $blob = $row['value']; + $data = unserialize($blob); if ($data !== false) { return $data; } + Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100))); } return null; } From 773eea196f028f7576345884df3269054073394a Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 16 Jul 2023 05:29:56 +0200 Subject: [PATCH 014/716] fix(sqlitecache): bug in prior refactor (#3540) fixes: rssbridge.WARNING Trying to access array offset on value of type bool at caches/SQLiteCache.php line 72 --- caches/SQLiteCache.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index bc110928..f9258a88 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -41,12 +41,14 @@ class SQLiteCache implements CacheInterface $result = $stmt->execute(); if ($result) { $row = $result->fetchArray(\SQLITE3_ASSOC); - $blob = $row['value']; - $data = unserialize($blob); - if ($data !== false) { - return $data; + if ($row !== false) { + $blob = $row['value']; + $data = unserialize($blob); + if ($data !== false) { + return $data; + } + Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100))); } - Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100))); } return null; } @@ -69,7 +71,9 @@ class SQLiteCache implements CacheInterface $result = $stmt->execute(); if ($result) { $row = $result->fetchArray(\SQLITE3_ASSOC); - return $row['updated']; + if ($row !== false) { + return $row['updated']; + } } return null; } From 310160fd9286799f5b41438b1c4abe3128567671 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 16 Jul 2023 07:18:38 +0200 Subject: [PATCH 015/716] feat: improve http 429 handling (#3541) --- actions/DisplayAction.php | 12 ++++++------ lib/RssBridge.php | 2 +- lib/contents.php | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 66acd4cc..91cfb95f 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -142,13 +142,14 @@ class DisplayAction implements ActionInterface 'donationUri' => $bridge->getDonationURI(), 'icon' => $bridge->getIcon() ]; - } catch (\Throwable $e) { + } catch (\Exception $e) { if ($e instanceof HttpException) { - // Produce a smaller log record for http exceptions - Logger::warning(sprintf('Exception in %s: %s', $bridgeClassName, create_sane_exception_message($e))); + Logger::warning(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e))); + if ($e->getCode() === 429) { + return new Response('503 Service Unavailable', 503); + } } else { - // Log the exception - Logger::error(sprintf('Exception in %s', $bridgeClassName), ['e' => $e]); + Logger::error(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e)), ['e' => $e]); } // Emit error only if we are passed the error report limit @@ -158,7 +159,6 @@ class DisplayAction implements ActionInterface // Emit the error as a feed item in a feed so that feed readers can pick it up $items[] = $this->createFeedItemFromException($e, $bridge); } elseif (Configuration::getConfig('error', 'output') === 'http') { - // Emit as a regular web response throw $e; } } diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 79d1d710..e8b07a65 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -16,7 +16,7 @@ final class RssBridge try { $this->run($request); } catch (\Throwable $e) { - Logger::error('Exception in main', ['e' => $e]); + Logger::error(sprintf('Exception in RssBridge::main(): %s', create_sane_exception_message($e)), ['e' => $e]); http_response_code(500); print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); } diff --git a/lib/contents.php b/lib/contents.php index 56dd4be6..c6edba7b 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -197,7 +197,7 @@ function getContents( } } - throw new HttpException($exceptionMessage, $result['code']); + throw new HttpException(trim($exceptionMessage), $result['code']); } if ($returnFull === true) { return $response; From 7b46b97abd4d1d53dea51175ddfe97113058a4e8 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 16 Jul 2023 21:50:44 +0200 Subject: [PATCH 016/716] refactor(spotify): replace manual curl with getContents (#3544) --- bridges/SpotifyBridge.php | 57 +++++++++++++-------------------------- 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/bridges/SpotifyBridge.php b/bridges/SpotifyBridge.php index 170f4c86..b58d14be 100644 --- a/bridges/SpotifyBridge.php +++ b/bridges/SpotifyBridge.php @@ -53,6 +53,7 @@ class SpotifyBridge extends BridgeAbstract public function collectData() { + $this->fetchAccessToken(); $entries = $this->getAllEntries(); usort($entries, function ($entry1, $entry2) { return $this->getDate($entry2) <=> $this->getDate($entry1); @@ -92,7 +93,7 @@ class SpotifyBridge extends BridgeAbstract 'show' => 'episode', ]; if (!isset($types[$type])) { - throw new \Exception('Spotify URI not supported'); + throw new \Exception(sprintf('Unsupported Spotify URI: %s', $uri)); } $entry_type = $types[$type]; @@ -111,7 +112,8 @@ class SpotifyBridge extends BridgeAbstract $offset = 0; while (true) { $query['offset'] = $offset; - $partial = $this->fetchContent($url . '?' . http_build_query($query)); + $json = getContents($url . '?' . http_build_query($query), ['Authorization: Bearer ' . $this->token]); + $partial = Json::decode($json); if (empty($partial['items'])) { break; } @@ -188,12 +190,11 @@ class SpotifyBridge extends BridgeAbstract return DateTime::createFromFormat('Y-m-d', $date)->getTimestamp(); } - private function getToken() + private function fetchAccessToken() { $cache = RssBridge::getCache(); - $cache->setScope('SpotifyBridge'); - $cacheKey = sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); + $cache->setScope('SpotifyBridge'); $cache->setKey([$cacheKey]); $time = null; @@ -202,45 +203,22 @@ class SpotifyBridge extends BridgeAbstract } if (!$cache->getTime() || $time >= 3600) { - $this->fetchToken(); + // fetch token + $basicAuth = base64_encode(sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret'))); + $json = getContents('https://accounts.spotify.com/api/token', [ + "Authorization: Basic $basicAuth" + ], [ + CURLOPT_POSTFIELDS => 'grant_type=client_credentials' + ]); + $data = Json::decode($json); + $this->token = $data['access_token']; + $cache->saveData($this->token); } else { $this->token = $cache->loadData(); } } - private function fetchToken() - { - $curl = curl_init(); - - curl_setopt($curl, CURLOPT_URL, 'https://accounts.spotify.com/api/token'); - curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($curl, CURLOPT_POST, 1); - curl_setopt($curl, CURLOPT_POSTFIELDS, 'grant_type=client_credentials'); - - $basic = sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); - curl_setopt($curl, CURLOPT_HTTPHEADER, ['Authorization: Basic ' . base64_encode($basic)]); - - $json = curl_exec($curl); - $json = json_decode($json)->access_token; - curl_close($curl); - - $this->token = $json; - } - - private function fetchContent($url) - { - $this->getToken(); - $curl = curl_init(); - curl_setopt($curl, CURLOPT_URL, $url); - curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); - curl_setopt($curl, CURLOPT_HTTPHEADER, ['Authorization: Bearer ' . $this->token]); - $json = curl_exec($curl); - $json = json_decode($json, true); - curl_close($curl); - return $json; - } - public function getURI() { if (empty($this->uri)) { @@ -273,7 +251,8 @@ class SpotifyBridge extends BridgeAbstract $query['market'] = $this->getInput('country'); } - $item = $this->fetchContent($uri . '?' . http_build_query($query)); + $json = getContents($uri . '?' . http_build_query($query), ['Authorization: Bearer ' . $this->token]); + $item = Json::decode($json); $this->uri = $item['external_urls']['spotify']; $this->name = $item['name'] . ' - Spotify'; From a59793e8d647c77a2537e41a4e8fa3fcca728cff Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 16 Jul 2023 22:07:34 +0200 Subject: [PATCH 017/716] refactor: extract CurlHttpClient (#3532) * refactor: extract CurlHttpClient * refactor * interface --- bridges/AO3Bridge.php | 7 +- .../prepare_release/fetch_contributors.php | 3 +- lib/RssBridge.php | 16 +- lib/contents.php | 257 +++++++++--------- 4 files changed, 145 insertions(+), 138 deletions(-) diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php index 6ca59cc5..57e12fbd 100644 --- a/bridges/AO3Bridge.php +++ b/bridges/AO3Bridge.php @@ -92,7 +92,12 @@ class AO3Bridge extends BridgeAbstract private function collectWork($id) { $url = self::URI . "/works/$id/navigate"; - $response = _http_request($url, ['useragent' => 'rss-bridge bot (https://github.com/RSS-Bridge/rss-bridge)']); + $httpClient = RssBridge::getHttpClient(); + + $response = $httpClient->request($url, [ + 'useragent' => 'rss-bridge bot (https://github.com/RSS-Bridge/rss-bridge)', + ]); + $html = \str_get_html($response['body']); $html = defaultLinkTo($html, self::URI); diff --git a/contrib/prepare_release/fetch_contributors.php b/contrib/prepare_release/fetch_contributors.php index ad04458a..cfe2c5b2 100644 --- a/contrib/prepare_release/fetch_contributors.php +++ b/contrib/prepare_release/fetch_contributors.php @@ -14,7 +14,8 @@ while ($next) { /* Collect all contributors */ 'Content-Type' => 'application/json', 'User-Agent' => 'RSS-Bridge', ]; - $result = _http_request($url, ['headers' => $headers]); + $httpClient = new CurlHttpClient(); + $result = $httpClient->request($url, ['headers' => $headers]); foreach (json_decode($result['body']) as $contributor) { $contributors[] = $contributor; diff --git a/lib/RssBridge.php b/lib/RssBridge.php index e8b07a65..8969dc54 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -2,6 +2,7 @@ final class RssBridge { + private static HttpClient $httpClient; private static CacheInterface $cache; public function main(array $argv = []) @@ -71,9 +72,10 @@ final class RssBridge // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); date_default_timezone_set(Configuration::getConfig('system', 'timezone')); - // Create cache $cacheFactory = new CacheFactory(); - self::setCache($cacheFactory->create()); + + self::$httpClient = new CurlHttpClient(); + self::$cache = $cacheFactory->create(); if (Configuration::getConfig('authentication', 'enable')) { $authenticationMiddleware = new AuthenticationMiddleware(); @@ -105,13 +107,13 @@ final class RssBridge } } + public static function getHttpClient(): HttpClient + { + return self::$httpClient; + } + public static function getCache(): CacheInterface { return self::$cache; } - - public static function setCache(CacheInterface $cache): void - { - self::$cache = $cache; - } } diff --git a/lib/contents.php b/lib/contents.php index c6edba7b..419432df 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -99,6 +99,7 @@ function getContents( array $curlOptions = [], bool $returnFull = false ) { + $httpClient = RssBridge::getHttpClient(); $cache = RssBridge::getCache(); $cache->setScope('server'); $cache->setKey([$url]); @@ -141,20 +142,14 @@ function getContents( $config['if_not_modified_since'] = $cache->getTime(); } - $result = _http_request($url, $config); - $response = [ - 'code' => $result['code'], - 'status_lines' => $result['status_lines'], - 'header' => $result['headers'], - 'content' => $result['body'], - ]; + $response = $httpClient->request($url, $config); - switch ($result['code']) { + switch ($response['code']) { case 200: case 201: case 202: - if (isset($result['headers']['cache-control'])) { - $cachecontrol = $result['headers']['cache-control']; + if (isset($response['headers']['cache-control'])) { + $cachecontrol = $response['headers']['cache-control']; $lastValue = array_pop($cachecontrol); $directives = explode(',', $lastValue); $directives = array_map('trim', $directives); @@ -163,7 +158,7 @@ function getContents( break; } } - $cache->saveData($result['body']); + $cache->saveData($response['body']); break; case 301: case 302: @@ -172,16 +167,16 @@ function getContents( break; case 304: // Not Modified - $response['content'] = $cache->loadData(); + $response['body'] = $cache->loadData(); break; default: $exceptionMessage = sprintf( '%s resulted in %s %s %s', $url, - $result['code'], - Response::STATUS_CODES[$result['code']] ?? '', + $response['code'], + Response::STATUS_CODES[$response['code']] ?? '', // If debug, include a part of the response body in the exception message - Debug::isEnabled() ? mb_substr($result['body'], 0, 500) : '', + Debug::isEnabled() ? mb_substr($response['body'], 0, 500) : '', ); // The following code must be extracted if it grows too much @@ -192,137 +187,141 @@ function getContents( 'Security | Glassdoor', ]; foreach ($cloudflareTitles as $cloudflareTitle) { - if (str_contains($result['body'], $cloudflareTitle)) { - throw new CloudFlareException($exceptionMessage, $result['code']); + if (str_contains($response['body'], $cloudflareTitle)) { + throw new CloudFlareException($exceptionMessage, $response['code']); } } - throw new HttpException(trim($exceptionMessage), $result['code']); } if ($returnFull === true) { + // For legacy reasons, use content instead of body + $response['content'] = $response['body']; + unset($response['body']); return $response; } - return $response['content']; + return $response['body']; } -/** - * Fetch content from url - * - * @internal Private function used internally - * @throws HttpException - */ -function _http_request(string $url, array $config = []): array +interface HttpClient { - $defaults = [ - 'useragent' => null, - 'timeout' => 5, - 'headers' => [], - 'proxy' => null, - 'curl_options' => [], - 'if_not_modified_since' => null, - 'retries' => 3, - 'max_filesize' => null, - 'max_redirections' => 5, - ]; - $config = array_merge($defaults, $config); + public function request(string $url, array $config = []): array; +} - $ch = curl_init($url); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_MAXREDIRS, $config['max_redirections']); - curl_setopt($ch, CURLOPT_HEADER, false); - $httpHeaders = []; - foreach ($config['headers'] as $name => $value) { - $httpHeaders[] = sprintf('%s: %s', $name, $value); - } - curl_setopt($ch, CURLOPT_HTTPHEADER, $httpHeaders); - if ($config['useragent']) { - curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']); - } - curl_setopt($ch, CURLOPT_TIMEOUT, $config['timeout']); - curl_setopt($ch, CURLOPT_ENCODING, ''); - curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); +final class CurlHttpClient implements HttpClient +{ + public function request(string $url, array $config = []): array + { + $defaults = [ + 'useragent' => null, + 'timeout' => 5, + 'headers' => [], + 'proxy' => null, + 'curl_options' => [], + 'if_not_modified_since' => null, + 'retries' => 3, + 'max_filesize' => null, + 'max_redirections' => 5, + ]; + $config = array_merge($defaults, $config); - if ($config['max_filesize']) { - // This option inspects the Content-Length header - curl_setopt($ch, CURLOPT_MAXFILESIZE, $config['max_filesize']); - curl_setopt($ch, CURLOPT_NOPROGRESS, false); - // This progress function will monitor responses who omit the Content-Length header - curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function ($ch, $downloadSize, $downloaded, $uploadSize, $uploaded) use ($config) { - if ($downloaded > $config['max_filesize']) { - // Return a non-zero value to abort the transfer - return -1; + $ch = curl_init($url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_MAXREDIRS, $config['max_redirections']); + curl_setopt($ch, CURLOPT_HEADER, false); + $httpHeaders = []; + foreach ($config['headers'] as $name => $value) { + $httpHeaders[] = sprintf('%s: %s', $name, $value); + } + curl_setopt($ch, CURLOPT_HTTPHEADER, $httpHeaders); + if ($config['useragent']) { + curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']); + } + curl_setopt($ch, CURLOPT_TIMEOUT, $config['timeout']); + curl_setopt($ch, CURLOPT_ENCODING, ''); + curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + + if ($config['max_filesize']) { + // This option inspects the Content-Length header + curl_setopt($ch, CURLOPT_MAXFILESIZE, $config['max_filesize']); + curl_setopt($ch, CURLOPT_NOPROGRESS, false); + // This progress function will monitor responses who omit the Content-Length header + curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function ($ch, $downloadSize, $downloaded, $uploadSize, $uploaded) use ($config) { + if ($downloaded > $config['max_filesize']) { + // Return a non-zero value to abort the transfer + return -1; + } + return 0; + }); + } + + if ($config['proxy']) { + curl_setopt($ch, CURLOPT_PROXY, $config['proxy']); + } + if (curl_setopt_array($ch, $config['curl_options']) === false) { + throw new \Exception('Tried to set an illegal curl option'); + } + + if ($config['if_not_modified_since']) { + curl_setopt($ch, CURLOPT_TIMEVALUE, $config['if_not_modified_since']); + curl_setopt($ch, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + } + + $responseStatusLines = []; + $responseHeaders = []; + curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($ch, $rawHeader) use (&$responseHeaders, &$responseStatusLines) { + $len = strlen($rawHeader); + if ($rawHeader === "\r\n") { + return $len; } - return 0; + if (preg_match('#^HTTP/(2|1.1|1.0)#', $rawHeader)) { + $responseStatusLines[] = $rawHeader; + return $len; + } + $header = explode(':', $rawHeader); + if (count($header) === 1) { + return $len; + } + $name = mb_strtolower(trim($header[0])); + $value = trim(implode(':', array_slice($header, 1))); + if (!isset($responseHeaders[$name])) { + $responseHeaders[$name] = []; + } + $responseHeaders[$name][] = $value; + return $len; }); - } - if ($config['proxy']) { - curl_setopt($ch, CURLOPT_PROXY, $config['proxy']); - } - if (curl_setopt_array($ch, $config['curl_options']) === false) { - throw new \Exception('Tried to set an illegal curl option'); - } + $attempts = 0; + while (true) { + $attempts++; + $data = curl_exec($ch); + if ($data !== false) { + // The network call was successful, so break out of the loop + break; + } + if ($attempts > $config['retries']) { + // Finally give up + $curl_error = curl_error($ch); + $curl_errno = curl_errno($ch); + throw new HttpException(sprintf( + 'cURL error %s: %s (%s) for %s', + $curl_error, + $curl_errno, + 'https://curl.haxx.se/libcurl/c/libcurl-errors.html', + $url + )); + } + } - if ($config['if_not_modified_since']) { - curl_setopt($ch, CURLOPT_TIMEVALUE, $config['if_not_modified_since']); - curl_setopt($ch, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + $statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + return [ + 'code' => $statusCode, + 'status_lines' => $responseStatusLines, + 'headers' => $responseHeaders, + 'body' => $data, + ]; } - - $responseStatusLines = []; - $responseHeaders = []; - curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($ch, $rawHeader) use (&$responseHeaders, &$responseStatusLines) { - $len = strlen($rawHeader); - if ($rawHeader === "\r\n") { - return $len; - } - if (preg_match('#^HTTP/(2|1.1|1.0)#', $rawHeader)) { - $responseStatusLines[] = $rawHeader; - return $len; - } - $header = explode(':', $rawHeader); - if (count($header) === 1) { - return $len; - } - $name = mb_strtolower(trim($header[0])); - $value = trim(implode(':', array_slice($header, 1))); - if (!isset($responseHeaders[$name])) { - $responseHeaders[$name] = []; - } - $responseHeaders[$name][] = $value; - return $len; - }); - - $attempts = 0; - while (true) { - $attempts++; - $data = curl_exec($ch); - if ($data !== false) { - // The network call was successful, so break out of the loop - break; - } - if ($attempts > $config['retries']) { - // Finally give up - $curl_error = curl_error($ch); - $curl_errno = curl_errno($ch); - throw new HttpException(sprintf( - 'cURL error %s: %s (%s) for %s', - $curl_error, - $curl_errno, - 'https://curl.haxx.se/libcurl/c/libcurl-errors.html', - $url - )); - } - } - - $statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - curl_close($ch); - return [ - 'code' => $statusCode, - 'status_lines' => $responseStatusLines, - 'headers' => $responseHeaders, - 'body' => $data, - ]; } /** From 440adf2f3b62a99160166a531f985064344ba4e7 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 16 Jul 2023 22:28:20 +0200 Subject: [PATCH 018/716] fix(githubissue): add 10 min cache (#3545) --- bridges/GithubIssueBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/GithubIssueBridge.php b/bridges/GithubIssueBridge.php index 9ca84010..a75aa252 100644 --- a/bridges/GithubIssueBridge.php +++ b/bridges/GithubIssueBridge.php @@ -5,7 +5,7 @@ class GithubIssueBridge extends BridgeAbstract const MAINTAINER = 'Pierre Mazière'; const NAME = 'Github Issue'; const URI = 'https://github.com/'; - const CACHE_TIMEOUT = 0; // 10min + const CACHE_TIMEOUT = 600; // 10m const DESCRIPTION = 'Returns the issues or comments of an issue of a github project'; const PARAMETERS = [ From 08d16322e1066f207875bcbd7f9930aadb080860 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 16 Jul 2023 22:37:37 +0200 Subject: [PATCH 019/716] fix: bug in prior conflict merge (#3546) --- lib/contents.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contents.php b/lib/contents.php index 419432df..b6b74539 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -191,7 +191,7 @@ function getContents( throw new CloudFlareException($exceptionMessage, $response['code']); } } - throw new HttpException(trim($exceptionMessage), $result['code']); + throw new HttpException(trim($exceptionMessage), $response['code']); } if ($returnFull === true) { // For legacy reasons, use content instead of body From a1bae7a9a823d5d54074c70596ec59528cf2ed15 Mon Sep 17 00:00:00 2001 From: Paroleen <matteoparolin99@gmail.com> Date: Tue, 18 Jul 2023 00:43:08 +0200 Subject: [PATCH 020/716] [SpotifyBridge] Add search API support (#3548) --- bridges/SpotifyBridge.php | 164 +++++++++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 39 deletions(-) diff --git a/bridges/SpotifyBridge.php b/bridges/SpotifyBridge.php index b58d14be..169075e2 100644 --- a/bridges/SpotifyBridge.php +++ b/bridges/SpotifyBridge.php @@ -7,45 +7,86 @@ class SpotifyBridge extends BridgeAbstract const DESCRIPTION = 'Fetches the latest items from one or more artists, playlists or podcasts'; const MAINTAINER = 'Paroleen'; const CACHE_TIMEOUT = 3600; - const PARAMETERS = [ [ - 'clientid' => [ - 'name' => 'Client ID', - 'type' => 'text', - 'required' => true + const PARAMETERS = [ + 'By Spotify URIs' => [ + 'clientid' => [ + 'name' => 'Client ID', + 'type' => 'text', + 'required' => true + ], + 'clientsecret' => [ + 'name' => 'Client secret', + 'type' => 'text', + 'required' => true + ], + 'country' => [ + 'name' => 'Country/Market', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'US', + 'defaultValue' => 'US' + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'exampleValue' => 10, + 'defaultValue' => 10 + ], + 'spotifyuri' => [ + 'name' => 'Spotify URIs', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'spotify:artist:4lianjyuR1tqf6oUX8kjrZ [,spotify:playlist:37i9dQZF1DXcBWIGoYBM5M,spotify:show:6ShFMYxeDNMo15COLObDvC]', + ], + 'albumtype' => [ + 'name' => 'Album type', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'album,single,appears_on,compilation', + 'defaultValue' => 'album,single' + ] ], - 'clientsecret' => [ - 'name' => 'Client secret', - 'type' => 'text', - 'required' => true + 'By Spotify Search' => [ + 'clientid' => [ + 'name' => 'Client ID', + 'type' => 'text', + 'required' => true + ], + 'clientsecret' => [ + 'name' => 'Client secret', + 'type' => 'text', + 'required' => true + ], + 'market' => [ + 'name' => 'Market', + 'type' => 'text', + 'required' => false, + 'exampleValue' => 'US', + 'defaultValue' => 'US' + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'exampleValue' => 10, + 'defaultValue' => 10 + ], + 'query' => [ + 'name' => 'Search query', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'artist:The Beatles', + ], + 'type' => [ + 'name' => 'Type', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'album,episode', + 'defaultValue' => 'album,episode' + ] ], - 'country' => [ - 'name' => 'Country/Market', - 'type' => 'text', - 'required' => false, - 'exampleValue' => 'US', - 'defaultValue' => 'US' - ], - 'limit' => [ - 'name' => 'Limit', - 'type' => 'number', - 'required' => false, - 'exampleValue' => 10, - 'defaultValue' => 10 - ], - 'spotifyuri' => [ - 'name' => 'Spotify URIs', - 'type' => 'text', - 'required' => true, - 'exampleValue' => 'spotify:artist:4lianjyuR1tqf6oUX8kjrZ [,spotify:playlist:37i9dQZF1DXcBWIGoYBM5M,spotify:show:6ShFMYxeDNMo15COLObDvC]', - ], - 'albumtype' => [ - 'name' => 'Album type', - 'type' => 'text', - 'required' => false, - 'exampleValue' => 'album,single,appears_on,compilation', - 'defaultValue' => 'album,single' - ] - ] ]; + ]; private $uri = ''; private $name = ''; @@ -54,7 +95,13 @@ class SpotifyBridge extends BridgeAbstract public function collectData() { $this->fetchAccessToken(); - $entries = $this->getAllEntries(); + + if ($this->queriedContext === 'By Spotify URIs') { + $entries = $this->getEntriesFromURIs(); + } else { + $entries = $this->getEntriesFromQuery(); + } + usort($entries, function ($entry1, $entry2) { return $this->getDate($entry2) <=> $this->getDate($entry1); }); @@ -78,7 +125,46 @@ class SpotifyBridge extends BridgeAbstract } } - private function getAllEntries() + private function getEntriesFromQuery() + { + $entries = []; + + $types = [ + 'albums', + 'episodes', + ]; + + $query = [ + 'q' => $this->getInput('query'), + 'type' => $this->getInput('type'), + 'market' => $this->getInput('market'), + 'limit' => 50, + ]; + + $hasItems = true; + $offset = 0; + + while ($hasItems && $offset < 1000) { + $hasItems = false; + + $query['offset'] = $offset; + $json = getContents('https://api.spotify.com/v1/search?' . http_build_query($query), ['Authorization: Bearer ' . $this->token]); + $partial = Json::decode($json); + + foreach ($types as $type) { + if (isset($partial[$type]['items'])) { + $entries = array_merge($entries, $partial[$type]['items']); + $hasItems = true; + } + } + + $offset += 50; + } + + return $entries; + } + + private function getEntriesFromURIs() { $entries = []; $uris = explode(',', $this->getInput('spotifyuri')); From 4ce63c88aae24466c6990fa23b73f8ad74dbe3af Mon Sep 17 00:00:00 2001 From: mrtnvgr <48406064+mrtnvgr@users.noreply.github.com> Date: Wed, 19 Jul 2023 01:48:29 +0700 Subject: [PATCH 021/716] Add DoujinStyleBridge (#3549) * v1 * improve title * search support * random support * fix categories * add metadata to content * fix linter errors * i'm sorry --- bridges/DoujinStyleBridge.php | 148 ++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 bridges/DoujinStyleBridge.php diff --git a/bridges/DoujinStyleBridge.php b/bridges/DoujinStyleBridge.php new file mode 100644 index 00000000..84469739 --- /dev/null +++ b/bridges/DoujinStyleBridge.php @@ -0,0 +1,148 @@ +<?php + +class DoujinStyleBridge extends BridgeAbstract +{ + const NAME = 'DoujinStyle Bridge'; + const URI = 'https://doujinstyle.com/'; + const DESCRIPTION = 'Returns submissions from DoujinStyle'; + const MAINTAINER = 'mrtnvgr'; + + // TODO: "Games" support + + const PARAMETERS = [ + 'Most recent submissions' => [], + 'Randomly selected items' => [], + 'From search results' => [ + 'query' => [ + 'name' => 'Search query', + 'required' => true, + 'exampleValue' => 'FELT', + ], + 'flac' => [ + 'name' => 'Include FLAC', + 'type' => 'checkbox', + 'defaultValue' => false, + ], + 'mp3' => [ + 'name' => 'Include MP3', + 'type' => 'checkbox', + 'defaultValue' => false, + ], + 'tta' => [ + 'name' => 'Include TTA', + 'type' => 'checkbox', + 'defaultValue' => false, + ], + 'opus' => [ + 'name' => 'Include Opus', + 'type' => 'checkbox', + 'defaultValue' => false, + ], + 'ogg' => [ + 'name' => 'Include OGG', + 'type' => 'checkbox', + 'defaultValue' => false, + ] + ] + ]; + + public function collectData() + { + $html = getSimpleHTMLDOM($this->getURI()); + $html = defaultLinkTo($html, $this->getURI()); + + $submissions = $html->find('.gridBox .gridDetails'); + foreach ($submissions as $submission) { + $item = []; + + $item['uri'] = $submission->find('a', 0)->href; + + $content = getSimpleHTMLDOM($item['uri']); + $content = defaultLinkTo($content, $this->getURI()); + + $title = $content->find('h2', 0)->plaintext; + + $cover = $content->find('#imgClick a', 0); + if (is_null($cover)) { + $cover = $content->find('.coverWrap', 0)->src; + } else { + $cover = $cover->href; + } + + $item['content'] = "<img src='$cover'/>"; + + $keys = []; + foreach ($content->find('.pageWrap .pageSpan') as $key) { + $keys[] = $key->plaintext; + } + + $values = $content->find('.pageWrap .pageSpan2'); + $metadata = array_combine($keys, $values); + + $format = 'Unknown'; + + foreach ($metadata as $key => $value) { + switch ($key) { + case 'Artist': + $artist = $value->find('a', 0)->plaintext; + $item['title'] = "$artist - $title"; + $item['content'] .= "<br>Artist: $artist"; + break; + case 'Tags:': + $item['categories'] = []; + foreach ($value->find('a') as $tag) { + $tag = str_replace('-', '-', $tag->plaintext); + $item['categories'][] = $tag; + } + + $item['content'] .= '<br>Tags: ' . join(', ', $item['categories']); + break; + case 'Format:': + $item['content'] .= "<br>Format: $value->plaintext"; + break; + case 'Date Added:': + $item['timestamp'] = $value->plaintext; + break; + case 'Provided By:': + $item['author'] = $value->find('a', 0)->plaintext; + break; + } + } + + $this->items[] = $item; + } + } + + public function getURI() + { + $url = self::URI; + + switch ($this->queriedContext) { + case 'From search results': + $url .= '?p=search&type=blanket'; + $url .= '&result=' . $this->getInput('query'); + + if ($this->getInput('flac') == 1) { + $url .= '&format0=on'; + } + if ($this->getInput('mp3') == 1) { + $url .= '&format1=on'; + } + if ($this->getInput('tta') == 1) { + $url .= '&format2=on'; + } + if ($this->getInput('opus') == 1) { + $url .= '&format3=on'; + } + if ($this->getInput('ogg') == 1) { + $url .= '&format4=on'; + } + break; + case 'Randomly selected items': + $url .= '?p=random'; + break; + } + + return $url; + } +} From 087e790ec10d287f944e3abeb5ab3bda9a1a045a Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Wed, 19 Jul 2023 03:28:14 +0200 Subject: [PATCH 022/716] [ImgsedBridge] Add new Instagram Bridge Alternative (#3550) * [ImgsedBridge] Add new Instagram Bridge Alternative Imgsed is a Website adverstised on instagram website, that's is not behind Cloudflare Anti Bot feature. You can select to display Posts, Tags, and Stories of a specific username * [ImgsedBridge] Fix empty defaultValue --- bridges/ImgsedBridge.php | 256 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 bridges/ImgsedBridge.php diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php new file mode 100644 index 00000000..6c49facb --- /dev/null +++ b/bridges/ImgsedBridge.php @@ -0,0 +1,256 @@ +<?php + +class ImgsedBridge extends BridgeAbstract +{ + const MAINTAINER = 'sysadminstory'; + const NAME = 'Imgsed Bridge'; + const URI = 'https://imgsed.com/'; + const INSTAGRAMURI = 'https://www.instagram.com/'; + const CACHE_TIMEOUT = 3600; // 1h + const DESCRIPTION = 'Returns Imgsed (Instagram viewer) content by user'; + + const PARAMETERS = [ + 'Username' => [ + 'u' => [ + 'name' => 'username', + 'type' => 'text', + 'title' => 'Instagram username you want to follow', + 'exampleValue' => 'aesoprockwins', + 'required' => true, + ], + 'post' => [ + 'name' => 'posts', + 'type' => 'checkbox', + 'title' => 'Show posts for this Instagram user', + 'defaultValue' => 'checked', + ], + 'story' => [ + 'name' => 'stories', + 'type' => 'checkbox', + 'title' => 'Show stories for this Instagram user', + ], + 'tagged' => [ + 'name' => 'tagged', + 'type' => 'checkbox', + 'title' => 'Show tagged post for this Instagram user', + ], + ] + ]; + + public function getURI() + { + if (!is_null($this->getInput('u'))) { + return urljoin(self::URI, '/' . $this->getInput('u') . '/'); + } + + return parent::getURI(); + } + + public function collectData() + { + $username = $this->getInput('u'); + try { + // Check if the user exist + $html = getSimpleHTMLDOMCached(self::URI . $username . '/'); + if ($this->getInput('post')) { + $this->collectPosts(); + } + if ($this->getInput('story')) { + $this->collectStories(); + } + if ($this->getInput('tagged')) { + $this->collectTaggeds(); + } + } catch (HttpException $e) { + throw new \Exception(sprintf('Unable to find user `%s`', $username)); + } + } + + private function collectPosts() + { + $username = $this->getInput('u'); + $html = getSimpleHTMLDOMCached(self::URI . $username . '/'); + $html = defaultLinkTo($html, self::URI); + + foreach ($html->find('div[class=item]') as $post) { + $url = $post->find('a', 0)->href; + $instagramURL = $this->convertURLToInstagram($url); + $date = $this->parseDate($post->find('div[class=time]', 0)->plaintext); + $description = $post->find('img', 0)->alt; + $imageUrl = $post->find('img', 0)->src; + // Sometimes, there is some lazy image instead of the real URL + if ($imageUrl == 'https://imgsed.com/img/lazy.jpg') { + $imageUrl = $post->find('img', 0)->getAttribute('data-src'); + } + $download = $post->find('a[class=download]', 0)->href; + $author = $username; + $uid = $post->find('a', 0)->href; + $title = 'Post - ' . $username . ' - ' . $this->descriptionToTitle($description); + + // Checking post type + $isVideo = (bool) $post->find('i[class=video]', 0); + $videoNote = $isVideo ? '<p><i>(video)</i></p>' : ''; + + $isMoreContent = (bool) $post->find('svg', 0); + $moreContentNote = $isMoreContent ? '<p><i>(multiple images and/or videos)</i></p>' : ''; + + + + + $this->items[] = [ + 'uri' => $url, + 'author' => $author, + 'timestamp' => $date, + 'title' => $title, + 'thumbnail' => $imageUrl, + 'enclosures' => [$imageUrl, $download], + 'content' => <<<HTML +<a href="{$url}"> + <img loading="lazy" src="{$imageUrl}" alt="{$description}"/> +</a> +{$videoNote} +{$moreContentNote} +<p>{$description}<p> +<p><a href="{$download}">Download</a></p> +<p><a href="{$instagramURL}">Display on Instagram</a></p> +HTML, + 'uid' => $uid + ]; + } + } + + private function collectStories() + { + try { + $username = $this->getInput('u'); + $html = getSimpleHTMLDOMCached(self::URI . 'api/media/?name=' . $username); + $json = Json::decode($html); + + foreach ($json as $post) { + $url = $post['src']; + $imageUrl = $post['thumb']; + $download = $url; + $author = $username; + $uid = $url; + $title = 'Story - ' . $username; + + $this->items[] = [ + 'uri' => $url, + 'author' => $author, + 'title' => $title, + 'thumbnail' => $imageUrl, + 'enclosures' => [$imageUrl, $download], + 'content' => <<<HTML + <a href="{$url}"> + <img loading="lazy" src="{$imageUrl}" alt="story"/> + </a> + <p><a href="{$download}">Download</a></p> + HTML, + 'uid' => $uid + ]; + } + } catch (Exception $e) { + // If it fails, it's because there are no stories, so don't do anything + } + } + + private function collectTaggeds() + { + $username = $this->getInput('u'); + try { + $html = getSimpleHTMLDOMCached(self::URI . 'tagged/' . $username . '/'); + $html = defaultLinkTo($html, self::URI); + + foreach ($html->find('div[class=item]') as $post) { + $url = $post->find('a', 1)->href; + $instagramURL = $this->convertURLToInstagram($url); + $fromURL = $post->find('div[class=username]', 0)->find('a', 0)->href; + $fromUsername = $post->find('div[class=username]', 0)->plaintext; + $date = $this->parseDate($post->find('div[class=time]', 0)->plaintext); + $description = $post->find('img', 0)->alt; + $imageUrl = $post->find('img', 0)->src; + $download = $post->find('a[class=download]', 0)->href; + $author = $fromUsername; + $uid = $post->find('a', 0)->href; + $title = 'Tagged - ' . $fromUsername . ' - ' . $this->descriptionToTitle($description); + + // Checking post type + $isVideo = (bool) $post->find('i[class=video]', 0); + $videoNote = $isVideo ? '<p><i>(video)</i></p>' : ''; + + $isMoreContent = (bool) $post->find('svg', 0); + $moreContentNote = $isMoreContent ? '<p><i>(multiple images and/or videos)</i></p>' : ''; + + + $this->items[] = [ + 'uri' => $url, + 'author' => $author, + 'timestamp' => $date, + 'title' => $title, + 'thumbnail' => $imageUrl, + 'enclosures' => [$imageUrl, $download], + 'content' => <<<HTML +<a href="{$url}"> + <img loading="lazy" src="{$imageUrl}" alt="{$description}"/> +</a> +{$videoNote} +{$moreContentNote} +<p>From <a href="{$fromURL}">{$fromUsername}</a></p> +<p>{$description}<p> +<p><a href="{$download}">Download</a></p> +<p><a href="{$instagramURL}">Display on Instagram</a></p> +HTML, + 'uid' => $uid + ]; + } + } catch (Exception $e) { + // If it fails, it's because the account was not tagged + } + } + + // Parse date, and transform the date into a timetamp, even in a case of a relative date + private function parseDate($content) + { + $date = date_create(); + $relativeDate = date_interval_create_from_date_string(str_replace(' ago', '', $content)); + if ($relativeDate) { + date_sub($date, $relativeDate); + } + return date_format($date, 'r'); + } + + private function convertURLToInstagram($url) + { + return str_replace(self::URI, self::INSTAGRAMURI, $url); + } + private function descriptionToTitle($description) + { + return strlen($description) > 60 ? mb_substr($description, 0, 57) . '...' : $description; + } + + public function getName() + { + if (!is_null($this->getInput('u'))) { + $types = []; + if ($this->getInput('post')) { + $types[] = 'Posts'; + } + if ($this->getInput('story')) { + $types[] = 'Stories'; + } + if ($this->getInput('tagged')) { + $types[] = 'Tags'; + } + $typesText = $types[0]; + if (count($types) > 1) { + for ($i = 1; $i < count($types) - 1; $i++) { + $typesText .= ', ' . $types[$i]; + } + $typesText .= ' & ' . $types[$i]; + } + + return 'Username ' . $this->getInput('u') . ' - ' . $typesText . ' - Imgsed Bridge'; + } + return parent::getName(); + } +} From 6254b8593e2f7636db65db23c1228482e38be44f Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 19 Jul 2023 05:05:49 +0200 Subject: [PATCH 023/716] refactor(cache): extract and encapsulate cache expiration logic (#3547) * refactor(cache): extract and encapsulate cache expiration logic * fix: logic bug in getSimpleHTMLDOMCached * fix: silly me, index should of course be on the key column * silly me again, PRIMARY keys get index by default lol * comment out the delete portion in loadData * remove a few log statements * tweak twitter cache timeout --- actions/DisplayAction.php | 29 +++++++++++----------- bridges/DemoBridge.php | 1 + bridges/SpotifyBridge.php | 17 +++++-------- bridges/TwitterBridge.php | 6 +---- caches/FileCache.php | 30 ++++++++++++++--------- caches/MemcachedCache.php | 51 ++++++++++++++++++++------------------- caches/NullCache.php | 4 +-- caches/SQLiteCache.php | 40 ++++++++++++++++++------------ lib/BridgeAbstract.php | 22 ++++------------- lib/CacheInterface.php | 4 +-- lib/TwitterClient.php | 15 ++++++------ lib/contents.php | 42 ++++++++++++-------------------- 12 files changed, 124 insertions(+), 137 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 91cfb95f..157937fb 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -90,36 +90,34 @@ class DisplayAction implements ActionInterface $cache = RssBridge::getCache(); $cache->setScope(''); $cache->setKey($cache_params); - // This cache purge will basically delete all cache items older than 24h, regardless of scope and key - $cache->purgeCache(86400); $items = []; $infos = []; - $mtime = $cache->getTime(); + + $feed = $cache->loadData($cacheTimeout); if ( - $mtime - && (time() - $cacheTimeout < $mtime) + $feed && !Debug::isEnabled() ) { - // At this point we found the feed in the cache and debug mode is disabled - if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { + $modificationTime = $cache->getTime(); // The client wants to know if the feed has changed since its last check - $stime = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']); - if ($mtime <= $stime) { - $lastModified2 = gmdate('D, d M Y H:i:s ', $mtime) . 'GMT'; + $modifiedSince = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']); + if ($modificationTime <= $modifiedSince) { + $lastModified2 = gmdate('D, d M Y H:i:s ', $modificationTime) . 'GMT'; return new Response('', 304, ['Last-Modified' => $lastModified2]); } } - // Load the feed from cache and prepare it - $cached = $cache->loadData(); - if (isset($cached['items']) && isset($cached['extraInfos'])) { - foreach ($cached['items'] as $item) { + if ( + isset($feed['items']) + && isset($feed['extraInfos']) + ) { + foreach ($feed['items'] as $item) { $items[] = new FeedItem($item); } - $infos = $cached['extraInfos']; + $infos = $feed['extraInfos']; } } else { // At this point we did NOT find the feed in the cache or debug mode is enabled. @@ -173,6 +171,7 @@ class DisplayAction implements ActionInterface }, $items), 'extraInfos' => $infos ]); + $cache->purgeCache(); } $format->setItems($items); diff --git a/bridges/DemoBridge.php b/bridges/DemoBridge.php index 06ec4e1e..15ab7377 100644 --- a/bridges/DemoBridge.php +++ b/bridges/DemoBridge.php @@ -6,6 +6,7 @@ class DemoBridge extends BridgeAbstract const NAME = 'DemoBridge'; const URI = 'http://github.com/rss-bridge/rss-bridge'; const DESCRIPTION = 'Bridge used for demos'; + const CACHE_TIMEOUT = 15; const PARAMETERS = [ 'testCheckbox' => [ diff --git a/bridges/SpotifyBridge.php b/bridges/SpotifyBridge.php index 169075e2..7b7e2b1d 100644 --- a/bridges/SpotifyBridge.php +++ b/bridges/SpotifyBridge.php @@ -282,14 +282,10 @@ class SpotifyBridge extends BridgeAbstract $cacheKey = sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); $cache->setScope('SpotifyBridge'); $cache->setKey([$cacheKey]); - - $time = null; - if ($cache->getTime()) { - $time = (new DateTime())->getTimestamp() - $cache->getTime(); - } - - if (!$cache->getTime() || $time >= 3600) { - // fetch token + $token = $cache->loadData(3600); + if ($token) { + $this->token = $token; + } else { $basicAuth = base64_encode(sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret'))); $json = getContents('https://accounts.spotify.com/api/token', [ "Authorization: Basic $basicAuth" @@ -298,10 +294,9 @@ class SpotifyBridge extends BridgeAbstract ]); $data = Json::decode($json); $this->token = $data['access_token']; - + $cache->setScope('SpotifyBridge'); + $cache->setKey([$cacheKey]); $cache->saveData($this->token); - } else { - $this->token = $cache->loadData(); } } diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index befb8064..f0f0ee52 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -7,7 +7,7 @@ class TwitterBridge extends BridgeAbstract const API_URI = 'https://api.twitter.com'; const GUEST_TOKEN_USES = 100; const GUEST_TOKEN_EXPIRY = 10800; // 3hrs - const CACHE_TIMEOUT = 300; // 5min + const CACHE_TIMEOUT = 60 * 15; // 15min const DESCRIPTION = 'returns tweets'; const MAINTAINER = 'arnd-s'; const PARAMETERS = [ @@ -224,10 +224,6 @@ EOD switch ($this->queriedContext) { case 'By username': $cache = RssBridge::getCache(); - $cache->setScope('twitter'); - $cache->setKey(['cache']); - // todo: inspect mtime instead of purging with 3h - $cache->purgeCache(60 * 60 * 3); $api = new TwitterClient($cache); $screenName = $this->getInput('u'); diff --git a/caches/FileCache.php b/caches/FileCache.php index a4f48962..6e150cb4 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -1,5 +1,8 @@ <?php +/** + * @link https://www.php.net/manual/en/function.clearstatcache.php + */ class FileCache implements CacheInterface { private array $config; @@ -25,18 +28,25 @@ class FileCache implements CacheInterface return $this->config; } - public function loadData() + public function loadData(int $timeout = 86400) { + clearstatcache(); if (!file_exists($this->getCacheFile())) { return null; } - $data = unserialize(file_get_contents($this->getCacheFile())); - if ($data === false) { - // Intentionally not throwing an exception - Logger::warning(sprintf('Failed to unserialize: %s', $this->getCacheFile())); - return null; + $modificationTime = filemtime($this->getCacheFile()); + if (time() - $timeout < $modificationTime) { + $data = unserialize(file_get_contents($this->getCacheFile())); + if ($data === false) { + Logger::warning(sprintf('Failed to unserialize: %s', $this->getCacheFile())); + // Intentionally not throwing an exception + return null; + } + return $data; } - return $data; + // It's a good idea to delete the expired item here, but commented out atm + // unlink($this->getCacheFile()); + return null; } public function saveData($data): void @@ -49,9 +59,7 @@ class FileCache implements CacheInterface public function getTime(): ?int { - // https://www.php.net/manual/en/function.clearstatcache.php clearstatcache(); - $cacheFile = $this->getCacheFile(); if (file_exists($cacheFile)) { $time = filemtime($cacheFile); @@ -64,7 +72,7 @@ class FileCache implements CacheInterface return null; } - public function purgeCache(int $seconds): void + public function purgeCache(int $timeout = 86400): void { if (! $this->config['enable_purge']) { return; @@ -90,7 +98,7 @@ class FileCache implements CacheInterface continue; } elseif ($cacheFile->isFile()) { $filepath = $cacheFile->getPathname(); - if (filemtime($filepath) < time() - $seconds) { + if (filemtime($filepath) < time() - $timeout) { // todo: sometimes this file doesn't exists unlink($filepath); } diff --git a/caches/MemcachedCache.php b/caches/MemcachedCache.php index 8cd1932b..afc654ea 100644 --- a/caches/MemcachedCache.php +++ b/caches/MemcachedCache.php @@ -6,8 +6,6 @@ class MemcachedCache implements CacheInterface private string $key; private $conn; private $expiration = 0; - private $time = null; - private $data = null; public function __construct() { @@ -43,50 +41,53 @@ class MemcachedCache implements CacheInterface $this->conn = $conn; } - public function loadData() + public function loadData(int $timeout = 86400) { - if ($this->data) { - return $this->data; - } - $result = $this->conn->get($this->getCacheKey()); - if ($result === false) { + $value = $this->conn->get($this->getCacheKey()); + if ($value === false) { return null; } - - $this->time = $result['time']; - $this->data = $result['data']; - return $result['data']; + if (time() - $timeout < $value['time']) { + return $value['data']; + } + return null; } public function saveData($data): void { - $time = time(); - $object_to_save = [ + $value = [ 'data' => $data, - 'time' => $time, + 'time' => time(), ]; - $result = $this->conn->set($this->getCacheKey(), $object_to_save, $this->expiration); - + $result = $this->conn->set($this->getCacheKey(), $value, $this->expiration); if ($result === false) { - throw new \Exception('Cannot write the cache to memcached server'); + Logger::warning('Failed to store an item in memcached', [ + 'scope' => $this->scope, + 'key' => $this->key, + 'expiration' => $this->expiration, + 'code' => $this->conn->getLastErrorCode(), + 'message' => $this->conn->getLastErrorMessage(), + 'number' => $this->conn->getLastErrorErrno(), + ]); + // Intentionally not throwing an exception } - - $this->time = $time; } public function getTime(): ?int { - if ($this->time === null) { - $this->loadData(); + $value = $this->conn->get($this->getCacheKey()); + if ($value === false) { + return null; } - return $this->time; + return $value['time']; } - public function purgeCache(int $seconds): void + public function purgeCache(int $timeout = 86400): void { + $this->conn->flush(); // Note: does not purges cache right now // Just sets cache expiration and leave cache purging for memcached itself - $this->expiration = $seconds; + $this->expiration = $timeout; } public function setScope(string $scope): void diff --git a/caches/NullCache.php b/caches/NullCache.php index 1a01df2f..fe43fe06 100644 --- a/caches/NullCache.php +++ b/caches/NullCache.php @@ -12,7 +12,7 @@ class NullCache implements CacheInterface { } - public function loadData() + public function loadData(int $timeout = 86400) { } @@ -25,7 +25,7 @@ class NullCache implements CacheInterface return null; } - public function purgeCache(int $seconds): void + public function purgeCache(int $timeout = 86400): void { } } diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index f9258a88..d7ab1374 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -29,27 +29,37 @@ class SQLiteCache implements CacheInterface $this->db = new \SQLite3($config['file']); $this->db->enableExceptions(true); $this->db->exec("CREATE TABLE storage ('key' BLOB PRIMARY KEY, 'value' BLOB, 'updated' INTEGER)"); - $this->db->exec('CREATE INDEX idx_storage_updated ON storage (updated)'); } $this->db->busyTimeout($config['timeout']); } - public function loadData() + public function loadData(int $timeout = 86400) { - $stmt = $this->db->prepare('SELECT value FROM storage WHERE key = :key'); + $stmt = $this->db->prepare('SELECT value, updated FROM storage WHERE key = :key'); $stmt->bindValue(':key', $this->getCacheKey()); $result = $stmt->execute(); - if ($result) { - $row = $result->fetchArray(\SQLITE3_ASSOC); - if ($row !== false) { - $blob = $row['value']; - $data = unserialize($blob); - if ($data !== false) { - return $data; - } - Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100))); - } + if (!$result) { + return null; } + $row = $result->fetchArray(\SQLITE3_ASSOC); + if ($row === false) { + return null; + } + $value = $row['value']; + $modificationTime = $row['updated']; + if (time() - $timeout < $modificationTime) { + $data = unserialize($value); + if ($data === false) { + Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($value, 0, 100))); + return null; + } + return $data; + } + // It's a good idea to delete expired cache items. + // However I'm seeing lots of SQLITE_BUSY errors so commented out for now + // $stmt = $this->db->prepare('DELETE FROM storage WHERE key = :key'); + // $stmt->bindValue(':key', $this->getCacheKey()); + // $stmt->execute(); return null; } @@ -78,13 +88,13 @@ class SQLiteCache implements CacheInterface return null; } - public function purgeCache(int $seconds): void + public function purgeCache(int $timeout = 86400): void { if (!$this->config['enable_purge']) { return; } $stmt = $this->db->prepare('DELETE FROM storage WHERE updated < :expired'); - $stmt->bindValue(':expired', time() - $seconds); + $stmt->bindValue(':expired', time() - $timeout); $stmt->execute(); } diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index f024393d..eb9d5a3c 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -409,26 +409,15 @@ abstract class BridgeAbstract implements BridgeInterface /** * Loads a cached value for the specified key * - * @param int $duration Cache duration (optional) + * @param int $timeout Cache duration (optional) * @return mixed Cached value or null if the key doesn't exist or has expired */ - protected function loadCacheValue(string $key, $duration = null) + protected function loadCacheValue(string $key, int $timeout = 86400) { $cache = RssBridge::getCache(); - // Create class name without the namespace part - $scope = $this->getShortName(); - $cache->setScope($scope); + $cache->setScope($this->getShortName()); $cache->setKey([$key]); - $timestamp = $cache->getTime(); - - if ( - $duration - && $timestamp - && $timestamp < time() - $duration - ) { - return null; - } - return $cache->loadData(); + return $cache->loadData($timeout); } /** @@ -439,8 +428,7 @@ abstract class BridgeAbstract implements BridgeInterface protected function saveCacheValue(string $key, $value) { $cache = RssBridge::getCache(); - $scope = $this->getShortName(); - $cache->setScope($scope); + $cache->setScope($this->getShortName()); $cache->setKey([$key]); $cache->saveData($value); } diff --git a/lib/CacheInterface.php b/lib/CacheInterface.php index 414a9c84..85aa830f 100644 --- a/lib/CacheInterface.php +++ b/lib/CacheInterface.php @@ -6,11 +6,11 @@ interface CacheInterface public function setKey(array $key): void; - public function loadData(); + public function loadData(int $timeout = 86400); public function saveData($data): void; public function getTime(): ?int; - public function purgeCache(int $seconds): void; + public function purgeCache(int $timeout = 86400): void; } diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index cdedcbc1..341610a4 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -11,8 +11,13 @@ class TwitterClient public function __construct(CacheInterface $cache) { $this->cache = $cache; - $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; + + $cache->setScope('twitter'); + $cache->setKey(['cache']); + $cache->purgeCache(60 * 60 * 3); + $this->data = $this->cache->loadData() ?? []; + $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; } public function fetchUserTweets(string $screenName): \stdClass @@ -22,7 +27,6 @@ class TwitterClient $userInfo = $this->fetchUserInfoByScreenName($screenName); } catch (HttpException $e) { if ($e->getCode() === 403) { - Logger::info('The guest token has expired'); $this->data['guest_token'] = null; $this->fetchGuestToken(); $userInfo = $this->fetchUserInfoByScreenName($screenName); @@ -35,7 +39,6 @@ class TwitterClient $timeline = $this->fetchTimeline($userInfo->rest_id); } catch (HttpException $e) { if ($e->getCode() === 403) { - Logger::info('The guest token has expired'); $this->data['guest_token'] = null; $this->fetchGuestToken(); $timeline = $this->fetchTimeline($userInfo->rest_id); @@ -88,7 +91,6 @@ class TwitterClient private function fetchGuestToken(): void { if (isset($this->data['guest_token'])) { - Logger::info('Reusing cached guest token: ' . $this->data['guest_token']); return; } $url = 'https://api.twitter.com/1.1/guest/activate.json'; @@ -99,7 +101,6 @@ class TwitterClient $this->cache->setScope('twitter'); $this->cache->setKey(['cache']); $this->cache->saveData($this->data); - Logger::info("Fetch new guest token: $guest_token"); } private function fetchUserInfoByScreenName(string $screenName) @@ -115,7 +116,7 @@ class TwitterClient 'https://twitter.com/i/api/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName?variables=%s', urlencode(json_encode($variables)) ); - $response = json_decode(getContents($url, $this->createHttpHeaders())); + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); if (isset($response->errors)) { // Grab the first error message throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message)); @@ -168,7 +169,7 @@ class TwitterClient urlencode(json_encode($variables)), urlencode(json_encode($features)) ); - $response = json_decode(getContents($url, $this->createHttpHeaders())); + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); return $response; } diff --git a/lib/contents.php b/lib/contents.php index b6b74539..454f2066 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -100,9 +100,6 @@ function getContents( bool $returnFull = false ) { $httpClient = RssBridge::getHttpClient(); - $cache = RssBridge::getCache(); - $cache->setScope('server'); - $cache->setKey([$url]); // Snagged from https://github.com/lwthiker/curl-impersonate/blob/main/firefox/curl_ff102 $defaultHttpHeaders = [ @@ -138,6 +135,11 @@ function getContents( if (Configuration::getConfig('proxy', 'url') && !defined('NOPROXY')) { $config['proxy'] = Configuration::getConfig('proxy', 'url'); } + + $cache = RssBridge::getCache(); + $cache->setScope('server'); + $cache->setKey([$url]); + if (!Debug::isEnabled() && $cache->getTime()) { $config['if_not_modified_since'] = $cache->getTime(); } @@ -384,7 +386,7 @@ function getSimpleHTMLDOM( * _Notice_: Cached contents are forcefully removed after 24 hours (86400 seconds). * * @param string $url The URL. - * @param int $duration Cache duration in seconds. + * @param int $timeout Cache duration in seconds. * @param array $header (optional) A list of cURL header. * For more information follow the links below. * * https://php.net/manual/en/function.curl-setopt.php @@ -409,7 +411,7 @@ function getSimpleHTMLDOM( */ function getSimpleHTMLDOMCached( $url, - $duration = 86400, + $timeout = 86400, $header = [], $opts = [], $lowercase = true, @@ -422,29 +424,15 @@ function getSimpleHTMLDOMCached( $cache = RssBridge::getCache(); $cache->setScope('pages'); $cache->setKey([$url]); - - // Determine if cached file is within duration - $time = $cache->getTime(); - if ( - $time - && time() - $duration < $time - && !Debug::isEnabled() - ) { - // Cache hit - $content = $cache->loadData(); - } else { - $content = getContents( - $url, - $header ?? [], - $opts ?? [] - ); - if ($content) { - $cache->setScope('pages'); - $cache->setKey([$url]); - $cache->saveData($content); - } + $content = $cache->loadData($timeout); + if (!$content || Debug::isEnabled()) { + $content = getContents($url, $header ?? [], $opts ?? []); + } + if ($content) { + $cache->setScope('pages'); + $cache->setKey([$url]); + $cache->saveData($content); } - return str_get_html( $content, $lowercase, From f91723d9e5b3880122bafc365411a4ff6f5bbda1 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 19 Jul 2023 05:18:26 +0200 Subject: [PATCH 024/716] fix(memcached): do not flush entire cache, oops (#3551) --- caches/MemcachedCache.php | 1 - 1 file changed, 1 deletion(-) diff --git a/caches/MemcachedCache.php b/caches/MemcachedCache.php index afc654ea..dcb572c7 100644 --- a/caches/MemcachedCache.php +++ b/caches/MemcachedCache.php @@ -84,7 +84,6 @@ class MemcachedCache implements CacheInterface public function purgeCache(int $timeout = 86400): void { - $this->conn->flush(); // Note: does not purges cache right now // Just sets cache expiration and leave cache purging for memcached itself $this->expiration = $timeout; From a4a328583a1870fc3cb2617d46072d54553402f0 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 19 Jul 2023 06:39:17 +0200 Subject: [PATCH 025/716] fix(reddit): set custom http ua to fix 429 errors (#3552) * refactor * refactor * fix(reddit): set custom http ua to fix 429 errors * lint --- bridges/RedditBridge.php | 108 ++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index de80f09d..86d7884b 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -73,47 +73,6 @@ class RedditBridge extends BridgeAbstract ] ]; - public function detectParameters($url) - { - $parsed_url = parse_url($url); - - $host = $parsed_url['host'] ?? null; - - if ($host != 'www.reddit.com' && $host != 'old.reddit.com') { - return null; - } - - $path = explode('/', $parsed_url['path']); - - if ($path[1] == 'r') { - return [ - 'r' => $path[2] - ]; - } elseif ($path[1] == 'user') { - return [ - 'u' => $path[2] - ]; - } else { - return null; - } - } - - public function getIcon() - { - return 'https://www.redditstatic.com/desktop2x/img/favicon/favicon-96x96.png'; - } - - public function getName() - { - if ($this->queriedContext == 'single') { - return 'Reddit r/' . $this->getInput('r'); - } elseif ($this->queriedContext == 'user') { - return 'Reddit u/' . $this->getInput('u'); - } else { - return self::NAME; - } - } - public function collectData() { $user = false; @@ -152,18 +111,22 @@ class RedditBridge extends BridgeAbstract foreach ($subreddits as $subreddit) { $name = trim($subreddit); - $values = getContents(self::URI - . '/search.json?q=' - . $keywords - . $flair - . ($user ? 'author%3A' : 'subreddit%3A') - . $name - . '&sort=' - . $this->getInput('d') - . '&include_over_18=on'); - $decodedValues = json_decode($values); + $url = self::URI + . '/search.json?q=' + . $keywords + . $flair + . ($user ? 'author%3A' : 'subreddit%3A') + . $name + . '&sort=' + . $this->getInput('d') + . '&include_over_18=on'; - foreach ($decodedValues->data->children as $post) { + $version = 'v0.0.1'; + $useragent = "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"; + $json = getContents($url, ['User-Agent: ' . $useragent]); + $parsedJson = Json::decode($json, false); + + foreach ($parsedJson->data->children as $post) { if ($post->kind == 't1' && !$comments) { continue; } @@ -288,6 +251,22 @@ class RedditBridge extends BridgeAbstract }); } + public function getIcon() + { + return 'https://www.redditstatic.com/desktop2x/img/favicon/favicon-96x96.png'; + } + + public function getName() + { + if ($this->queriedContext == 'single') { + return 'Reddit r/' . $this->getInput('r'); + } elseif ($this->queriedContext == 'user') { + return 'Reddit u/' . $this->getInput('u'); + } else { + return self::NAME; + } + } + private function encodePermalink($link) { return self::URI . implode( @@ -307,4 +286,29 @@ class RedditBridge extends BridgeAbstract { return '<a href="' . $href . '">' . $text . '</a>'; } + + public function detectParameters($url) + { + $parsed_url = parse_url($url); + + $host = $parsed_url['host'] ?? null; + + if ($host != 'www.reddit.com' && $host != 'old.reddit.com') { + return null; + } + + $path = explode('/', $parsed_url['path']); + + if ($path[1] == 'r') { + return [ + 'r' => $path[2] + ]; + } elseif ($path[1] == 'user') { + return [ + 'u' => $path[2] + ]; + } else { + return null; + } + } } From 93620aa1058e9aca6e3c7f7bd8458630cd478360 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 19 Jul 2023 22:05:26 +0200 Subject: [PATCH 026/716] fix(cache): bug in cache logic (#3553) It is possible to have a cached item with a very old mtime but it's technically expired. So, check for presence of time and whether the time it is within 10 days --- lib/contents.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contents.php b/lib/contents.php index 454f2066..12a98b0c 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -140,7 +140,7 @@ function getContents( $cache->setScope('server'); $cache->setKey([$url]); - if (!Debug::isEnabled() && $cache->getTime()) { + if (!Debug::isEnabled() && $cache->getTime() && $cache->loadData(86400 * 7)) { $config['if_not_modified_since'] = $cache->getTime(); } From 517c7f5c9b003977aa7151b91c6e79342964c60f Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 19 Jul 2023 22:18:42 +0200 Subject: [PATCH 027/716] fix(cache): bug (#3554) --- lib/contents.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contents.php b/lib/contents.php index 12a98b0c..4429a75c 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -169,7 +169,7 @@ function getContents( break; case 304: // Not Modified - $response['body'] = $cache->loadData(); + $response['body'] = $cache->loadData(86400 * 7); break; default: $exceptionMessage = sprintf( From 2ffb54c7c2b961c83b0a876bd5d34f7d698c2b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pred=C3=A4?= <46051820+PredaaA@users.noreply.github.com> Date: Thu, 20 Jul 2023 00:52:09 +0200 Subject: [PATCH 028/716] [PicukiBridge] Add count parameter (#3556) --- bridges/PicukiBridge.php | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/bridges/PicukiBridge.php b/bridges/PicukiBridge.php index 7a4f9eb5..f1d45e2a 100644 --- a/bridges/PicukiBridge.php +++ b/bridges/PicukiBridge.php @@ -9,6 +9,14 @@ class PicukiBridge extends BridgeAbstract const DESCRIPTION = 'Returns Picuki (Instagram viewer) posts by user and by hashtag'; const PARAMETERS = [ + 'global' => [ + 'count' => [ + 'name' => 'Count', + 'type' => 'number', + 'title' => 'How many posts to fetch', + 'defaultValue' => 12 + ] + ], 'Username' => [ 'u' => [ 'name' => 'username', @@ -43,6 +51,13 @@ class PicukiBridge extends BridgeAbstract $re = '#let short_code = "(.*?)";\s*$#m'; $html = getSimpleHTMLDOM($this->getURI()); + $requestedCount = $this->getInput('count'); + if ($requestedCount > 12) { + // Picuki shows 12 posts per page at initial load. + throw new \Exception('Maximum count is 12'); + } + + $count = 0; foreach ($html->find('.box-photos .box-photo') as $element) { // skip ad items if (in_array('adv', explode(' ', $element->class))) { @@ -86,14 +101,19 @@ class PicukiBridge extends BridgeAbstract 'source' => $sourceUrl, 'enclosures' => [$imageUrl], 'content' => <<<HTML -<a href="{$url}"> - <img loading="lazy" src="{$imageUrl}" /> -</a> -<a href="{$sourceUrl}">{$sourceUrl}</a> -{$videoNote} -<p>{$description}<p> -HTML + <a href="{$url}"> + <img loading="lazy" src="{$imageUrl}" /> + </a> + <a href="{$sourceUrl}">{$sourceUrl}</a> + {$videoNote} + <p>{$description}<p> + HTML ]; + + $count++; + if ($count >= $requestedCount) { + break; + } } } From 663729cf19da206e62cb65ddc4fdb151c66b3498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pred=C3=A4?= <46051820+PredaaA@users.noreply.github.com> Date: Thu, 20 Jul 2023 05:50:45 +0200 Subject: [PATCH 029/716] [TikTokBridge] Use another way to get videos infos to include video link (#3557) * [TikTokBridge] Use another way to get videos infos to include video link * [TikTokBridge] Use cover if dynamicCover is empty * [TikTokBridge] Add support for the rest of item params --- bridges/TikTokBridge.php | 42 ++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php index 1a30570d..e7cac825 100644 --- a/bridges/TikTokBridge.php +++ b/bridges/TikTokBridge.php @@ -33,32 +33,36 @@ class TikTokBridge extends BridgeAbstract $title = $html->find('h1', 0)->plaintext ?? self::NAME; $this->feedName = htmlspecialchars_decode($title); - foreach ($html->find('div.tiktok-x6y88p-DivItemContainerV2') as $div) { + $SIGI_STATE_RAW = $html->find('script[id=SIGI_STATE]', 0)->innertext; + $SIGI_STATE = json_decode($SIGI_STATE_RAW); + + foreach ($SIGI_STATE->ItemModule as $key => $value) { $item = []; - // todo: find proper link to tiktok item - $link = $div->find('a', 0)->href; - - $image = $div->find('img', 0)->src ?? ''; - - $views = $div->find('strong.video-count', 0)->plaintext; - - if ($link === 'https://www.tiktok.com/') { - $link = $this->getURI(); + $link = 'https://www.tiktok.com/@' . $value->author . '/video/' . $value->id; + $image = $value->video->dynamicCover; + if (empty($image)) { + $image = $value->video->cover; } + $views = $value->stats->playCount; + $hastags = []; + foreach ($value->textExtra as $tag) { + $hastags[] = $tag->hashtagName; + } + $hastags_str = ''; + foreach ($hastags as $tag) { + $hastags_str .= '<a href="https://www.tiktok.com/tag/' . $tag . '">#' . $tag . '</a> '; + } + $item['uri'] = $link; - - $a = $div->find('a', 1); - if ($a) { - $item['title'] = $a->plaintext; - } else { - $item['title'] = $this->getName(); - } + $item['title'] = $value->desc; + $item['timestamp'] = $value->createTime; + $item['author'] = '@' . $value->author; $item['enclosures'][] = $image; - + $item['categories'] = $hastags; $item['content'] = <<<EOD <a href="{$link}"><img src="{$image}"/></a> -<p>{$views} views<p> +<p>{$views} views<p><br/>Hashtags: {$hastags_str} EOD; $this->items[] = $item; From 0a118310cb123460e49778ec74791cb2d0b6ba6f Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 20 Jul 2023 19:11:13 +0200 Subject: [PATCH 030/716] fix(sqlitecache): store blob as blob (#3555) serialize() can return output with null bytes and other non-text data. The prior behavior truncated data which later results in unserialize() errors. This happens when e.g. caching an object with a private field or when caching e.g. a JPEG file (starts with 0xFFD8FFE1) Fixes errors such as e.g.: unserialize(): Error at offset 20 of 24 bytes at caches/SQLiteCache.php line 51 --- caches/SQLiteCache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index d7ab1374..92235862 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -69,7 +69,7 @@ class SQLiteCache implements CacheInterface $stmt = $this->db->prepare('INSERT OR REPLACE INTO storage (key, value, updated) VALUES (:key, :value, :updated)'); $stmt->bindValue(':key', $this->getCacheKey()); - $stmt->bindValue(':value', $blob); + $stmt->bindValue(':value', $blob, \SQLITE3_BLOB); $stmt->bindValue(':updated', time()); $stmt->execute(); } From d08b2616ef1adfa69edc10f42e4e7fcb0c9226cb Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Fri, 21 Jul 2023 20:26:22 +0200 Subject: [PATCH 031/716] feat(twitter): use account icon as feed icon, fix #3348 (#3561) --- bridges/TwitterBridge.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index f0f0ee52..189203bd 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -124,6 +124,7 @@ EOD private $apiKey = null; private $guestToken = null; private $authHeaders = []; + private ?string $feedIconUrl = null; public function detectParameters($url) { @@ -309,6 +310,10 @@ EOD } } + if ($this->queriedContext === 'By username') { + $this->feedIconUrl = $data->user_info->legacy->profile_image_url_https ?? null; + } + foreach ($tweets as $tweet) { // Skip own Retweets... if (isset($tweet->retweeted_status) && $tweet->retweeted_status->user->id_str === $tweet->user->id_str) { @@ -497,6 +502,11 @@ EOD; usort($this->items, ['TwitterBridge', 'compareTweetId']); } + public function getIcon() + { + return $this->feedIconUrl ?? parent::getIcon(); + } + private static function compareTweetId($tweet1, $tweet2) { return (intval($tweet1['id']) < intval($tweet2['id']) ? 1 : -1); From 39a8346c53997551f0aa68392a1e9387106a356a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Fri, 21 Jul 2023 20:52:20 +0200 Subject: [PATCH 032/716] fix(pokemonnews): throw if antibot, #3327 (#3562) --- bridges/PokemonNewsBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/PokemonNewsBridge.php b/bridges/PokemonNewsBridge.php index 954e584c..3dacb163 100644 --- a/bridges/PokemonNewsBridge.php +++ b/bridges/PokemonNewsBridge.php @@ -14,7 +14,10 @@ final class PokemonNewsBridge extends BridgeAbstract // todo: parse json api instead: https://www.pokemon.com/api/1/us/news/get-news.json $url = 'https://www.pokemon.com/us/pokemon-news'; $dom = getSimpleHTMLDOM($url); - + $haystack = (string)$dom; + if (str_contains($haystack, 'Request unsuccessful. Incapsula incident')) { + throw new \Exception('Blocked by anti-bot'); + } foreach ($dom->find('.news-list ul li') as $item) { $title = $item->find('h3', 0)->plaintext; $description = $item->find('p.hidden-mobile', 0); From 38ca124de0886de6e808d1a21526f04f47343326 Mon Sep 17 00:00:00 2001 From: Eugene Molotov <eugene.molotov@yandex.ru> Date: Sat, 22 Jul 2023 17:00:12 +0500 Subject: [PATCH 033/716] [VkBridge] Better title generation (#3563) 1. Use first parargraph only 2. Remove tags 3. Allow to use comma and colon in title --- bridges/VkBridge.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 0cd2fba8..c5a9d4cc 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -448,7 +448,9 @@ class VkBridge extends BridgeAbstract private function getTitle($content) { - preg_match('/^["\w\ \p{L}\(\)\?#«»-]+/mu', htmlspecialchars_decode($content), $result); + $content = explode('<br>', $content)[0]; + $content = strip_tags($content); + preg_match('/^[:,"\w\ \p{L}\(\)\?#«»-]+/mu', htmlspecialchars_decode($content), $result); if (count($result) == 0) { return 'untitled'; } From 74635fd752b4421760b4b80fb058c8488d608b8d Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 23 Jul 2023 23:05:35 +0200 Subject: [PATCH 034/716] fix(DisplayAction): improve error handling and cache logic (#3558) * fix(DisplayAction): improve error handling and cache logic * restore prev timeouts * refactor * yup * test: fix unit test * leave twitter client unchanged * leave twitter bridge unchanged --- actions/DisplayAction.php | 126 +++++++++++++++++++-------------- bridges/GoogleSearchBridge.php | 2 +- lib/contents.php | 5 ++ lib/utils.php | 6 +- 4 files changed, 81 insertions(+), 58 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 157937fb..129d4587 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -1,27 +1,38 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - class DisplayAction implements ActionInterface { + private CacheInterface $cache; + public function execute(array $request) { if (Configuration::getConfig('system', 'enable_maintenance_mode')) { return new Response('503 Service Unavailable', 503); } + $this->cache = RssBridge::getCache(); + $this->cache->setScope('http'); + $this->cache->setKey($request); + // avg timeout of 20m + $timeout = 60 * 15 + rand(1, 60 * 10); + /** @var Response $cachedResponse */ + $cachedResponse = $this->cache->loadData($timeout); + if ($cachedResponse && !Debug::isEnabled()) { + //Logger::info(sprintf('Returning cached (http) response: %s', $cachedResponse->getBody())); + return $cachedResponse; + } + $response = $this->createResponse($request); + if (in_array($response->getCode(), [429, 503])) { + //Logger::info(sprintf('Storing cached (http) response: %s', $response->getBody())); + $this->cache->setScope('http'); + $this->cache->setKey($request); + $this->cache->saveData($response); + } + return $response; + } + private function createResponse(array $request) + { $bridgeFactory = new BridgeFactory(); - $bridgeClassName = $bridgeFactory->createBridgeClassName($request['bridge'] ?? ''); $format = $request['format'] ?? null; @@ -87,46 +98,36 @@ class DisplayAction implements ActionInterface ) ); - $cache = RssBridge::getCache(); - $cache->setScope(''); - $cache->setKey($cache_params); + $this->cache->setScope(''); + $this->cache->setKey($cache_params); $items = []; $infos = []; - $feed = $cache->loadData($cacheTimeout); + $feed = $this->cache->loadData($cacheTimeout); - if ( - $feed - && !Debug::isEnabled() - ) { + if ($feed && !Debug::isEnabled()) { if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { - $modificationTime = $cache->getTime(); + $modificationTime = $this->cache->getTime(); // The client wants to know if the feed has changed since its last check $modifiedSince = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']); if ($modificationTime <= $modifiedSince) { - $lastModified2 = gmdate('D, d M Y H:i:s ', $modificationTime) . 'GMT'; - return new Response('', 304, ['Last-Modified' => $lastModified2]); + $modificationTimeGMT = gmdate('D, d M Y H:i:s ', $modificationTime); + return new Response('', 304, ['Last-Modified' => $modificationTimeGMT . 'GMT']); } } - if ( - isset($feed['items']) - && isset($feed['extraInfos']) - ) { + if (isset($feed['items']) && isset($feed['extraInfos'])) { foreach ($feed['items'] as $item) { $items[] = new FeedItem($item); } $infos = $feed['extraInfos']; } } else { - // At this point we did NOT find the feed in the cache or debug mode is enabled. try { $bridge->setDatas($bridge_params); $bridge->collectData(); - $items = $bridge->getItems(); - if (isset($items[0]) && is_array($items[0])) { $feedItems = []; foreach ($items as $item) { @@ -141,46 +142,62 @@ class DisplayAction implements ActionInterface 'icon' => $bridge->getIcon() ]; } catch (\Exception $e) { + $errorOutput = Configuration::getConfig('error', 'output'); + $reportLimit = Configuration::getConfig('error', 'report_limit'); if ($e instanceof HttpException) { - Logger::warning(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e))); + // Reproduce (and log) these responses regardless of error output and report limit if ($e->getCode() === 429) { + Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e))); + return new Response('429 Too Many Requests', 429); + } + if ($e->getCode() === 503) { + Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e))); return new Response('503 Service Unavailable', 503); } - } else { + // Might want to cache other codes such as 504 Gateway Timeout + } + if (in_array($errorOutput, ['feed', 'none'])) { Logger::error(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e)), ['e' => $e]); } - - // Emit error only if we are passed the error report limit - $errorCount = self::logBridgeError($bridge->getName(), $e->getCode()); - if ($errorCount >= Configuration::getConfig('error', 'report_limit')) { - if (Configuration::getConfig('error', 'output') === 'feed') { - // Emit the error as a feed item in a feed so that feed readers can pick it up + $errorCount = 1; + if ($reportLimit > 1) { + $errorCount = $this->logBridgeError($bridge->getName(), $e->getCode()); + } + // Let clients know about the error if we are passed the report limit + if ($errorCount >= $reportLimit) { + if ($errorOutput === 'feed') { + // Render the exception as a feed item $items[] = $this->createFeedItemFromException($e, $bridge); - } elseif (Configuration::getConfig('error', 'output') === 'http') { + } elseif ($errorOutput === 'http') { + // Rethrow so that the main exception handler in RssBridge.php produces an HTTP 500 throw $e; + } elseif ($errorOutput === 'none') { + // Do nothing (produces an empty feed) + } else { + // Do nothing, unknown error output? Maybe throw exception or validate in Configuration.php } } } // Unfortunately need to set scope and key again because they might be modified - $cache->setScope(''); - $cache->setKey($cache_params); - $cache->saveData([ + $this->cache->setScope(''); + $this->cache->setKey($cache_params); + $this->cache->saveData([ 'items' => array_map(function (FeedItem $item) { return $item->toArray(); }, $items), 'extraInfos' => $infos ]); - $cache->purgeCache(); + $this->cache->purgeCache(); } $format->setItems($items); $format->setExtraInfos($infos); - $lastModified = $cache->getTime(); - $format->setLastModified($lastModified); + $newModificationTime = $this->cache->getTime(); + $format->setLastModified($newModificationTime); $headers = []; - if ($lastModified) { - $headers['Last-Modified'] = gmdate('D, d M Y H:i:s ', $lastModified) . 'GMT'; + if ($newModificationTime) { + $headers['Last-Modified'] = gmdate('D, d M Y H:i:s ', $newModificationTime) . 'GMT'; } $headers['Content-Type'] = $format->getMimeType() . '; charset=' . $format->getCharset(); return new Response($format->stringify(), 200, $headers); @@ -210,13 +227,12 @@ class DisplayAction implements ActionInterface return $item; } - private static function logBridgeError($bridgeName, $code) + private function logBridgeError($bridgeName, $code) { - $cache = RssBridge::getCache(); - $cache->setScope('error_reporting'); - $cache->setkey([$bridgeName . '_' . $code]); - - if ($report = $cache->loadData()) { + $this->cache->setScope('error_reporting'); + $this->cache->setkey([$bridgeName . '_' . $code]); + $report = $this->cache->loadData(); + if ($report) { $report = Json::decode($report); $report['time'] = time(); $report['count']++; @@ -227,7 +243,7 @@ class DisplayAction implements ActionInterface 'count' => 1, ]; } - $cache->saveData(Json::encode($report)); + $this->cache->saveData(Json::encode($report)); return $report['count']; } diff --git a/bridges/GoogleSearchBridge.php b/bridges/GoogleSearchBridge.php index 59465e89..9b0713ac 100644 --- a/bridges/GoogleSearchBridge.php +++ b/bridges/GoogleSearchBridge.php @@ -5,7 +5,7 @@ class GoogleSearchBridge extends BridgeAbstract const MAINTAINER = 'sebsauvage'; const NAME = 'Google search'; const URI = 'https://www.google.com/'; - const CACHE_TIMEOUT = 1800; // 30min + const CACHE_TIMEOUT = 60 * 30; // 30m const DESCRIPTION = 'Returns max 100 results from the past year.'; const PARAMETERS = [[ diff --git a/lib/contents.php b/lib/contents.php index 4429a75c..5587a98e 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -63,6 +63,11 @@ final class Response return $this->body; } + public function getCode() + { + return $this->code; + } + public function getHeaders() { return $this->headers; diff --git a/lib/utils.php b/lib/utils.php index ea329f8d..94f928cd 100644 --- a/lib/utils.php +++ b/lib/utils.php @@ -51,11 +51,13 @@ function get_current_url(): string function create_sane_exception_message(\Throwable $e): string { + $sanitizedMessage = sanitize_root($e->getMessage()); + $sanitizedFilepath = sanitize_root($e->getFile()); return sprintf( '%s: %s in %s line %s', get_class($e), - sanitize_root($e->getMessage()), - sanitize_root($e->getFile()), + $sanitizedMessage, + $sanitizedFilepath, $e->getLine() ); } From b6fab206010f23145a864da5b896e2aebd384bff Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 23 Jul 2023 23:05:56 +0200 Subject: [PATCH 035/716] docs: improve readme (#3560) * docs: improve readme --- README.md | 159 ++++++++++++++++++++++++++++++++--------- config.default.ini.php | 1 + 2 files changed, 128 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index e9f7360d..e0487e6b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,11 @@ ![RSS-Bridge](static/logo_600px.png) -RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one. +RSS-Bridge is a web application. + +It generates web feeds for websites that don't have one. + +Officially hosted instance: https://rss-bridge.org/bridge01/ [![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg?logo=github)](https://github.com/rss-bridge/rss-bridge/releases/latest) @@ -17,34 +21,43 @@ RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for website |![Screenshot #5](/static/screenshot-5.png?raw=true)|![Screenshot #6](/static/screenshot-6.png?raw=true)| |![Screenshot #7](/static/twitter-form.png?raw=true)|![Screenshot #8](/static/twitter-rasmus.png?raw=true)| -## A subset of bridges +## A subset of bridges (17/412) -* `YouTube` : YouTube user channel, playlist or search -* `Twitter` : Return keyword/hashtag search or user timeline -* `Telegram` : Return the latest posts from a public group -* `Reddit` : Return the latest posts from a subreddit or user -* `Filter` : Filter an existing feed url -* `Vk` : Latest posts from a user or group -* `FeedMerge` : Merge two or more existing feeds into one -* `Twitch` : Fetch the latest videos from a channel -* `ThePirateBay` : Returns the newest indexed torrents from [The Pirate Bay](https://thepiratebay.se/) with keywords - -And [many more](bridges/), thanks to the community! +* `CssSelectorBridge`: [Scrape out a feed using CSS selectors](https://rss-bridge.org/bridge01/#bridge-CssSelectorBridge) +* `FeedMergeBridge`: [Combine multiple feeds into one](https://rss-bridge.org/bridge01/#bridge-FeedMergeBridge) +* `FeedReducerBridge`: [Reduce a noisy feed by some percentage](https://rss-bridge.org/bridge01/#bridge-FeedReducerBridge) +* `FilterBridge`: [Filter a feed by excluding/including items by keyword](https://rss-bridge.org/bridge01/#bridge-FilterBridge) +* `GettrBridge`: [Fetches the latest posts from a GETTR user](https://rss-bridge.org/bridge01/#bridge-GettrBridge) +* `MastodonBridge`: [Fetches statuses from a Mastodon (ActivityPub) instance](https://rss-bridge.org/bridge01/#bridge-MastodonBridge) +* `RedditBridge`: [Fetches posts from a user/subredit (with filtering options)](https://rss-bridge.org/bridge01/#bridge-RedditBridge) +* `RumbleBridge`: [Fetches channel/user videos](https://rss-bridge.org/bridge01/#bridge-RumbleBridge) +* `SoundcloudBridge`: [Fetches music by username](https://rss-bridge.org/bridge01/#bridge-SoundcloudBridge) +* `TelegramBridge`: [Fetches posts from a public channel](https://rss-bridge.org/bridge01/#bridge-TelegramBridge) +* `ThePirateBayBridge:` [Fetches torrents by search/user/category](https://rss-bridge.org/bridge01/#bridge-ThePirateBayBridge) +* `TikTokBridge`: [Fetches posts by username](https://rss-bridge.org/bridge01/#bridge-TikTokBridge) +* `TwitchBridge`: [Fetches videos from channel](https://rss-bridge.org/bridge01/#bridge-TwitchBridge) +* `TwitterBridge`: [Fetches tweets](https://rss-bridge.org/bridge01/#bridge-TwitterBridge) +* `VkBridge`: [Fetches posts from user/group](https://rss-bridge.org/bridge01/#bridge-VkBridge) +* `XPathBridge`: [Scrape out a feed using XPath expressions](https://rss-bridge.org/bridge01/#bridge-XPathBridge) +* `YoutubeBridge`: [Fetches videos by username/channel/playlist/search](https://rss-bridge.org/bridge01/#bridge-YoutubeBridge) +* `YouTubeCommunityTabBridge`: [Fetches posts from a channel's community tab](https://rss-bridge.org/bridge01/#bridge-YouTubeCommunityTabBridge) [Full documentation](https://rss-bridge.github.io/rss-bridge/index.html) -Check out RSS-Bridge right now on https://rss-bridge.org/bridge01 or find another +Check out RSS-Bridge right now on https://rss-bridge.org/bridge01/ + +Alternatively find another [public instance](https://rss-bridge.github.io/rss-bridge/General/Public_Hosts.html). ## Tutorial -RSS-Bridge requires php 7.4 (or higher). - ### Install with composer or git +Requires minimum PHP 7.4. + ```shell cd /var/www -composer create-project --no-dev rss-bridge/rss-bridge +composer create-project -v --no-dev rss-bridge/rss-bridge ``` ```shell @@ -80,9 +93,9 @@ server { } ``` -### Install with Docker: +### Install from Docker Hub: -Install by using docker image from Docker Hub: +Install by downloading the docker image from Docker Hub: ```bash # Create container @@ -94,7 +107,7 @@ docker start rss-bridge Browse http://localhost:3000/ -Install by locally building the image: +### Install by locally building from Dockerfile ```bash # Build image from Dockerfile @@ -103,13 +116,13 @@ docker build -t rss-bridge . # Create container docker create --name rss-bridge --publish 3000:80 rss-bridge -# Start the container +# Start container docker start rss-bridge ``` Browse http://localhost:3000/ -#### Install with docker-compose +### Install with docker-compose Create a `docker-compose.yml` file locally with with the following content: ```yml @@ -132,7 +145,7 @@ docker-compose up Browse http://localhost:3000/ -### Alternative installation methods +### Other installation methods [![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge) [![Deploy to Heroku](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) @@ -175,6 +188,8 @@ Learn more in [bridge api](https://rss-bridge.github.io/rss-bridge/Bridge_API/in ### How to enable all bridges +Modify `config.ini.php`: + enabled_bridges[] = * ### How to enable some bridges @@ -186,14 +201,82 @@ enabled_bridges[] = GettrBridge ### How to enable debug mode +The +[debug mode](https://rss-bridge.github.io/rss-bridge/For_Developers/Debug_mode.html) +disables the majority of caching operations. + enable_debug_mode = true -Learn more in [debug mode](https://rss-bridge.github.io/rss-bridge/For_Developers/Debug_mode.html). +### How to switch to memcached as cache backend + +``` +[cache] + +; Cache backend: file (default), sqlite, memcached, null +type = "memcached" +``` + +### How to switch to sqlite3 as cache backend + + type = "sqlite" + +### How to disable bridge errors (as feed items) + +When a bridge fails, RSS-Bridge will produce a feed with a single item describing the error. + +This way, feed readers pick it up and you are notified. + +If you don't want this behaviour, switch the error output to `http`: + + [error] + + ; Defines how error messages are returned by RSS-Bridge + ; + ; "feed" = As part of the feed (default) + ; "http" = As HTTP error message + ; "none" = No errors are reported + output = "http" + +### How to accumulate errors before finally reporting it + +Modify `report_limit` so that an error must occur 3 times before it is reported. + + ; Defines how often an error must occur before it is reported to the user + report_limit = 3 + +### How to password-protect the instance + +HTTP basic access authentication: + + [authentication] + + enable = true + username = "alice" + password = "cat" + +Will typically require feed readers to be configured with the credentials. + +It may also be possible to manually include the credentials in the URL: + +https://alice:cat@rss-bridge.org/bridge01/?action=display&bridge=FabriceBellardBridge&format=Html ### How to create a new output format [Create a new format](https://rss-bridge.github.io/rss-bridge/Format_API/index.html). +### How to run unit tests and linter + +These commands require that you have installed the dev dependencies in `composer.json`. + + ./vendor/bin/phpunit + ./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./ + +### How to spawn a minimal development environment + + php -S 127.0.0.1:9001 + +http://127.0.0.1:9001/ + ## Explanation We are RSS-Bridge community, a group of developers continuing the project initiated by sebsauvage, @@ -205,15 +288,19 @@ webmaster of See [CONTRIBUTORS.md](CONTRIBUTORS.md) RSS-Bridge uses caching to prevent services from banning your server for repeatedly updating feeds. -The specific cache duration can be different between bridges. Cached files are deleted automatically after 24 hours. +The specific cache duration can be different between bridges. +Cached files are deleted automatically after 24 hours. RSS-Bridge allows you to take full control over which bridges are displayed to the user. That way you can host your own RSS-Bridge service with your favorite collection of bridges! +Current maintainers (as of 2023): @dvikan and @Mynacol #2519 ## Reference -### FeedItem properties +### Feed item structure + +This is the feed item structure that bridges are expected to produce. ```php $item = [ @@ -236,13 +323,21 @@ That way you can host your own RSS-Bridge service with your favorite collection ] ``` -### Output formats: +### Output formats -* `Atom` : Atom feed, for use in feed readers -* `Html` : Simple HTML page -* `Json` : JSON, for consumption by other applications -* `Mrss` : MRSS feed, for use in feed readers -* `Plaintext` : Raw text, for consumption by other applications +* `Atom`: Atom feed, for use in feed readers +* `Html`: Simple HTML page +* `Json`: JSON, for consumption by other applications +* `Mrss`: MRSS feed, for use in feed readers +* `Plaintext`: Raw text, for consumption by other applications +* `Sfeed`: Text, TAB separated + +### Cache backends + +* `file` +* `sqlite` +* `memcached` +* `null` ### Licenses diff --git a/config.default.ini.php b/config.default.ini.php index 1c9a20ae..d0c508f4 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -8,6 +8,7 @@ ; Only these bridges are available for feed production ; How to enable all bridges: enabled_bridges[] = * +enabled_bridges[] = CssSelectorBridge enabled_bridges[] = FeedMerge enabled_bridges[] = FeedReducerBridge enabled_bridges[] = Filter From 1f6c2cd32c9668c5945580a9cfb04948188b0a1d Mon Sep 17 00:00:00 2001 From: Simon Alberny <contact@simounet.net> Date: Mon, 24 Jul 2023 16:25:09 +0200 Subject: [PATCH 036/716] Allocine Sorties movie date added (#3569) --- bridges/AllocineFRSortiesBridge.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/bridges/AllocineFRSortiesBridge.php b/bridges/AllocineFRSortiesBridge.php index b77c2f9b..a75187be 100644 --- a/bridges/AllocineFRSortiesBridge.php +++ b/bridges/AllocineFRSortiesBridge.php @@ -24,6 +24,7 @@ class AllocineFRSortiesBridge extends BridgeAbstract $thumb = $element->find('figure.thumbnail', 0); $meta = $element->find('div.meta-body', 0); $synopsis = $element->find('div.synopsis', 0); + $date = $element->find('span.date', 0); $title = $element->find('a[class*=meta-title-link]', 0); $content = trim(defaultLinkTo($thumb->outertext . $meta->outertext . $synopsis->outertext, static::URI)); @@ -34,8 +35,32 @@ class AllocineFRSortiesBridge extends BridgeAbstract $item['content'] = $content; $item['title'] = trim($title->innertext); + $item['timestamp'] = $this->frenchPubDateToTimestamp($date->plaintext); $item['uri'] = static::BASE_URI . '/' . substr($title->href, 1); $this->items[] = $item; } } + + private function frenchPubDateToTimestamp($date) + { + return strtotime( + strtr( + strtolower($date), + [ + 'janvier' => 'jan', + 'février' => 'feb', + 'mars' => 'march', + 'avril' => 'apr', + 'mai' => 'may', + 'juin' => 'jun', + 'juillet' => 'jul', + 'août' => 'aug', + 'septembre' => 'sep', + 'octobre' => 'oct', + 'novembre' => 'nov', + 'décembre' => 'dec' + ] + ) + ); + } } From 2cc89b767c7f357da724c551b8cf2a512fc2f744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Wr=C3=B3bel?= <me@dawidwrobel.com> Date: Tue, 25 Jul 2023 20:52:47 +0200 Subject: [PATCH 037/716] [AllegroBridge] fix non-functional bridge (#3571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — fix cookie pattern – use data analytics attributes wherever possible to avoid relying on obfuscated class names — add support for promoted offers — include sponsored and promoted offers by default — some additional refactoring --- bridges/AllegroBridge.php | 83 ++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/bridges/AllegroBridge.php b/bridges/AllegroBridge.php index 55457416..be240857 100644 --- a/bridges/AllegroBridge.php +++ b/bridges/AllegroBridge.php @@ -16,14 +16,20 @@ class AllegroBridge extends BridgeAbstract 'sessioncookie' => [ 'name' => 'The \'wdctx\' session cookie', 'title' => 'Paste the value of the \'wdctx\' cookie from your browser if you want to prevent Allegro imposing rate limits', - 'pattern' => '^.{250,};?$', + 'pattern' => '^.{70,};?$', // phpcs:ignore 'exampleValue' => 'v4.1-oCrmXTMqv2ppC21GTUCKLmUwRPP1ssQVALKuqwsZ1VXjcKgL2vO5TTRM5xMxS9GiyqxF1gAeyc-63dl0coUoBKXCXi_nAmr95yyqGpq2RAFoneZ4L399E8n6iYyemcuGARjAoSfjvLHJCEwvvHHynSgaxlFBu7hUnKfuy39zo9sSQdyTUjotJg3CAZ53q9v2raAnPCyGOAR4ytRILd9p24EJnxp7_oR0XbVPIo1hDa4WmjXFOxph8rHaO5tWd', 'required' => false, ], 'includeSponsoredOffers' => [ 'type' => 'checkbox', - 'name' => 'Include Sponsored Offers' + 'name' => 'Include Sponsored Offers', + 'defaultValue' => 'checked' + ], + 'includePromotedOffers' => [ + 'type' => 'checkbox', + 'name' => 'Include Promoted Offers', + 'defaultValue' => 'checked' ] ]]; @@ -63,58 +69,57 @@ class AllegroBridge extends BridgeAbstract return; } - $results = $html->find('._6a66d_V7Lel article'); + $results = $html->find('article[data-analytics-view-custom-context="REGULAR"]'); if (!$this->getInput('includeSponsoredOffers')) { - $results = array_filter($results, function ($node) { - return $node->{'data-analytics-view-label'} != 'showSponsoredItems'; - }); + $results = array_merge($results, $html->find('article[data-analytics-view-custom-context="SPONSORED"]')); + } + + if (!$this->getInput('includePromotedOffers')) { + $results = array_merge($results, $html->find('article[data-analytics-view-custom-context="PROMOTED"]')); } foreach ($results as $post) { $item = []; - $item['uri'] = $post->find('._6a66d_LX75-', 0)->href; - -//TODO: port this over, whatever it does, from https://github.com/MK-PL/AllegroRSS -// if (arrayLinks.includes('events/clicks?')) { -// let sponsoredLink = new URL(arrayLinks).searchParams.get('redirect') -// arrayLinks = sponsoredLink.slice(0, sponsoredLink.indexOf('?')) -// } - - $item['title'] = $post->find('._6a66d_LX75-', 0)->innertext; - $item['uid'] = $post->{'data-analytics-view-value'}; + $item_link = $post->find('a[href*="' . $item['uid'] . '"], a[href*="allegrolokalnie"]', 0); + + $item['uri'] = $item_link->href; + + $item['title'] = $item_link->find('img', 0)->alt; + + $image = $item_link->find('img', 0)->{'data-src'} ?: $item_link->find('img', 0)->src ?? false; + + if ($image) { + $item['enclosures'] = [$image . '#.image']; + } + + $price = $post->{'data-analytics-view-json-custom-price'}; + if ($price) { + $priceDecoded = json_decode(html_entity_decode($price)); + $price = $priceDecoded->amount . ' ' . $priceDecoded->currency; + } + $descriptionPatterns = ['/<\s*dt[^>]*>\b/', '/<\/dt>/', '/<\s*dd[^>]*>\b/', '/<\/dd>/']; $descriptionReplacements = ['<span>', ':</span> ', '<strong>', ' </strong> ']; $description = $post->find('.m7er_k4.mpof_5r.mpof_z0_s', 0)->innertext; $descriptionPretty = preg_replace($descriptionPatterns, $descriptionReplacements, $description); - $buyNowAuction = $post->find('.mqu1_g3.mvrt_0.mgn2_12', 0)->innertext ?? ''; - $buyNowAuction = str_replace('</span><span', '</span> <span', $buyNowAuction); - - $auctionTimeLeft = $post->find('._6a66d_ImOzU', 0)->innertext ?? ''; - - $price = $post->find('._6a66d_6R3iN', 0)->plaintext; - $price = empty($auctionTimeLeft) ? $price : $price . '- kwota licytacji'; - - $image = $post->find('._6a66d_44ioA img', 0)->{'data-src'} ?: $post->find('._6a66d_44ioA img', 0)->src ?? false; - if ($image) { - $item['enclosures'] = [$image . '#.image']; - } - - $offerExtraInfo = array_filter($post->find('.mqu1_g3.mgn2_12'), function ($node) { + $pricingExtraInfo = array_filter($post->find('.mqu1_g3.mgn2_12'), function ($node) { return empty($node->find('.mvrt_0')); }); - $offerExtraInfo = $offerExtraInfo[0]->plaintext ?? ''; + $pricingExtraInfo = $pricingExtraInfo[0]->plaintext ?? ''; - $isSmart = $post->find('._6a66d_TC2Zk', 0)->innertext ?? ''; - if (str_contains($isSmart, 'z kurierem')) { - $offerExtraInfo .= ', Smart z kurierem'; - } else { - $offerExtraInfo .= ', Smart'; + $offerExtraInfo = array_map(function ($node) { + return str_contains($node->plaintext, 'zapłać później') ? '' : $node->outertext; + }, $post->find('div.mpof_ki.mwdn_1.mj7a_4.mgn2_12')); + + $isSmart = $post->find('img[alt="Smart!"]', 0) ?? false; + if ($isSmart) { + $pricingExtraInfo .= $isSmart->outertext; } $item['categories'] = []; @@ -131,11 +136,9 @@ class AllegroBridge extends BridgeAbstract . '<div><strong>' . $price . '</strong></div><div>' - . $auctionTimeLeft - . '</div><div>' - . $buyNowAuction + . implode('</div><div>', $offerExtraInfo) . '</div><dl>' - . $offerExtraInfo + . $pricingExtraInfo . '</dl><hr>'; $this->items[] = $item; From 556bca58cf9ce9c2038c387cbfba9dfa70de6ac6 Mon Sep 17 00:00:00 2001 From: csisoap <33269526+csisoap@users.noreply.github.com> Date: Wed, 26 Jul 2023 03:36:41 +0700 Subject: [PATCH 038/716] [TwitterBridge] Fix search, user, list ID (#3566) * Add ability to fetch user, list tweet * Fix user, search, list ID although list still broke * clear whitespace * Revert CACHE_TIMEOUT * clear whitespace, change single quote * Clear PHP warning, add ability to get full-text if truncated * Clear PHP warning * clear warning * clear whitespace * Add check condition for mediaDetails. * Add whitespace * Add try catch exception for get full-text tweet * clear warning * clear warning --- bridges/TwitterBridge.php | 120 ++++++++++----- lib/TwitterClient.php | 314 +++++++++++++++++++++++++++++++++----- 2 files changed, 364 insertions(+), 70 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 189203bd..1ba00c66 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -210,6 +210,16 @@ EOD } } + private function getFullText($id) + { + $url = sprintf( + 'https://cdn.syndication.twimg.com/tweet-result?id=%s&lang=en', + $id + ); + + return json_decode(getContents($url), false); + } + public function collectData() { // $data will contain an array of all found tweets (unfiltered) @@ -220,13 +230,11 @@ EOD $tweets = []; // Get authentication information - + $cache = RssBridge::getCache(); + $api = new TwitterClient($cache); // Try to get all tweets switch ($this->queriedContext) { case 'By username': - $cache = RssBridge::getCache(); - $api = new TwitterClient($cache); - $screenName = $this->getInput('u'); $screenName = trim($screenName); $screenName = ltrim($screenName, '@'); @@ -238,35 +246,45 @@ EOD case 'By keyword or hashtag': // Does not work with the recent twitter changes $params = [ - 'q' => urlencode($this->getInput('q')), - 'tweet_mode' => 'extended', - 'tweet_search_mode' => 'live', + 'q' => urlencode($this->getInput('q')), + 'tweet_mode' => 'extended', + 'tweet_search_mode' => 'live', ]; - $data = $this->makeApiCall('/1.1/search/tweets.json', $params)->statuses; + $tweets = $api->search($params)->statuses; + $data = (object) [ + 'tweets' => $tweets + ]; break; case 'By list': // Does not work with the recent twitter changes - $params = [ - 'slug' => strtolower($this->getInput('list')), - 'owner_screen_name' => strtolower($this->getInput('user')), - 'tweet_mode' => 'extended', + // $params = [ + // 'slug' => strtolower($this->getInput('list')), + // 'owner_screen_name' => strtolower($this->getInput('user')), + // 'tweet_mode' => 'extended', + // ]; + $query = [ + 'screenName' => strtolower($this->getInput('user')), + 'listSlug' => strtolower($this->getInput('list')) ]; - $data = $this->makeApiCall('/1.1/lists/statuses.json', $params); + $data = $api->fetchListTweets($query, $this->queriedContext); break; case 'By list ID': // Does not work with the recent twitter changes - $params = [ - 'list_id' => $this->getInput('listid'), - 'tweet_mode' => 'extended', + // $params = [ + // 'list_id' => $this->getInput('listid'), + // 'tweet_mode' => 'extended', + // ]; + + $query = [ + 'listId' => $this->getInput('listid') ]; - $data = $this->makeApiCall('/1.1/lists/statuses.json', $params); + $data = $api->fetchListTweets($query, $this->queriedContext); break; - default: returnServerError('Invalid query context !'); } @@ -314,6 +332,7 @@ EOD $this->feedIconUrl = $data->user_info->legacy->profile_image_url_https ?? null; } + $i = 0; foreach ($tweets as $tweet) { // Skip own Retweets... if (isset($tweet->retweeted_status) && $tweet->retweeted_status->user->id_str === $tweet->user->id_str) { @@ -325,14 +344,6 @@ EOD continue; } - switch ($this->queriedContext) { - case 'By username': - if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) { - continue 2; - } - break; - } - $item = []; $realtweet = $tweet; @@ -341,11 +352,40 @@ EOD $realtweet = $tweet->retweeted_status; } - $item['username'] = $data->user_info->legacy->screen_name; - $item['fullname'] = $data->user_info->legacy->name; - $item['avatar'] = $data->user_info->legacy->profile_image_url_https; + if (isset($realtweet->truncated) && $realtweet->truncated) { + try { + $realtweet = $this->getFullText($realtweet->id_str); + } catch (HttpException $e) { + $realtweet = $tweet; + } + } + + switch ($this->queriedContext) { + case 'By username': + if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) { + continue 2; + } + $item['username'] = $data->user_info->legacy->screen_name; + $item['fullname'] = $data->user_info->legacy->name; + $item['avatar'] = $data->user_info->legacy->profile_image_url_https; + $item['id'] = $realtweet->id_str; + break; + case 'By list': + case 'By list ID': + $item['username'] = $data->userIds[$i]->legacy->screen_name; + $item['fullname'] = $data->userIds[$i]->legacy->name; + $item['avatar'] = $data->userIds[$i]->legacy->profile_image_url_https; + $item['id'] = $realtweet->conversation_id_str; + break; + case 'By keyword or hashtag': + $item['username'] = $realtweet->user->screen_name; + $item['fullname'] = $realtweet->user->name; + $item['avatar'] = $realtweet->user->profile_image_url_https; + $item['id'] = $realtweet->id_str; + break; + } + $item['timestamp'] = $realtweet->created_at; - $item['id'] = $realtweet->id_str; $item['uri'] = self::URI . $item['username'] . '/status/' . $item['id']; $item['author'] = (isset($tweet->retweeted_status) ? 'RT: ' : '') . $item['fullname'] @@ -353,7 +393,11 @@ EOD . $item['username'] . ')'; // Convert plain text URLs into HTML hyperlinks - $fulltext = $realtweet->full_text; + if (isset($realtweet->full_text)) { + $fulltext = $realtweet->full_text; + } else { + $fulltext = $realtweet->text; + } $cleanedTweet = $fulltext; $foundUrls = false; @@ -385,7 +429,7 @@ EOD if ($foundUrls === false) { // fallback to regex'es $reg_ex = '/(http|https|ftp|ftps)\:\/\/[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(\/\S*)?/'; - if (preg_match($reg_ex, $realtweet->full_text, $url)) { + if (preg_match($reg_ex, $fulltext, $url)) { $cleanedTweet = preg_replace( $reg_ex, "<a href='{$url[0]}' target='_blank'>{$url[0]}</a> ", @@ -410,10 +454,17 @@ EOD EOD; } + $medias = []; + if (isset($realtweet->extended_entities->media)) { + $medias = $realtweet->extended_entities->media; + } else if (isset($realtweet->mediaDetails)) { + $medias = $realtweet->mediaDetails; + } + // Get images $media_html = ''; - if (isset($realtweet->extended_entities->media) && !$this->getInput('noimg')) { - foreach ($realtweet->extended_entities->media as $media) { + if (!$this->getInput('noimg')) { + foreach ($medias as $media) { switch ($media->type) { case 'photo': $image = $media->media_url_https . '?name=orig'; @@ -496,6 +547,7 @@ EOD; EOD; // put out + $i++; $this->items[] = $item; } diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 341610a4..92c797d6 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -20,6 +20,88 @@ class TwitterClient $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; } + private function extractTweetAndUsersFromGraphQL($timeline) + { + if (isset($timeline->data->user)) { + $result = $timeline->data->user->result; + $instructions = $result->timeline_v2->timeline->instructions; + } else { + $result = $timeline->data->list->timeline_response; + $instructions = $result->timeline->instructions; + } + if (isset($result->__typename) && $result->__typename === 'UserUnavailable') { + throw new \Exception('UserUnavailable'); + } + $instructionTypes = [ + 'TimelineAddEntries', + 'TimelineClearCache', + 'TimelinePinEntry', // unclear purpose, maybe pinned tweet? + ]; + if (!isset($instructions[1]) && isset($timeline->data->user)) { + throw new \Exception('The account exists but has not tweeted yet?'); + } + + $entries = null; + foreach ($instructions as $instruction) { + $instructionType = ''; + if (isset($instruction->type)) { + $instructionType = $instruction->type; + } else { + $instructionType = $instruction->__typename; + } + + if ($instructionType === 'TimelineAddEntries') { + $entries = $instruction->entries; + break; + } + } + if (!$entries) { + throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type')))); + } + + $tweets = []; + $userIds = []; + foreach ($entries as $entry) { + $entryType = ''; + + if (isset($entry->content->entryType)) { + $entryType = $entry->content->entryType; + } else { + $entryType = $entry->content->__typename; + } + + if ($entryType !== 'TimelineTimelineItem') { + continue; + } + + if (isset($timeline->data->user)) { + if (!isset($entry->content->itemContent->tweet_results->result->legacy)) { + continue; + } + $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; + + $userIds[] = $entry->content->itemContent->tweet_results->result->core->user_results->result; + } else { + if (!isset($entry->content->content->tweetResult->result->legacy)) { + continue; + } + $tweets[] = $entry->content->content->tweetResult->result->legacy; + + $userIds[] = $entry->content->content->tweetResult->result->core->user_result->result; + } + } + + return (object) [ + 'userIds' => $userIds, + 'tweets' => $tweets, + ]; + } + + private function extractTweetFromSearch($searchResult) + { + return $searchResult->statuses; + } + public function fetchUserTweets(string $screenName): \stdClass { $this->fetchGuestToken(); @@ -36,58 +118,66 @@ class TwitterClient } try { - $timeline = $this->fetchTimeline($userInfo->rest_id); + $timeline = $this->fetchTimelineUsingSearch($screenName); } catch (HttpException $e) { if ($e->getCode() === 403) { $this->data['guest_token'] = null; $this->fetchGuestToken(); - $timeline = $this->fetchTimeline($userInfo->rest_id); + $timeline = $this->fetchTimelineUsingSearch($screenName); } else { throw $e; } } - $result = $timeline->data->user->result; - if ($result->__typename === 'UserUnavailable') { - throw new \Exception('UserUnavailable'); - } - $instructionTypes = [ - 'TimelineAddEntries', - 'TimelineClearCache', - 'TimelinePinEntry', // unclear purpose, maybe pinned tweet? - ]; - $instructions = $result->timeline_v2->timeline->instructions; - if (!isset($instructions[1])) { - throw new \Exception('The account exists but has not tweeted yet?'); - } + $tweets = $this->extractTweetFromSearch($timeline); - $entries = null; - foreach ($instructions as $instruction) { - if ($instruction->type === 'TimelineAddEntries') { - $entries = $instruction->entries; - break; - } - } - if (!$entries) { - throw new \Exception(sprintf('Unable to find time line tweets in: %s', implode(',', array_column($instructions, 'type')))); - } - - $tweets = []; - foreach ($entries as $entry) { - if ($entry->content->entryType !== 'TimelineTimelineItem') { - continue; - } - if (!isset($entry->content->itemContent->tweet_results->result->legacy)) { - continue; - } - $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; - } return (object) [ 'user_info' => $userInfo, 'tweets' => $tweets, ]; } + public function fetchListTweets($query, $operation = '') + { + $id = ''; + $this->fetchGuestToken(); + if ($operation == 'By list') { + try { + $listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']); + $id = $listInfo->id_str; + } catch (HttpException $e) { + if ($e->getCode() === 403) { + $this->data['guest_token'] = null; + $this->fetchGuestToken(); + $listInfo = $this->fetchListInfoBySlug($query['screenName'], $query['listSlug']); + $id = $listInfo->id_str; + } else { + throw $e; + } + } + } else if ($operation == 'By list ID') { + $id = $query['listId']; + } else { + throw new \Exception('Unknown operation to make list tweets'); + } + + try { + $timeline = $this->fetchListTimeline($id); + } catch (HttpException $e) { + if ($e->getCode() === 403) { + $this->data['guest_token'] = null; + $this->fetchGuestToken(); + $timeline = $this->fetchListTimeline($id); + } else { + throw $e; + } + } + + $data = $this->extractTweetAndUsersFromGraphQL($timeline); + + return $data; + } + private function fetchGuestToken(): void { if (isset($this->data['guest_token'])) { @@ -173,6 +263,158 @@ class TwitterClient return $response; } + private function fetchTimelineUsingSearch($screenName) + { + $params = [ + 'q' => 'from:' . $screenName, + 'modules' => 'status', + 'result_type' => 'recent' + ]; + $response = $this->search($params); + return $response; + } + + public function search($queryParam) + { + $url = sprintf( + 'https://api.twitter.com/1.1/search/tweets.json?%s', + http_build_query($queryParam) + ); + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + return $response; + } + + private function fetchListInfoBySlug($screenName, $listSlug) + { + if (isset($this->data[$screenName . '-' . $listSlug])) { + return $this->data[$screenName . '-' . $listSlug]; + } + + $features = [ + 'android_graphql_skip_api_media_color_palette' => false, + 'blue_business_profile_image_shape_enabled' => false, + 'creator_subscriptions_subscription_count_enabled' => false, + 'creator_subscriptions_tweet_preview_api_enabled' => true, + 'freedom_of_speech_not_reach_fetch_enabled' => false, + 'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false, + 'hidden_profile_likes_enabled' => false, + 'highlights_tweets_tab_ui_enabled' => false, + 'interactive_text_enabled' => false, + 'longform_notetweets_consumption_enabled' => true, + 'longform_notetweets_inline_media_enabled' => false, + 'longform_notetweets_richtext_consumption_enabled' => true, + 'longform_notetweets_rich_text_read_enabled' => false, + 'responsive_web_edit_tweet_api_enabled' => false, + 'responsive_web_enhance_cards_enabled' => false, + 'responsive_web_graphql_exclude_directive_enabled' => true, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false, + 'responsive_web_graphql_timeline_navigation_enabled' => false, + 'responsive_web_media_download_video_enabled' => false, + 'responsive_web_text_conversations_enabled' => false, + 'responsive_web_twitter_article_tweet_consumption_enabled' => false, + 'responsive_web_twitter_blue_verified_badge_is_enabled' => true, + 'rweb_lists_timeline_redesign_enabled' => true, + 'spaces_2022_h2_clipping' => true, + 'spaces_2022_h2_spaces_communities' => true, + 'standardized_nudges_misinfo' => false, + 'subscriptions_verification_info_enabled' => true, + 'subscriptions_verification_info_reason_enabled' => true, + 'subscriptions_verification_info_verified_since_enabled' => true, + 'super_follow_badge_privacy_enabled' => false, + 'super_follow_exclusive_tweet_notifications_enabled' => false, + 'super_follow_tweet_api_enabled' => false, + 'super_follow_user_api_enabled' => false, + 'tweet_awards_web_tipping_enabled' => false, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false, + 'tweetypie_unmention_optimization_enabled' => false, + 'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false, + 'verified_phone_label_enabled' => false, + 'vibe_api_enabled' => false, + 'view_counts_everywhere_api_enabled' => false + ]; + $variables = [ + 'screenName' => $screenName, + 'listSlug' => $listSlug + ]; + + $url = sprintf( + 'https://twitter.com/i/api/graphql/-kmqNvm5Y-cVrfvBy6docg/ListBySlug?variables=%s&features=%s', + urlencode(json_encode($variables)), + urlencode(json_encode($features)) + ); + + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + if (isset($response->errors)) { + // Grab the first error message + throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message)); + } + $listInfo = $response->data->user_by_screen_name->list; + $this->data[$screenName . '-' . $listSlug] = $listInfo; + + $this->cache->setScope('twitter'); + $this->cache->setKey(['cache']); + $this->cache->saveData($this->data); + return $listInfo; + } + + private function fetchListTimeline($listId) + { + $features = [ + 'android_graphql_skip_api_media_color_palette' => false, + 'blue_business_profile_image_shape_enabled' => false, + 'creator_subscriptions_subscription_count_enabled' => false, + 'creator_subscriptions_tweet_preview_api_enabled' => true, + 'freedom_of_speech_not_reach_fetch_enabled' => false, + 'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => false, + 'hidden_profile_likes_enabled' => false, + 'highlights_tweets_tab_ui_enabled' => false, + 'interactive_text_enabled' => false, + 'longform_notetweets_consumption_enabled' => true, + 'longform_notetweets_inline_media_enabled' => false, + 'longform_notetweets_richtext_consumption_enabled' => true, + 'longform_notetweets_rich_text_read_enabled' => false, + 'responsive_web_edit_tweet_api_enabled' => false, + 'responsive_web_enhance_cards_enabled' => false, + 'responsive_web_graphql_exclude_directive_enabled' => true, + 'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false, + 'responsive_web_graphql_timeline_navigation_enabled' => false, + 'responsive_web_media_download_video_enabled' => false, + 'responsive_web_text_conversations_enabled' => false, + 'responsive_web_twitter_article_tweet_consumption_enabled' => false, + 'responsive_web_twitter_blue_verified_badge_is_enabled' => true, + 'rweb_lists_timeline_redesign_enabled' => true, + 'spaces_2022_h2_clipping' => true, + 'spaces_2022_h2_spaces_communities' => true, + 'standardized_nudges_misinfo' => false, + 'subscriptions_verification_info_enabled' => true, + 'subscriptions_verification_info_reason_enabled' => true, + 'subscriptions_verification_info_verified_since_enabled' => true, + 'super_follow_badge_privacy_enabled' => false, + 'super_follow_exclusive_tweet_notifications_enabled' => false, + 'super_follow_tweet_api_enabled' => false, + 'super_follow_user_api_enabled' => false, + 'tweet_awards_web_tipping_enabled' => false, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false, + 'tweetypie_unmention_optimization_enabled' => false, + 'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => false, + 'verified_phone_label_enabled' => false, + 'vibe_api_enabled' => false, + 'view_counts_everywhere_api_enabled' => false + ]; + $variables = [ + 'rest_id' => $listId, + 'count' => 20 + ]; + + $url = sprintf( + 'https://twitter.com/i/api/graphql/BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline?variables=%s&features=%s', + urlencode(json_encode($variables)), + urlencode(json_encode($features)) + ); + $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + return $response; + } + private function createHttpHeaders(): array { $headers = [ From 977c0db38222e22b364578ba1d78e800445e44f9 Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Wed, 26 Jul 2023 19:41:29 +0200 Subject: [PATCH 039/716] [CssSelectorBridge] Improvements (#3537) (#3573) * [CssSelectorBridge] Improvements (#3537) * Improve parameter documentation / add tooltips * Allow extracting content from home page instead of article page * Keep titles from home page when every page <title> is the same * [CssSelectorBridge] Code linting * [CssSelectorBridge] Code linting (2) * [CssSelectorBridge] Code linting (3) --- bridges/CssSelectorBridge.php | 103 ++++++++++++++++++++++++++-------- 1 file changed, 80 insertions(+), 23 deletions(-) diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index ae135113..2d7489de 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -15,23 +15,40 @@ class CssSelectorBridge extends BridgeAbstract ], 'url_selector' => [ 'name' => 'Selector for article links or their parent elements', + 'title' => <<<EOT + This bridge works using CSS selectors, e.g. "a.article" will match all <a class="article" + href="URL">TITLE</a> on home page, each one being treated as a feed item. + Instead of just a link you can selet one of its parent element. Everything inside that + element becomes feed item content, e.g. image and summary present on home page. + When doing so, the first link inside the selected element becomes feed item URL/Title. + EOT, 'exampleValue' => 'a.article', 'required' => true ], 'url_pattern' => [ 'name' => '[Optional] Pattern for site URLs to keep in feed', - 'exampleValue' => 'https://example.com/article/.*', + 'title' => 'Optionally filter items by applying a regular expression on their URL', + 'exampleValue' => '/blog/article/.*', ], 'content_selector' => [ - 'name' => '[Optional] Selector to extract each article content', + 'name' => '[Optional] Selector to expand each article content', + 'title' => <<<EOT + When specified, the bridge will fetch each article from its URL + and extract content using the provided selector (Slower!) + EOT, 'exampleValue' => 'article.content', ], 'content_cleanup' => [ 'name' => '[Optional] Content cleanup: List of items to remove', + 'title' => 'Selector for unnecessary elements to remove inside article contents.', 'exampleValue' => 'div.ads, div.comments', ], 'title_cleanup' => [ 'name' => '[Optional] Text to remove from expanded article title', + 'title' => <<<EOT + When fetching each article page, feed item title comes from page title. + Specify here some text from page title that need to be removed, e.g. " | BlogName". + EOT, 'exampleValue' => ' | BlogName', ], 'limit' => self::LIMIT @@ -69,7 +86,7 @@ class CssSelectorBridge extends BridgeAbstract $html = defaultLinkTo(getSimpleHTMLDOM($url), $url); $this->feedName = $this->getPageTitle($html, $title_cleanup); - $items = $this->htmlFindLinks($html, $url_selector, $url_pattern, $limit); + $items = $this->htmlFindEntries($html, $url_selector, $url_pattern, $limit, $content_cleanup); if (empty($content_selector)) { $this->items = $items; @@ -79,7 +96,8 @@ class CssSelectorBridge extends BridgeAbstract $item['uri'], $content_selector, $content_cleanup, - $title_cleanup + $title_cleanup, + $item['title'] ); } } @@ -127,30 +145,71 @@ class CssSelectorBridge extends BridgeAbstract } /** - * Retrieve first N links from webpage URL or DOM satisfying the specified criteria - * @param string|object $page URL or DOM to retrieve links from + * Remove all elements from HTML content matching cleanup selector + * @param string|object $content HTML content as HTML object or string + * @return string|object Cleaned content (same type as input) + */ + protected function cleanArticleContent($content, $cleanup_selector) + { + $string_convert = false; + if (is_string($content)) { + $string_convert = true; + $content = str_get_html($content); + } + + if (!empty($cleanup_selector)) { + foreach ($content->find($cleanup_selector) as $item_to_clean) { + $item_to_clean->outertext = ''; + } + } + + if ($string_convert) { + $content = $content->outertext; + } + return $content; + } + + /** + * Retrieve first N link+title+truncated-content from webpage URL or DOM satisfying the specified criteria + * @param string|object $page URL or DOM to retrieve feed items from * @param string $url_selector DOM selector for matching links or their parent element * @param string $url_pattern Optional filter to keep only links matching the pattern * @param int $limit Optional maximum amount of URLs to return - * @return array of minimal feed items {'uri': entry_url, 'title', entry_title} + * @param string $content_cleanup Optional selector for removing elements, e.g. "div.ads, div.comments" + * @return array of items {'uri': entry_url, 'title': entry_title, ['content': when present in DOM] } */ - protected function htmlFindLinks($page, $url_selector, $url_pattern = '', $limit = 0) + protected function htmlFindEntries($page, $url_selector, $url_pattern = '', $limit = 0, $content_cleanup = null) { + if (is_string($page)) { + $page = getSimpleHTMLDOM($page); + } + $links = $page->find($url_selector); if (empty($links)) { returnClientError('No results for URL selector'); } - $link_to_title = []; + $link_to_item = []; foreach ($links as $link) { + $item = []; + if ($link->innertext != $link->plaintext) { + $item['content'] = $link->innertext; + } if ($link->tag != 'a') { $link = $link->find('a', 0); } - $link_to_title[$link->href] = $link->plaintext; + $item['uri'] = $link->href; + $item['title'] = $link->plaintext; + if (isset($item['content'])) { + $item['content'] = convertLazyLoading($item['content']); + $item['content'] = defaultLinkTo($item['content'], $item['uri']); + $item['content'] = $this->cleanArticleContent($item['content'], $content_cleanup); + } + $link_to_item[$link->href] = $item; } - $links = $this->filterUrlList(array_keys($link_to_title), $url_pattern, $limit); + $links = $this->filterUrlList(array_keys($link_to_item), $url_pattern, $limit); if (empty($links)) { returnClientError('No results for URL pattern'); @@ -158,10 +217,7 @@ class CssSelectorBridge extends BridgeAbstract $items = []; foreach ($links as $link) { - $item = []; - $item['uri'] = $link; - $item['title'] = $link_to_title[$link]; - $items[] = $item; + $items[] = $link_to_item[$link]; } return $items; @@ -173,9 +229,10 @@ class CssSelectorBridge extends BridgeAbstract * @param string $content_selector HTML selector for extracting content, e.g. "article.content" * @param string $content_cleanup Optional selector for removing elements, e.g. "div.ads, div.comments" * @param string $title_cleanup Optional string to remove from article title, e.g. " | BlogName" + * @param string $title_default Optional title to use when could not extract title reliably * @return array Entry data: uri, title, content */ - protected function expandEntryWithSelector($entry_url, $content_selector, $content_cleanup = null, $title_cleanup = null) + protected function expandEntryWithSelector($entry_url, $content_selector, $content_cleanup = null, $title_cleanup = null, $title_default = null) { if (empty($content_selector)) { returnClientError('Please specify a content selector'); @@ -190,18 +247,18 @@ class CssSelectorBridge extends BridgeAbstract returnClientError('Could not find content selector at URL: ' . $entry_url); } - if (!empty($content_cleanup)) { - foreach ($article_content->find($content_cleanup) as $item_to_clean) { - $item_to_clean->outertext = ''; - } - } - $article_content = convertLazyLoading($article_content); $article_content = defaultLinkTo($article_content, $entry_url); + $article_content = $this->cleanArticleContent($article_content, $content_cleanup); + + $article_title = $this->getPageTitle($entry_html, $title_cleanup); + if (!empty($title_default) && (empty($article_title) || $article_title === $this->feedName)) { + $article_title = $title_default; + } $item = []; $item['uri'] = $entry_url; - $item['title'] = $this->getPageTitle($entry_html, $title_cleanup); + $item['title'] = $article_title; $item['content'] = $article_content; return $item; } From 235c084820cab9a75c88e46e5feda046cc63178f Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Wed, 26 Jul 2023 20:41:48 +0200 Subject: [PATCH 040/716] [DilbertBridge] Remove bridge (#3574) dilbert.com has closed down. --- bridges/DilbertBridge.php | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 bridges/DilbertBridge.php diff --git a/bridges/DilbertBridge.php b/bridges/DilbertBridge.php deleted file mode 100644 index cd509ea4..00000000 --- a/bridges/DilbertBridge.php +++ /dev/null @@ -1,36 +0,0 @@ -<?php - -class DilbertBridge extends BridgeAbstract -{ - const MAINTAINER = 'kranack'; - const NAME = 'Dilbert Daily Strip'; - const URI = 'https://dilbert.com'; - const CACHE_TIMEOUT = 21600; // 6h - const DESCRIPTION = 'The Unofficial Dilbert Daily Comic Strip'; - - public function collectData() - { - $html = getSimpleHTMLDOM(self::URI); - - foreach ($html->find('section.comic-item') as $element) { - $img = $element->find('img', 0); - $link = $element->find('a', 0); - $comic = $img->src; - $title = $img->alt; - $url = $link->href; - $date = substr(strrchr($url, '/'), 1); - if (empty($title)) { - $title = 'Dilbert Comic Strip on ' . $date; - } - $date = strtotime($date); - - $item = []; - $item['uri'] = $url; - $item['title'] = $title; - $item['author'] = 'Scott Adams'; - $item['timestamp'] = $date; - $item['content'] = '<img src="' . $comic . '" alt="' . $img->alt . '" />'; - $this->items[] = $item; - } - } -} From bf4ea127196a318b3e0440d0e8be9d96fc186b13 Mon Sep 17 00:00:00 2001 From: Korytov Pavel <thexcloud@gmail.com> Date: Wed, 26 Jul 2023 22:47:47 +0300 Subject: [PATCH 041/716] [ScientificAmerican] Fix bridge (#3575) --- bridges/ScientificAmericanBridge.php | 68 ++++++++++++---------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/bridges/ScientificAmericanBridge.php b/bridges/ScientificAmericanBridge.php index 88635f58..d575bf94 100644 --- a/bridges/ScientificAmericanBridge.php +++ b/bridges/ScientificAmericanBridge.php @@ -25,10 +25,11 @@ class ScientificAmericanBridge extends FeedExpander ]; const FEED = 'http://rss.sciam.com/ScientificAmerican-Global'; - const ISSUES = 'https://www.scientificamerican.com/store/archive/?magazineFilterID=all'; + const ISSUES = 'https://www.scientificamerican.com/archive/issues/2020s/'; public function collectData() { + $this->collectIssues(); $items = [ ...$this->collectFeed(), ...$this->collectIssues() @@ -49,7 +50,7 @@ class ScientificAmericanBridge extends FeedExpander if ($this->getInput('addContents') == 1) { usort($this->items, function ($item1, $item2) { - return $item1['timestamp'] < $item2['timestamp']; + return $item1['timestamp'] - $item2['timestamp']; }); } } @@ -65,8 +66,8 @@ class ScientificAmericanBridge extends FeedExpander private function collectIssues() { $html = getSimpleHTMLDOMCached(self::ISSUES); - $issues_root = $html->find('div.store-listing-group', 0); - $issues = $issues_root->find('div.store-listing-group__item'); + $content = $html->getElementById('content')->children(3); + $issues = $content->children(); $issues_count = min( (int)$this->getInput('parseIssues'), count($issues) @@ -74,7 +75,7 @@ class ScientificAmericanBridge extends FeedExpander $items = []; for ($i = 0; $i < $issues_count; $i++) { - $a = $issues[$i]->find('a.store-listing__cta', 0); + $a = $issues[$i]->find('a', 0); $link = 'https://scientificamerican.com' . $a->getAttribute('href'); array_push($items, ...$this->parseIssue($link)); } @@ -86,51 +87,42 @@ class ScientificAmericanBridge extends FeedExpander $items = []; $html = getSimpleHTMLDOMCached($issue_link); - $features = $html->find('section[data-issue-column="Features"]', 0); + $features = $html->find('[class^=Detail_issue__article__previews__featured]', 0); if ($features != null) { - $articles = $features->find('article'); + $articles = $features->find('div', 0)->children(); foreach ($articles as $article) { - $items[] = $this->parseIssueItem($article); + $h4 = $article->find('h4', 0); + $a = $h4->find('a', 0); + $link = 'https://scientificamerican.com' . $a->getAttribute('href'); + $title = $a->plaintext; + $items[] = [ + 'uri' => $link, + 'title' => $title, + 'uid' => $link, + 'content' => '' + ]; } } - $departments = $html->find('section[data-issue-column="Departments"]', 0); + $departments = $html->find('[class^=Detail_issue__article__previews__departments]', 0); if ($departments != null) { - $lis = $departments->find('ul', 0)->find('li'); - foreach ($lis as $li) { - $items[] = $this->parseIssueItem($li); + $headers = $departments->find('[class*=Listing_article__listing__title]'); + foreach ($headers as $header) { + $a = $header->find('a', 0); + $link = 'https://scientificamerican.com' . $a->getAttribute('href'); + $title = $a->plaintext; + $items[] = [ + 'uri' => $link, + 'title' => $title, + 'uid' => $link, + 'content' => '' + ]; } } return $items; } - private function parseIssueItem($article) - { - $title = $article->getAttribute('data-article-title'); - $a = $article->find('a', 0); - $link = null; - if ($a != null) { - $link = $a->href; - } else { - [$kind, $v] = explode('-', $article->getAttribute('id'), 2); - $link = 'https://scientificamerican.com/' . $kind . '/' . $v; - } - $content = ''; - - $desc = $article->find('p.listing-wide__inner__desc', 0); - if ($desc != null) { - $content = $desc->plaintext; - } - - return [ - 'uri' => $link, - 'title' => $title, - 'uid' => $link, - 'content' => $content - ]; - } - private function updateItem($item) { $html = getSimpleHTMLDOMCached($item['uri']); From f5f76f111b60b723791be05c1cbc521d1cad3aab Mon Sep 17 00:00:00 2001 From: Korytov Pavel <thexcloud@gmail.com> Date: Wed, 26 Jul 2023 23:59:49 +0300 Subject: [PATCH 042/716] [TldrTechBridge] Add Web Dev and Founders sections (#3576) --- bridges/TldrTechBridge.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bridges/TldrTechBridge.php b/bridges/TldrTechBridge.php index ba7ebf6e..7d8febe1 100644 --- a/bridges/TldrTechBridge.php +++ b/bridges/TldrTechBridge.php @@ -23,7 +23,9 @@ class TldrTechBridge extends BridgeAbstract 'values' => [ 'Tech' => 'tech', 'Crypto' => 'crypto', - 'AI' => 'ai' + 'AI' => 'ai', + 'Web Dev' => 'engineering', + 'Founders' => 'founders' ], 'defaultValue' => 'tech' ] From 11ce8b5dcd6f9b84249c974f1403a45b5b99d8f5 Mon Sep 17 00:00:00 2001 From: Aaron F <mail@aaron-fischer.net> Date: Thu, 27 Jul 2023 23:54:17 +0200 Subject: [PATCH 043/716] CVEDetails got a new HTML layout. (#3577) This fixes the parser for CVEDetails. --- bridges/CVEDetailsBridge.php | 41 ++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/bridges/CVEDetailsBridge.php b/bridges/CVEDetailsBridge.php index 38b37bb7..5334c170 100644 --- a/bridges/CVEDetailsBridge.php +++ b/bridges/CVEDetailsBridge.php @@ -61,7 +61,7 @@ class CVEDetailsBridge extends BridgeAbstract $html = getSimpleHTMLDOM($this->buildUrl()); $this->html = defaultLinkTo($html, self::URI); - $vendor = $html->find('#contentdiv > h1 > a', 0); + $vendor = $html->find('#contentdiv h1 > a', 0); if ($vendor == null) { returnServerError('Invalid Vendor ID ' . $this->getInput('vendor_id') . @@ -70,7 +70,7 @@ class CVEDetailsBridge extends BridgeAbstract } $this->vendor = $vendor->innertext; - $product = $html->find('#contentdiv > h1 > a', 1); + $product = $html->find('#contentdiv h1 > a', 1); if ($product != null) { $this->product = $product->innertext; } @@ -102,38 +102,43 @@ class CVEDetailsBridge extends BridgeAbstract $this->fetchContent(); } - foreach ($this->html->find('#vulnslisttable .srrowns') as $i => $tr) { + foreach ($this->html->find('#searchresults > .row') as $i => $tr) { // There are some optional vulnerability types, which will be // added to the categories as well as the CWE number -- which is // always given. $categories = [$this->vendor]; $enclosures = []; - $cwe = $tr->find('td', 2)->find('a', 0); - if ($cwe != null) { - $cwe = $cwe->innertext; - $categories[] = 'CWE-' . $cwe; - $enclosures[] = 'https://cwe.mitre.org/data/definitions/' . $cwe . '.html'; - } - $c = $tr->find('td', 4)->innertext; - if (trim($c) != '') { - $categories[] = $c; + $detailLink = $tr->find('.cveheader > h3 > a', 0); + $detailHtml = getSimpleHTMLDOM($detailLink->href); + + $div = $detailHtml->find('.cvedetailssummary', 0); + + // The CVE number itself + $title = $div->find('h1 > a', 0)->innertext; + $content = $div->find('.ssc-paragraph', 0)->innertext; + $cweList = $detailHtml->find('h2', 2)->next_sibling(); + foreach ($cweList->find('li') as $li) { + $cweWithDescription = $li->find('a', 0)->innertext; + preg_match('/CWE-(\d+)/', $cweWithDescription, $cwe); + if (count($cwe) > 1) { + $categories[] = 'CWE-' . $cwe[1]; + $enclosures[] = 'https://cwe.mitre.org/data/definitions/' . $cwe[1] . '.html'; + } } + if ($this->product != '') { $categories[] = $this->product; } - // The CVE number itself - $title = $tr->find('td', 1)->find('a', 0)->innertext; - $this->items[] = [ - 'uri' => $tr->find('td', 1)->find('a', 0)->href, + 'uri' => 'https://cvedetails.com/' . $detailHtml->find('h1 > a', 0)->href, 'title' => $title, 'timestamp' => $tr->find('td', 5)->innertext, - 'content' => $tr->next_sibling()->innertext, + 'content' => $content, 'categories' => $categories, 'enclosures' => $enclosures, - 'uid' => $tr->find('td', 1)->find('a', 0)->innertext, + 'uid' => $title, ]; // We only want to fetch the latest 10 CVEs From 701fe3cfeda0d14f1f49afe2bd0f89296214459e Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 29 Jul 2023 00:14:30 +0200 Subject: [PATCH 044/716] fix: various small fixes (#3578) --- bridges/AmazonPriceTrackerBridge.php | 3 +-- bridges/EBayBridge.php | 21 +++++++++++++++----- bridges/FB2Bridge.php | 6 +++++- bridges/FilterBridge.php | 13 +++++++++---- bridges/GithubIssueBridge.php | 3 ++- bridges/ImgsedBridge.php | 5 ++++- bridges/ReutersBridge.php | 28 +++++++++++++-------------- bridges/RoadAndTrackBridge.php | 7 ++++++- bridges/TikTokBridge.php | 5 +++-- bridges/VkBridge.php | 10 +++++++--- bridges/YouTubeCommunityTabBridge.php | 16 +++++++++++---- lib/Logger.php | 2 ++ lib/TwitterClient.php | 7 ++++++- 13 files changed, 87 insertions(+), 39 deletions(-) diff --git a/bridges/AmazonPriceTrackerBridge.php b/bridges/AmazonPriceTrackerBridge.php index 6de451f1..b07bdb7c 100644 --- a/bridges/AmazonPriceTrackerBridge.php +++ b/bridges/AmazonPriceTrackerBridge.php @@ -125,14 +125,13 @@ class AmazonPriceTrackerBridge extends BridgeAbstract */ private function getImage($html) { + $image = 'https://placekitten.com/200/300'; $imageSrc = $html->find('#main-image-container img', 0); - if ($imageSrc) { $hiresImage = $imageSrc->getAttribute('data-old-hires'); $dynamicImageAttribute = $imageSrc->getAttribute('data-a-dynamic-image'); $image = $hiresImage ?: $this->parseDynamicImage($dynamicImageAttribute); } - $image = $image ?: 'https://placekitten.com/200/300'; return <<<EOT <img width="300" style="max-width:300;max-height:300" src="$image" alt="{$this->title}" /> diff --git a/bridges/EBayBridge.php b/bridges/EBayBridge.php index a867a179..66fad10c 100644 --- a/bridges/EBayBridge.php +++ b/bridges/EBayBridge.php @@ -66,16 +66,27 @@ class EBayBridge extends BridgeAbstract $new_listing_label->remove(); } - $item['title'] = $listing->find('.s-item__title', 0)->plaintext; + $listingTitle = $listing->find('.s-item__title', 0); + if ($listingTitle) { + $item['title'] = $listingTitle->plaintext; + } $subtitle = implode('', $listing->find('.s-item__subtitle')); - $item['uri'] = $listing->find('.s-item__link', 0)->href; + $listingUrl = $listing->find('.s-item__link', 0); + if ($listingUrl) { + $item['uri'] = $listingUrl->href; + } else { + $item['uri'] = null; + } - preg_match('/.*\/itm\/(\d+).*/i', $item['uri'], $matches); - $item['uid'] = $matches[1]; + if (preg_match('/.*\/itm\/(\d+).*/i', $item['uri'], $matches)) { + $item['uid'] = $matches[1]; + } + + $priceDom = $listing->find('.s-item__details > .s-item__detail > .s-item__price', 0); + $price = $priceDom->plaintext ?? 'N/A'; - $price = $listing->find('.s-item__details > .s-item__detail > .s-item__price', 0)->plaintext; $shippingFree = $listing->find('.s-item__details > .s-item__detail > .s-item__freeXDays', 0)->plaintext ?? ''; $localDelivery = $listing->find('.s-item__details > .s-item__detail > .s-item__localDelivery', 0)->plaintext ?? ''; $logisticsCost = $listing->find('.s-item__details > .s-item__detail > .s-item__logisticsCost', 0)->plaintext ?? ''; diff --git a/bridges/FB2Bridge.php b/bridges/FB2Bridge.php index 19030dd2..141ea59b 100644 --- a/bridges/FB2Bridge.php +++ b/bridges/FB2Bridge.php @@ -304,7 +304,11 @@ EOD $regex = '/"pageID":"([0-9]*)"/'; preg_match($regex, $pageContent, $matches); - return ['userId' => $matches[1], 'username' => $username]; + $arr = [ + 'userId' => $matches[1] ?? null, + 'username' => $username, + ]; + return $arr; } public function getName() diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index 61ce6d78..992fe0c3 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -80,10 +80,15 @@ class FilterBridge extends FeedExpander // Generate title from first 50 characters of content? if ($this->getInput('title_from_content') && array_key_exists('content', $item)) { $content = str_get_html($item['content']); - $pos = strpos($item['content'], ' ', 50); - $item['title'] = substr($content->plaintext, 0, $pos); - if (strlen($content->plaintext) >= $pos) { - $item['title'] .= '...'; + $plaintext = $content->plaintext; + if (mb_strlen($plaintext) < 51) { + $item['title'] = $plaintext; + } else { + $pos = strpos($item['content'], ' ', 50); + $item['title'] = substr($plaintext, 0, $pos); + if (strlen($plaintext) >= $pos) { + $item['title'] .= '...'; + } } } diff --git a/bridges/GithubIssueBridge.php b/bridges/GithubIssueBridge.php index a75aa252..e4e995e3 100644 --- a/bridges/GithubIssueBridge.php +++ b/bridges/GithubIssueBridge.php @@ -137,7 +137,8 @@ class GithubIssueBridge extends BridgeAbstract { $uri = $this->buildGitHubIssueCommentUri($issueNbr, $comment->id); - $author = $comment->find('.author', 0)->plaintext; + $authorDom = $comment->find('.author', 0); + $author = $authorDom->plaintext ?? null; $header = $comment->find('.timeline-comment-header > h3', 0); $title .= ' / ' . ($header ? $header->plaintext : 'Activity'); diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index 6c49facb..1555c578 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -212,9 +212,12 @@ HTML, private function parseDate($content) { $date = date_create(); - $relativeDate = date_interval_create_from_date_string(str_replace(' ago', '', $content)); + $dateString = str_replace(' ago', '', $content); + $relativeDate = date_interval_create_from_date_string($dateString); if ($relativeDate) { date_sub($date, $relativeDate); + } else { + Logger::info(sprintf('Unable to parse date string: %s', $dateString)); } return date_format($date, 'r'); } diff --git a/bridges/ReutersBridge.php b/bridges/ReutersBridge.php index 63c838c2..2acadfc3 100644 --- a/bridges/ReutersBridge.php +++ b/bridges/ReutersBridge.php @@ -143,18 +143,6 @@ class ReutersBridge extends BridgeAbstract 'wire' ]; - /** - * Performs an HTTP request to the Reuters API and returns decoded JSON - * in the form of an associative array - * @param string $feed_uri Full API URL to fetch data - * @return array - */ - private function getJson($uri) - { - $returned_data = getContents($uri); - return json_decode($returned_data, true); - } - /** * Takes in data from Reuters Wire API and * creates structured data in the form of a list @@ -295,8 +283,19 @@ class ReutersBridge extends BridgeAbstract { // This will make another request to API to get full detail of article and author's name. $url = $this->getAPIURL($feed_uri, 'article', $is_article_uid); - $rawData = $this->getJson($url); + try { + $json = getContents($url); + $rawData = Json::decode($json); + } catch (\JsonException $e) { + return [ + 'content' => '', + 'author' => '', + 'category' => '', + 'images' => '', + 'published_at' => '' + ]; + } $article_content = ''; $authorlist = ''; $category = []; @@ -494,7 +493,8 @@ EOD; { $endpoint = $this->getSectionEndpoint(); $url = $this->getAPIURL($endpoint, 'section'); - $data = $this->getJson($url); + $json = getContents($url); + $data = Json::decode($json); $stories = []; $section_name = ''; diff --git a/bridges/RoadAndTrackBridge.php b/bridges/RoadAndTrackBridge.php index 31bd7ec8..c236036c 100644 --- a/bridges/RoadAndTrackBridge.php +++ b/bridges/RoadAndTrackBridge.php @@ -50,7 +50,12 @@ class RoadAndTrackBridge extends BridgeAbstract } $item['author'] = $article->find('.byline-name', 0)->innertext ?? ''; - $item['timestamp'] = strtotime($article->find('.content-info-date', 0)->getAttribute('datetime')); + + $contentInfoDate = $article->find('.content-info-date', 0); + if ($contentInfoDate) { + $datetime = $contentInfoDate->getAttribute('datetime'); + $item['timestamp'] = strtotime($datetime); + } $content = $article->find('.content-container', 0); if ($content->find('.content-rail', 0) !== null) { diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php index e7cac825..556e5ffc 100644 --- a/bridges/TikTokBridge.php +++ b/bridges/TikTokBridge.php @@ -33,8 +33,9 @@ class TikTokBridge extends BridgeAbstract $title = $html->find('h1', 0)->plaintext ?? self::NAME; $this->feedName = htmlspecialchars_decode($title); - $SIGI_STATE_RAW = $html->find('script[id=SIGI_STATE]', 0)->innertext; - $SIGI_STATE = json_decode($SIGI_STATE_RAW); + $var = $html->find('script[id=SIGI_STATE]', 0); + $SIGI_STATE_RAW = $var->innertext; + $SIGI_STATE = Json::decode($SIGI_STATE_RAW, false); foreach ($SIGI_STATE->ItemModule as $key => $value) { $item = []; diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index c5a9d4cc..967734ef 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -490,18 +490,22 @@ class VkBridge extends BridgeAbstract private function getContents() { - $header = ['Accept-language: en', 'Cookie: remixlang=3']; + $httpHeaders = [ + 'Accept-language: en', + 'Cookie: remixlang=3', + ]; $redirects = 0; $uri = $this->getURI(); while ($redirects < 2) { - $response = getContents($uri, $header, [CURLOPT_FOLLOWLOCATION => false], true); + $response = getContents($uri, $httpHeaders, [CURLOPT_FOLLOWLOCATION => false], true); if (in_array($response['code'], [200, 304])) { return $response['content']; } - $uri = urljoin(self::URI, $response['header']['location'][0]); + $headers = $response['headers']; + $uri = urljoin(self::URI, $headers['location'][0]); if (str_contains($uri, '/429.html')) { returnServerError('VK responded "Too many requests"'); diff --git a/bridges/YouTubeCommunityTabBridge.php b/bridges/YouTubeCommunityTabBridge.php index 32502f61..20822828 100644 --- a/bridges/YouTubeCommunityTabBridge.php +++ b/bridges/YouTubeCommunityTabBridge.php @@ -78,19 +78,27 @@ class YouTubeCommunityTabBridge extends BridgeAbstract returnServerError('Channel does not have a community tab'); } - foreach ($this->getCommunityPosts($json) as $key => $post) { + $posts = $this->getCommunityPosts($json); + foreach ($posts as $key => $post) { $this->itemTitle = ''; if (!isset($post->backstagePostThreadRenderer)) { continue; } - $details = $post->backstagePostThreadRenderer->post->backstagePostRenderer; + if (isset($post->backstagePostThreadRenderer->post->backstagePostRenderer)) { + $details = $post->backstagePostThreadRenderer->post->backstagePostRenderer; + } elseif (isset($post->backstagePostThreadRenderer->post->sharedPostRenderer)) { + // todo: properly extract data from this shared post + $details = $post->backstagePostThreadRenderer->post->sharedPostRenderer; + } else { + continue; + } $item = []; $item['uri'] = self::URI . '/post/' . $details->postId; - $item['author'] = $details->authorText->runs[0]->text; - $item['content'] = ''; + $item['author'] = $details->authorText->runs[0]->text ?? null; + $item['content'] = $item['uri']; if (isset($details->contentText->runs)) { $text = $this->getText($details->contentText->runs); diff --git a/lib/Logger.php b/lib/Logger.php index 91b8e3c1..5423f62c 100644 --- a/lib/Logger.php +++ b/lib/Logger.php @@ -55,6 +55,8 @@ final class Logger 'Unable to find channel. The channel is non-existing or non-public', // fb 'This group is not public! RSS-Bridge only supports public groups!', + 'You must be logged in to view this page', + 'Unable to get the page id. You should consider getting the ID by hand', // tiktok 404 'https://www.tiktok.com/@', ]; diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 92c797d6..0c6b9535 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -155,7 +155,7 @@ class TwitterClient throw $e; } } - } else if ($operation == 'By list ID') { + } elseif ($operation === 'By list ID') { $id = $query['listId']; } else { throw new \Exception('Unknown operation to make list tweets'); @@ -348,6 +348,11 @@ class TwitterClient // Grab the first error message throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message)); } + if (!isset($response->data->user_by_screen_name->list)) { + throw new \Exception( + sprintf('Unable to find list in twitter response for %s, %s', $screenName, $listSlug) + ); + } $listInfo = $response->data->user_by_screen_name->list; $this->data[$screenName . '-' . $listSlug] = $listInfo; From 3a57fc800bfb399ee708794c31255566700369da Mon Sep 17 00:00:00 2001 From: mrtnvgr <48406064+mrtnvgr@users.noreply.github.com> Date: Sun, 30 Jul 2023 11:46:16 +0700 Subject: [PATCH 045/716] DoujinStyleBridge: Update html tags (#3581) --- bridges/DoujinStyleBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/DoujinStyleBridge.php b/bridges/DoujinStyleBridge.php index 84469739..0df96280 100644 --- a/bridges/DoujinStyleBridge.php +++ b/bridges/DoujinStyleBridge.php @@ -72,7 +72,7 @@ class DoujinStyleBridge extends BridgeAbstract $item['content'] = "<img src='$cover'/>"; $keys = []; - foreach ($content->find('.pageWrap .pageSpan') as $key) { + foreach ($content->find('.pageWrap .pageSpan1') as $key) { $keys[] = $key->plaintext; } From 93eecdf79f83e9c9d19671b0e65e14d6287b4da4 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Sun, 30 Jul 2023 23:26:59 +0200 Subject: [PATCH 046/716] [core] fix new bridge PRs not generating html preview artifacts (#3583) * [core] replace everything except bridge name to get a valid whitelist.txt * [core] do not use hard code repository name to improve working with forks * [core] trim bridge names from whitelist.txt to reduce chance of failure --- .github/workflows/prhtmlgenerator.yml | 6 +++--- lib/Configuration.php | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/prhtmlgenerator.yml b/.github/workflows/prhtmlgenerator.yml index cacb6642..ce82aef1 100644 --- a/.github/workflows/prhtmlgenerator.yml +++ b/.github/workflows/prhtmlgenerator.yml @@ -18,11 +18,11 @@ jobs: - name: Check out rss-bridge run: | PR=${{github.event.number}}; - wget -O requirements.txt https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester-requirements.txt; - wget https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester.py; + wget -O requirements.txt https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester-requirements.txt; + wget https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester.py; wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch; touch DEBUG; - cat $PR.patch | grep " bridges/.*\.php" | sed "s= bridges/\(.*\)Bridge.php.*=\1=g" | sort | uniq > whitelist.txt + cat $PR.patch | grep "\bbridges/.*Bridge\.php\b" | sed "s=.*\bbridges/\(.*\)Bridge\.php\b.*=\1=g" | sort | uniq > whitelist.txt - name: Start Docker - Current run: | docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3000:80 ghcr.io/rss-bridge/rss-bridge:latest diff --git a/lib/Configuration.php b/lib/Configuration.php index 57a7db7e..f5615009 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -113,7 +113,7 @@ final class Configuration if ($enabledBridges === '*') { self::setConfig('system', 'enabled_bridges', ['*']); } else { - self::setConfig('system', 'enabled_bridges', array_filter(explode("\n", $enabledBridges))); + self::setConfig('system', 'enabled_bridges', array_filter(array_map('trim', explode("\n", $enabledBridges)))); } } From f957eea3003a734acd8f9fa72ef13661cee277ed Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Mon, 31 Jul 2023 01:05:38 +0200 Subject: [PATCH 047/716] [FallGuysBridge] new bridge (#3584) --- bridges/FallGuysBridge.php | 134 +++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 bridges/FallGuysBridge.php diff --git a/bridges/FallGuysBridge.php b/bridges/FallGuysBridge.php new file mode 100644 index 00000000..dbb34792 --- /dev/null +++ b/bridges/FallGuysBridge.php @@ -0,0 +1,134 @@ +<?php + +class FallGuysBridge extends BridgeAbstract +{ + const MAINTAINER = 'User123698745'; + const NAME = 'Fall Guys'; + const BASE_URI = 'https://www.fallguys.com'; + const URI = self::BASE_URI . '/news'; + const CACHE_TIMEOUT = 600; // 10min + const DESCRIPTION = 'News from the Fall Guys website'; + const DEFAULT_LOCALE = 'en-US'; + const PARAMETERS = [ + [ + 'locale' => [ + 'name' => 'Language', + 'type' => 'list', + 'values' => [ + 'English' => 'en-US', + 'لعربية' => 'ar', + 'Deutsch' => 'de', + 'Español (Spain)' => 'es-ES', + 'Español (LA)' => 'es-MX', + 'Français' => 'fr', + 'Italiano' => 'it', + '日本語' => 'ja', + '한국어' => 'ko', + 'Polski' => 'pl', + 'Português (Brasil)' => 'pt-BR', + 'Русский' => 'ru', + 'Türkçe' => 'tr', + '简体中文' => 'zh-CN', + ], + 'defaultValue' => self::DEFAULT_LOCALE, + ] + ] + ]; + + public function collectData() + { + $html = getSimpleHTMLDOM(self::getURI()); + + $data = json_decode($html->find('#__NEXT_DATA__', 0)->innertext); + + foreach ($data->props->pageProps->newsList as $newsItem) { + $headerDescription = property_exists($newsItem->header, 'description') ? $newsItem->header->description : ''; + $headerImage = $newsItem->header->image->src; + + $contentImages = [$headerImage]; + + $content = <<<HTML + <p>{$headerDescription}</p> + <p><img src="{$headerImage}"></p> + HTML; + + foreach ($newsItem->content->items as $contentItem) { + if (property_exists($contentItem, 'articleCopy')) { + if (property_exists($contentItem->articleCopy, 'title')) { + $title = $contentItem->articleCopy->title; + + $content .= <<<HTML + <h2>{$title}</h2> + HTML; + } + + $text = $contentItem->articleCopy->copy; + + $content .= <<<HTML + <p>{$text}</p> + HTML; + } elseif (property_exists($contentItem, 'articleImage')) { + $image = $contentItem->articleImage->imageSrc; + + if ($image != $headerImage) { + $contentImages[] = $image; + + $content .= <<<HTML + <p><img src="{$image}"></p> + HTML; + } + } elseif (property_exists($contentItem, 'embeddedVideo')) { + $mediaOptions = $contentItem->embeddedVideo->mediaOptions; + $mainContentOptions = $contentItem->embeddedVideo->mainContentOptions; + + if (count($mediaOptions) == count($mainContentOptions)) { + for ($i = 0; $i < count($mediaOptions); $i++) { + if (property_exists($mediaOptions[$i], 'youtubeVideo')) { + $videoUrl = 'https://youtu.be/' . $mediaOptions[$i]->youtubeVideo->contentId; + $image = $mainContentOptions[$i]->image->src; + + $content .= '<p>'; + + if ($image != $headerImage) { + $contentImages[] = $image; + + $content .= <<<HTML + <a href="{$videoUrl}"><img src="{$image}"></a><br> + HTML; + } + + $content .= <<<HTML + <i>(Video: <a href="{$videoUrl}">{$videoUrl}</a>)</i> + HTML; + + $content .= '</p>'; + } + } + } + } + } + + $item = [ + 'uid' => $newsItem->_id, + 'uri' => self::getURI() . '/' . $newsItem->_slug, + 'title' => $newsItem->_title, + 'timestamp' => $newsItem->lastModified, + 'content' => $content, + 'enclosures' => $contentImages, + ]; + + $this->items[] = $item; + } + } + + public function getURI() + { + $locale = $this->getInput('locale') ?? self::DEFAULT_LOCALE; + return self::BASE_URI . '/' . $locale . '/news'; + } + + public function getIcon() + { + return self::BASE_URI . '/favicon.ico'; + } +} From f8fd05f08f114c0fa007075a1da6b780eb6a593f Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Mon, 31 Jul 2023 19:07:34 +0200 Subject: [PATCH 048/716] [CssSelectorBridge] Handling of missing links (#3585) When using parent element as URL selector: * If no <a> inside some elements, ignore them * If no <a> inside ALL elements, report an error Fixes #3573 #issuecomment-1656943318 --- bridges/CssSelectorBridge.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index 2d7489de..ce158758 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -198,6 +198,9 @@ class CssSelectorBridge extends BridgeAbstract } if ($link->tag != 'a') { $link = $link->find('a', 0); + if (is_null($link)) { + continue; + } } $item['uri'] = $link->href; $item['title'] = $link->plaintext; @@ -209,6 +212,10 @@ class CssSelectorBridge extends BridgeAbstract $link_to_item[$link->href] = $item; } + if (empty($link_to_item)) { + returnClientError('The provided URL selector matches some elements, but they do not contain links.'); + } + $links = $this->filterUrlList(array_keys($link_to_item), $url_pattern, $limit); if (empty($links)) { From 8b6eecea25c7ffbc9c55b6905f07588881200a90 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Mon, 31 Jul 2023 20:43:11 +0200 Subject: [PATCH 049/716] docs: add note about expensive operation (#3579) --- bridges/VkBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 967734ef..8c18f26a 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -84,7 +84,10 @@ class VkBridge extends BridgeAbstract foreach ($html->find('div.replies') as $comment_block) { $comment_block->outertext = ''; } - $html->load($html->save()); + + // expensive operation + $save = $html->save(); + $html->load($save); $pinned_post_item = null; $last_post_id = 0; From 7e4807530ec6199ac9754bce54f025a113edfbbe Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Mon, 31 Jul 2023 20:43:18 +0200 Subject: [PATCH 050/716] fix: various small fixes (#3580) --- bridges/AppleMusicBridge.php | 4 +++- bridges/AskfmBridge.php | 3 ++- bridges/FeedReducerBridge.php | 17 +++++++++-------- bridges/FourchanBridge.php | 3 ++- bridges/GettrBridge.php | 3 ++- bridges/TelegramBridge.php | 12 ++++++++++-- bridges/TikTokBridge.php | 3 +++ lib/BridgeAbstract.php | 4 ++++ 8 files changed, 35 insertions(+), 14 deletions(-) diff --git a/bridges/AppleMusicBridge.php b/bridges/AppleMusicBridge.php index 4c3e0e2f..900a7009 100644 --- a/bridges/AppleMusicBridge.php +++ b/bridges/AppleMusicBridge.php @@ -40,6 +40,8 @@ class AppleMusicBridge extends BridgeAbstract foreach ($json->results as $obj) { if ($obj->wrapperType === 'collection') { + $copyright = $obj->copyright ?? ''; + $this->items[] = [ 'title' => $obj->artistName . ' - ' . $obj->collectionName, 'uri' => $obj->collectionViewUrl, @@ -49,7 +51,7 @@ class AppleMusicBridge extends BridgeAbstract . '><img src="' . $obj->artworkUrl100 . '" /></a><br><br>' . $obj->artistName . ' - ' . $obj->collectionName . '<br>' - . $obj->copyright, + . $copyright, ]; } } diff --git a/bridges/AskfmBridge.php b/bridges/AskfmBridge.php index 0a326417..d0422890 100644 --- a/bridges/AskfmBridge.php +++ b/bridges/AskfmBridge.php @@ -37,7 +37,8 @@ class AskfmBridge extends BridgeAbstract $item['timestamp'] = strtotime($element->find('time', 0)->datetime); - $answer = trim($element->find('div.streamItem_content', 0)->innertext); + $var = $element->find('div.streamItem_content', 0); + $answer = trim($var->innertext ?? ''); // This probably should be cleaned up, especially for YouTube embeds if ($visual = $element->find('div.streamItem_visual', 0)) { diff --git a/bridges/FeedReducerBridge.php b/bridges/FeedReducerBridge.php index a37824c9..37bf9809 100644 --- a/bridges/FeedReducerBridge.php +++ b/bridges/FeedReducerBridge.php @@ -23,8 +23,9 @@ class FeedReducerBridge extends FeedExpander public function collectData() { - if (preg_match('#^http(s?)://#i', $this->getInput('url'))) { - $this->collectExpandableDatas($this->getInput('url')); + $url = $this->getInput('url'); + if (preg_match('#^http(s?)://#i', $url)) { + $this->collectExpandableDatas($url); } else { throw new Exception('URI must begin with http(s)://'); } @@ -35,7 +36,7 @@ class FeedReducerBridge extends FeedExpander $filteredItems = []; $intPercentage = (int)preg_replace('/[^0-9]/', '', $this->getInput('percentage')); - foreach ($this->items as $thisItem) { + foreach ($this->items as $item) { // The URL is included in the hash: // - so you can change the output by adding a local-part to the URL // - so items with the same URI in different feeds won't be correlated @@ -43,13 +44,13 @@ class FeedReducerBridge extends FeedExpander // $pseudoRandomInteger will be a 16 bit unsigned int mod 100. // This won't be uniformly distributed 1-100, but should be close enough. - $pseudoRandomInteger = unpack( - 'S', // unsigned 16-bit int - hash('sha256', $thisItem['uri'] . '::' . $this->getInput('url'), true) - )[1] % 100; + $data = $item['uri'] . '::' . $this->getInput('url'); + $hash = hash('sha256', $data, true); + // S = unsigned 16-bit int + $pseudoRandomInteger = unpack('S', $hash)[1] % 100; if ($pseudoRandomInteger < $intPercentage) { - $filteredItems[] = $thisItem; + $filteredItems[] = $item; } } diff --git a/bridges/FourchanBridge.php b/bridges/FourchanBridge.php index 179ae91f..f67d6026 100644 --- a/bridges/FourchanBridge.php +++ b/bridges/FourchanBridge.php @@ -45,7 +45,8 @@ class FourchanBridge extends BridgeAbstract $file = $element->find('.file', 0); if (!empty($file)) { - $item['image'] = $element->find('.file a', 0)->href; + $var = $element->find('.file a', 0); + $item['image'] = $var->href ?? ''; $item['imageThumb'] = $element->find('.file img', 0)->src; if (!isset($item['imageThumb']) and strpos($item['image'], '.swf') !== false) { $item['imageThumb'] = 'http://i.imgur.com/eO0cxf9.jpg'; diff --git a/bridges/GettrBridge.php b/bridges/GettrBridge.php index 2b019523..74804043 100644 --- a/bridges/GettrBridge.php +++ b/bridges/GettrBridge.php @@ -27,9 +27,10 @@ class GettrBridge extends BridgeAbstract public function collectData() { + $user = $this->getInput('user'); $api = sprintf( 'https://api.gettr.com/u/user/%s/posts?offset=0&max=%s&dir=fwd&incl=posts&fp=f_uo', - $this->getInput('user'), + $user, min($this->getInput('limit'), 20) ); $data = json_decode(getContents($api), false); diff --git a/bridges/TelegramBridge.php b/bridges/TelegramBridge.php index 14359009..9d73e06e 100644 --- a/bridges/TelegramBridge.php +++ b/bridges/TelegramBridge.php @@ -169,11 +169,19 @@ EOD; $stickerDiv->find('picture', 0)->style = ''; return $stickerDiv; - } elseif (preg_match(self::BACKGROUND_IMAGE_REGEX, $stickerDiv->find('i', 0)->style, $sticker)) { - return <<<EOD + } + + $var = $stickerDiv->find('i', 0); + if ($var) { + $style = $var->style; + if (preg_match(self::BACKGROUND_IMAGE_REGEX, $style, $sticker)) { + return <<<EOD <a href="{$stickerDiv->children(0)->herf}"><img src="{$sticker[1]}"></a> EOD; + } } + + return ''; } private function processPoll($messageDiv) diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php index 556e5ffc..769bc625 100644 --- a/bridges/TikTokBridge.php +++ b/bridges/TikTokBridge.php @@ -34,6 +34,9 @@ class TikTokBridge extends BridgeAbstract $this->feedName = htmlspecialchars_decode($title); $var = $html->find('script[id=SIGI_STATE]', 0); + if (!$var) { + throw new \Exception('Unable to find tiktok user data for ' . $this->processUsername()); + } $SIGI_STATE_RAW = $var->innertext; $SIGI_STATE = Json::decode($SIGI_STATE_RAW, false); diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index eb9d5a3c..61eafb57 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -144,6 +144,10 @@ abstract class BridgeAbstract implements BridgeInterface } foreach ($contexts as $context) { + if (!isset(static::PARAMETERS[$context])) { + // unknown context provided by client, throw exception here? or continue? + } + foreach (static::PARAMETERS[$context] as $name => $properties) { if (isset($this->inputs[$context][$name]['value'])) { continue; From 8e2353ad3e683c94d9727c5cd9efa53df76ea4c7 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 1 Aug 2023 06:19:42 +0200 Subject: [PATCH 051/716] fix: write to cache only if data is was not cached, fix #3586 (#3588) --- lib/contents.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/contents.php b/lib/contents.php index 5587a98e..c842ccbc 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -432,8 +432,6 @@ function getSimpleHTMLDOMCached( $content = $cache->loadData($timeout); if (!$content || Debug::isEnabled()) { $content = getContents($url, $header ?? [], $opts ?? []); - } - if ($content) { $cache->setScope('pages'); $cache->setKey([$url]); $cache->saveData($content); From 10f7b6f4f6652039361bccbb9392a5e3f13d7ff2 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Wed, 2 Aug 2023 03:05:06 +0930 Subject: [PATCH 052/716] Fix php8.2 deprecated warning when using bridge specific configurations (#3587) * Fix php8.2 deprecated warning Fix php8.2 warning: `Deprecated: Creation of dynamic property is deprecated` * fix * refactor: remove unused method --------- Co-authored-by: Dag <me@dvikan.no> --- lib/BridgeAbstract.php | 13 +++++-------- lib/BridgeInterface.php | 5 ----- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 61eafb57..e58ddb91 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -58,8 +58,6 @@ abstract class BridgeAbstract implements BridgeInterface /** * Configuration for the bridge - * - * Use {@see BridgeAbstract::getConfiguration()} to read this parameter */ const CONFIGURATION = []; @@ -113,6 +111,11 @@ abstract class BridgeAbstract implements BridgeInterface */ protected $queriedContext = ''; + /** + * Holds the list of bridge-specific configurations from config.ini.php, used by the bridge. + */ + private array $configuration = []; + /** {@inheritdoc} */ public function getItems() { @@ -365,12 +368,6 @@ abstract class BridgeAbstract implements BridgeInterface return static::URI . '/favicon.ico'; } - /** {@inheritdoc} */ - public function getConfiguration() - { - return static::CONFIGURATION; - } - /** {@inheritdoc} */ public function getParameters() { diff --git a/lib/BridgeInterface.php b/lib/BridgeInterface.php index b461ed12..977ad7f6 100644 --- a/lib/BridgeInterface.php +++ b/lib/BridgeInterface.php @@ -60,11 +60,6 @@ interface BridgeInterface */ public function collectData(); - /** - * Get the user's supplied configuration for the bridge - */ - public function getConfiguration(); - /** * Returns the value for the selected configuration * From ed97ce8646f869450cf94fa75828d15d5bacf53e Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 1 Aug 2023 19:35:15 +0200 Subject: [PATCH 053/716] fix: dont fail for non-existing enabled bridge (#3589) * fix: dont fail for non-existing enabled bridge * yup --- actions/ConnectivityAction.php | 10 ++++++---- actions/DisplayAction.php | 9 +++++++-- actions/SetBridgeCacheAction.php | 6 +++++- lib/BridgeFactory.php | 12 +++++++----- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/actions/ConnectivityAction.php b/actions/ConnectivityAction.php index c11e6595..604b7806 100644 --- a/actions/ConnectivityAction.php +++ b/actions/ConnectivityAction.php @@ -37,12 +37,14 @@ class ConnectivityAction implements ActionInterface throw new \Exception('This action is only available in debug mode!'); } - if (!isset($request['bridge'])) { + $bridgeName = $request['bridge'] ?? null; + if (!$bridgeName) { return render_template('connectivity.html.php'); } - - $bridgeClassName = $this->bridgeFactory->createBridgeClassName($request['bridge']); - + $bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName); + if (!$bridgeClassName) { + throw new \Exception(sprintf('Bridge not found: %s', $bridgeName)); + } return $this->reportBridgeConnectivity($bridgeClassName); } diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 129d4587..7b2efec1 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -33,9 +33,15 @@ class DisplayAction implements ActionInterface private function createResponse(array $request) { $bridgeFactory = new BridgeFactory(); - $bridgeClassName = $bridgeFactory->createBridgeClassName($request['bridge'] ?? ''); + $formatFactory = new FormatFactory(); + $bridgeName = $request['bridge'] ?? null; $format = $request['format'] ?? null; + + $bridgeClassName = $bridgeFactory->createBridgeClassName($bridgeName); + if (!$bridgeClassName) { + throw new \Exception(sprintf('Bridge not found: %s', $bridgeName)); + } if (!$format) { throw new \Exception('You must specify a format!'); } @@ -43,7 +49,6 @@ class DisplayAction implements ActionInterface throw new \Exception('This bridge is not whitelisted'); } - $formatFactory = new FormatFactory(); $format = $formatFactory->create($format); $bridge = $bridgeFactory->create($bridgeClassName); diff --git a/actions/SetBridgeCacheAction.php b/actions/SetBridgeCacheAction.php index a9a598bd..416f2378 100644 --- a/actions/SetBridgeCacheAction.php +++ b/actions/SetBridgeCacheAction.php @@ -23,7 +23,11 @@ class SetBridgeCacheAction implements ActionInterface $bridgeFactory = new BridgeFactory(); - $bridgeClassName = $bridgeFactory->createBridgeClassName($request['bridge'] ?? ''); + $bridgeName = $request['bridge'] ?? null; + $bridgeClassName = $bridgeFactory->createBridgeClassName($bridgeName); + if (!$bridgeClassName) { + throw new \Exception(sprintf('Bridge not found: %s', $bridgeName)); + } // whitelist control if (!$bridgeFactory->isEnabled($bridgeClassName)) { diff --git a/lib/BridgeFactory.php b/lib/BridgeFactory.php index db2c394a..63633f4a 100644 --- a/lib/BridgeFactory.php +++ b/lib/BridgeFactory.php @@ -23,7 +23,12 @@ final class BridgeFactory $this->enabledBridges = $this->bridgeClassNames; break; } - $this->enabledBridges[] = $this->createBridgeClassName($enabledBridge); + $bridgeClassName = $this->createBridgeClassName($enabledBridge); + if ($bridgeClassName) { + $this->enabledBridges[] = $bridgeClassName; + } else { + Logger::info(sprintf('Bridge not found: %s', $enabledBridge)); + } } } @@ -42,13 +47,10 @@ final class BridgeFactory $name = self::normalizeBridgeName($bridgeName); $namesLoweredCase = array_map('strtolower', $this->bridgeClassNames); $nameLoweredCase = strtolower($name); - if (! in_array($nameLoweredCase, $namesLoweredCase)) { - throw new \Exception(sprintf('Bridge name invalid: %s', $bridgeName)); + return null; } - $index = array_search($nameLoweredCase, $namesLoweredCase); - return $this->bridgeClassNames[$index]; } From 7661a78a43dbeb9ccba90a83bab4b68203a3c908 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Thu, 3 Aug 2023 03:10:24 +0200 Subject: [PATCH 054/716] [core] add bridge not found warning message to frontpage (#3591) --- actions/FrontpageAction.php | 10 +++++++++- lib/BridgeFactory.php | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/actions/FrontpageAction.php b/actions/FrontpageAction.php index 40d25ea4..64281b1e 100644 --- a/actions/FrontpageAction.php +++ b/actions/FrontpageAction.php @@ -4,12 +4,20 @@ final class FrontpageAction implements ActionInterface { public function execute(array $request) { + $messages = []; $showInactive = (bool) ($request['show_inactive'] ?? null); $activeBridges = 0; $bridgeFactory = new BridgeFactory(); $bridgeClassNames = $bridgeFactory->getBridgeClassNames(); + foreach ($bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) { + $messages[] = [ + 'body' => sprintf('Warning : Bridge "%s" not found', $missingEnabledBridge), + 'level' => 'warning' + ]; + } + $formatFactory = new FormatFactory(); $formats = $formatFactory->getFormatNames(); @@ -24,7 +32,7 @@ final class FrontpageAction implements ActionInterface } return render(__DIR__ . '/../templates/frontpage.html.php', [ - 'messages' => [], + 'messages' => $messages, 'admin_email' => Configuration::getConfig('admin', 'email'), 'admin_telegram' => Configuration::getConfig('admin', 'telegram'), 'bridges' => $body, diff --git a/lib/BridgeFactory.php b/lib/BridgeFactory.php index 63633f4a..f302a27a 100644 --- a/lib/BridgeFactory.php +++ b/lib/BridgeFactory.php @@ -4,6 +4,7 @@ final class BridgeFactory { private $bridgeClassNames = []; private $enabledBridges = []; + private $missingEnabledBridges = []; public function __construct() { @@ -27,6 +28,7 @@ final class BridgeFactory if ($bridgeClassName) { $this->enabledBridges[] = $bridgeClassName; } else { + $this->missingEnabledBridges[] = $enabledBridge; Logger::info(sprintf('Bridge not found: %s', $enabledBridge)); } } @@ -69,4 +71,9 @@ final class BridgeFactory { return $this->bridgeClassNames; } + + public function getMissingEnabledBridges(): array + { + return $this->missingEnabledBridges; + } } From d32419ffcf979754a4a73219729af0745a73e5bf Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Thu, 3 Aug 2023 22:43:55 +0200 Subject: [PATCH 055/716] added the option for a sessioncookie in heiseBridge (#3596) * added the option for a sessioncookie with a valid cookie you can get full heise+ (paywall) articles * formating * lint --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/HeiseBridge.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index ec8bb96f..08c43dc5 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -103,6 +103,16 @@ class HeiseBridge extends FeedExpander 'required' => false, 'title' => 'Specify number of full articles to return', 'defaultValue' => 5 + ], + 'sessioncookie' => [ + 'name' => 'Session Cookie', + 'required' => false, + 'title' => <<<'TITLE' + If you have a heise+ subscription, + you can enter your cookie (ssohls) here to + have heise+ articles displayed in full. + By default the cookie is 1 year valid. + TITLE, ] ]]; const LIMIT = 5; @@ -118,6 +128,7 @@ class HeiseBridge extends FeedExpander protected function parseItem($feedItem) { $item = parent::parseItem($feedItem); + $sessioncookie = $this->getInput('sessioncookie'); // strip rss parameter $item['uri'] = explode('?', $item['uri'])[0]; @@ -128,13 +139,15 @@ class HeiseBridge extends FeedExpander } // abort on heise+ articles and link to archive.ph for full-text content - if (str_starts_with($item['title'], 'heise+ |')) { + if ($sessioncookie == '' && str_starts_with($item['title'], 'heise+ |')) { $item['uri'] = 'https://archive.ph/?run=1&url=' . urlencode($item['uri']); return $item; } $item['uri'] .= '?seite=all'; - $article = getSimpleHTMLDOMCached($item['uri']); + $article = getSimpleHTMLDOM($item['uri'], [ + 'cookie: ssohls=' . $sessioncookie + ]); if ($article) { $article = defaultLinkTo($article, $item['uri']); From 4976cd227ea8c92003bbf252fb8236f3980e3a8a Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Fri, 4 Aug 2023 22:14:08 +0200 Subject: [PATCH 056/716] [FeedExpander] support xhtml content / content with child elements (#3598) * [core] support xhtml content type in FeedExpander * [FilterBridge] change defaultValue to exampleValue * [core] support content with child elements in FeedExpander --- bridges/FilterBridge.php | 2 +- lib/FeedExpander.php | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index 992fe0c3..ef739de3 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -12,7 +12,7 @@ class FilterBridge extends FeedExpander 'url' => [ 'name' => 'Feed URL', 'type' => 'text', - 'defaultValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day', + 'exampleValue' => 'https://lorem-rss.herokuapp.com/feed?unit=day', 'required' => true, ], 'filter' => [ diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index 1bac6179..c91586d7 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -308,7 +308,16 @@ abstract class FeedExpander extends BridgeAbstract $item['author'] = (string)$feedItem->author->name; } if (isset($feedItem->content)) { - $item['content'] = (string)$feedItem->content; + $contentChildren = $feedItem->content->children(); + if (count($contentChildren) > 0) { + $content = ''; + foreach ($contentChildren as $contentChild) { + $content .= $contentChild->asXML(); + } + $item['content'] = $content; + } else { + $item['content'] = (string)$feedItem->content; + } } //When "link" field is present, URL is more reliable than "id" field From 3e3481bd7ae8ec1fde063202748a2e41a468d393 Mon Sep 17 00:00:00 2001 From: Niehztog <Niehztog@users.noreply.github.com> Date: Mon, 7 Aug 2023 05:33:35 +0200 Subject: [PATCH 057/716] adds Nius bridge (#3599) * adds Nius bridge * fix linter errors * fix linter errors * fix linter errors * fix extract author --- bridges/NiusBridge.php | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 bridges/NiusBridge.php diff --git a/bridges/NiusBridge.php b/bridges/NiusBridge.php new file mode 100644 index 00000000..e5773a7d --- /dev/null +++ b/bridges/NiusBridge.php @@ -0,0 +1,32 @@ +<?php + +class NiusBridge extends XPathAbstract +{ + const NAME = 'Nius'; + const URI = 'https://www.nius.de/news'; + const DESCRIPTION = 'Die Stimme der Mehrheit'; + const MAINTAINER = 'Niehztog'; + //const PARAMETERS = array(); + const CACHE_TIMEOUT = 3600; + + const FEED_SOURCE_URL = 'https://www.nius.de/news'; + const XPATH_EXPRESSION_ITEM = './/div[contains(@class, "compact-story") or contains(@class, "regular-story")]'; + const XPATH_EXPRESSION_ITEM_TITLE = './/h2[@class="title"]//node()'; + const XPATH_EXPRESSION_ITEM_CONTENT = './/h2[@class="title"]//node()'; + const XPATH_EXPRESSION_ITEM_URI = './/a[1]/@href'; + + const XPATH_EXPR_AUTHOR_PART1 = 'normalize-space(.//span[@class="author"]/text()[1])'; + const XPATH_EXPR_AUTHOR_PART2 = 'normalize-space(.//span[@class="author"]/text()[2])'; + const XPATH_EXPRESSION_ITEM_AUTHOR = 'substring-after(concat(' . self::XPATH_EXPR_AUTHOR_PART1 . ', " ", ' . self::XPATH_EXPR_AUTHOR_PART2 . '), " ")'; + + //const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/td[3]'; + const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img[1]/@src'; + const XPATH_EXPRESSION_ITEM_CATEGORIES = './/div[@class="subtitle"]/text()'; + const SETTING_FIX_ENCODING = false; + + protected function cleanMediaUrl($mediaUrl) + { + $result = preg_match('~https:\/\/www\.nius\.de\/_next\/image\?url=(.*)\?~', $mediaUrl, $matches); + return $result ? $matches[1] : $mediaUrl; + } +} From cf6d94dc2ad9e96e8dc2dfdee26c63f936a92963 Mon Sep 17 00:00:00 2001 From: Korytov Pavel <thexcloud@gmail.com> Date: Tue, 8 Aug 2023 09:58:08 +0500 Subject: [PATCH 058/716] [EconomistBridge] Fix strange image urls (#3600) --- bridges/EconomistBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/EconomistBridge.php b/bridges/EconomistBridge.php index 1b15555d..9a73a852 100644 --- a/bridges/EconomistBridge.php +++ b/bridges/EconomistBridge.php @@ -159,7 +159,7 @@ class EconomistBridge extends FeedExpander $svelte->parent->removeChild($svelte); } foreach ($elem->find('img') as $strange_img) { - if (!str_contains($strange_img->src, 'https://economist.com')) { + if (!str_contains($strange_img->src, 'economist.com')) { $strange_img->src = 'https://economist.com' . $strange_img->src; } } From 43ec82179b0c255c5587449d7191307bda9434d4 Mon Sep 17 00:00:00 2001 From: adminvulcano <derwolf8@outlook.de> Date: Tue, 8 Aug 2023 07:00:07 +0200 Subject: [PATCH 059/716] [TldrTechBridge] Add Cybersecurity section (#3601) --- bridges/TldrTechBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/TldrTechBridge.php b/bridges/TldrTechBridge.php index 7d8febe1..b89686bb 100644 --- a/bridges/TldrTechBridge.php +++ b/bridges/TldrTechBridge.php @@ -25,7 +25,8 @@ class TldrTechBridge extends BridgeAbstract 'Crypto' => 'crypto', 'AI' => 'ai', 'Web Dev' => 'engineering', - 'Founders' => 'founders' + 'Founders' => 'founders', + 'Cybersecurity' => 'cybersecurity' ], 'defaultValue' => 'tech' ] From b86ee5778bc34fe032439f540c8a1e2c1e6b3ec9 Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Tue, 8 Aug 2023 15:02:01 +0200 Subject: [PATCH 060/716] [SitemapBridge] Add SitemapBridge (#3602) * [SitemapBridge] Add SitemapBridge This bridge is a variant of CssSelectorBridge. Instead of retrieving article list from home page, retrieves article list from SEO sitemap.xml. Requires CssSelectorBridge to be installed. * [SitemapBridge] Code linting --- bridges/SitemapBridge.php | 152 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 bridges/SitemapBridge.php diff --git a/bridges/SitemapBridge.php b/bridges/SitemapBridge.php new file mode 100644 index 00000000..eec9d658 --- /dev/null +++ b/bridges/SitemapBridge.php @@ -0,0 +1,152 @@ +<?php + +class SitemapBridge extends CssSelectorBridge +{ + const MAINTAINER = 'ORelio'; + const NAME = 'Sitemap Bridge'; + const URI = 'https://github.com/RSS-Bridge/rss-bridge/'; + const DESCRIPTION = 'Convert any site to RSS feed using SEO Sitemap and CSS selectors (Advanced Users)'; + const PARAMETERS = [ + [ + 'home_page' => [ + 'name' => 'Site URL: Home page with latest articles', + 'title' => <<<EOT + The bridge will analyze the site like a search engine does. + The URL specified here determines the feed title and URL. + EOT, + 'exampleValue' => 'https://example.com/blog/', + 'required' => true + ], + 'url_pattern' => [ + 'name' => 'Pattern for site URLs to take in feed', + 'title' => 'Select items by applying a regular expression on their URL', + 'exampleValue' => 'https://example.com/article/.*', + 'required' => true + ], + 'content_selector' => [ + 'name' => 'Selector for each article content', + 'title' => <<<EOT + This bridge works using CSS selectors, e.g. "div.article" will match <div class="article">. + Everything inside that element becomes feed item content. + EOT, + 'exampleValue' => 'article.content', + 'required' => true + ], + 'content_cleanup' => [ + 'name' => '[Optional] Content cleanup: List of items to remove', + 'title' => 'Selector for unnecessary elements to remove inside article contents.', + 'exampleValue' => 'div.ads, div.comments', + ], + 'title_cleanup' => [ + 'name' => '[Optional] Text to remove from article title', + 'title' => 'Specify here some text from page title that need to be removed, e.g. " | BlogName".', + 'exampleValue' => ' | BlogName', + ], + 'site_map' => [ + 'name' => '[Optional] sitemap.xml URL', + 'title' => <<<EOT + By default, the bridge will analyze robots.txt to find out URL for sitemap.xml. + Alternatively, you can specify here the direct URL for sitemap XML. + The sitemap.xml file must have <loc> and <lastmod> fields for the bridge to work: + Eg. <url><loc>https://article/url</loc><lastmod>2000-12-31T23:59Z</lastmod></url> + <loc> is feed item URL, <lastmod> for selecting the most recent entries. + EOT, + 'exampleValue' => 'https://example.com/sitemap.xml', + ], + 'limit' => self::LIMIT + ] + ]; + + public function collectData() + { + $url = $this->getInput('home_page'); + $url_pattern = $this->getInput('url_pattern'); + $content_selector = $this->getInput('content_selector'); + $content_cleanup = $this->getInput('content_cleanup'); + $title_cleanup = $this->getInput('title_cleanup'); + $site_map = $this->getInput('site_map'); + $limit = $this->getInput('limit'); + + $this->feedName = $this->getPageTitle($url, $title_cleanup); + $sitemap_url = empty($site_map) ? $url : $site_map; + $sitemap_xml = $this->getSitemapXml($sitemap_url, !empty($site_map)); + $links = $this->sitemapXmlToList($sitemap_xml, $url_pattern, empty($limit) ? 10 : $limit); + + if (empty($links) && empty(sitemapXmlToList($sitemap_xml))) { + returnClientError('Could not retrieve URLs with Timestamps from Sitemap: ' . $sitemap_url); + } + + foreach ($links as $link) { + $this->items[] = $this->expandEntryWithSelector($link, $content_selector, $content_cleanup, $title_cleanup); + } + } + + /** + * Retrieve site map from specified URL + * @param string $url URL pointing to any page of the site, e.g. "https://example.com/blog" OR directly to the site map e.g. "https://example.com/sitemap.xml" + * @param string $is_site_map TRUE if the specified URL points directly to the sitemap XML + * @return object Sitemap DOM (from parsed XML) + */ + protected function getSitemapXml(&$url, $is_site_map = false) + { + if (!$is_site_map) { + $robots_txt = getSimpleHTMLDOM(urljoin($url, '/robots.txt'))->outertext; + preg_match('/Sitemap: ([^ ]+)/', $robots_txt, $matches); + if (empty($matches)) { + returnClientError('Failed to determine Sitemap from robots.txt. Try setting it manually.'); + } + $url = $matches[1]; + } + return getSimpleHTMLDOM($url); + } + + /** + * Retrieve N most recent URLs from Site Map + * @param object $sitemap Site map XML DOM + * @param string $url_pattern Optional pattern to look for in URLs + * @param int $limit Optional maximum amount of URLs to return + * @param bool $keep_date TRUE to keep dates (url => date array instead of url array) + * @return array Array of URLs + */ + protected function sitemapXmlToList($sitemap, $url_pattern = '', $limit = 0, $keep_date = false) + { + $links = []; + + foreach ($sitemap->find('sitemap') as $nested_sitemap) { + $url = $nested_sitemap->find('loc'); + if (!empty($url)) { + $url = $url[0]->plaintext; + if (str_ends_with(strtolower($url), '.xml')) { + $nested_sitemap_xml = $this->getSitemapXml($url, true); + $nested_sitemap_links = $this->sitemapXmlToList($nested_sitemap_xml, $url_pattern, null, true); + $links = array_merge($links, $nested_sitemap_links); + } + } + } + + if (!empty($url_pattern)) { + $url_pattern = str_replace('/', '\/', $url_pattern); + } + + foreach ($sitemap->find('url') as $item) { + $url = $item->find('loc'); + $lastmod = $item->find('lastmod'); + if (!empty($url) && !empty($lastmod)) { + $url = $url[0]->plaintext; + $lastmod = $lastmod[0]->plaintext; + $timestamp = strtotime($lastmod); + if (empty($url_pattern) || preg_match('/' . $url_pattern . '/', $url) === 1) { + $links[$url] = $timestamp; + } + } + } + + arsort($links); + + if ($limit > 0 && count($links) > $limit) { + $links = array_slice($links, 0, $limit); + } + + return $keep_date ? $links : array_keys($links); + } +} From f3896ed543c5e026ee114fa37e243a01c71322a7 Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:35:35 +0200 Subject: [PATCH 061/716] [ImgsedBridge] Add detectParameters feature to the bridge (#3604) The bridge can detect the most common profile variation URL of instagram.com or imgsed.com websites to extract the username. --- bridges/ImgsedBridge.php | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index 1555c578..5fe3eee8 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -256,4 +256,30 @@ HTML, } return parent::getName(); } + + public function detectParameters($url) + { + $params = [ + 'post' => 'on', + 'story' => 'on', + 'tagged' => 'on' + ]; + $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})\/(reels\/|tagged\/|) +|(www\.|)(imgsed.com)\/(stories\/|tagged\/|)([a-zA-Z0-9_\.]{1,30})\/)/'; + if (preg_match($regex, $url, $matches) > 0) { + // Extract detected domain using the regex + $domain = $matches[8] ?? $matches[4]; + if ($domain == 'imgsed.com') { + $params['u'] = $matches[10]; + return $params; + } else if ($domain == 'instagram.com') { + $params['u'] = $matches[5]; + return $params; + } else { + return null; + } + } else { + return null; + } + } } From 1fcf67f14a8f738f22395d453cb39aeed573af78 Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Wed, 9 Aug 2023 17:36:02 +0200 Subject: [PATCH 062/716] [PepperBridgeAbstract] Fix deal origin (#3605) Origin display has chenged : this commit follow the websites changes. Fixes #3521 --- bridges/PepperBridgeAbstract.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 85178e54..c152d249 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -393,10 +393,10 @@ HEREDOC; */ private function getSource($deal) { - if ($deal->find('a[class*=text--color-greyShade]', 0) != null) { - return '<div>' . $this->i8n('origin') . ' : ' - . $deal->find('a[class*=text--color-greyShade]', 0)->outertext - . '</div>'; + if (($origin = $deal->find('button[class*=text--color-greyShade]', 0)) != null) { + $path = str_replace(' ', '/', trim(Json::decode($origin->{'data-cloak-link'})['path'])); + $text = $origin->find('span[class*=cept-merchant-name]', 0); + return '<div>' . $this->i8n('origin') . ' : <a href="' . static::URI . $path . '">' . $text . '</a></div>'; } else { return ''; } From 6cc4cf24dc2a52084b71d22a2759f28fe6d11ebc Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Wed, 9 Aug 2023 20:10:15 +0200 Subject: [PATCH 063/716] [FuturaSciences] Fix content extraction (#3487, #3488) (#3606) --- bridges/FuturaSciencesBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/FuturaSciencesBridge.php b/bridges/FuturaSciencesBridge.php index 0aa394fa..3fb8aafa 100644 --- a/bridges/FuturaSciencesBridge.php +++ b/bridges/FuturaSciencesBridge.php @@ -90,7 +90,7 @@ class FuturaSciencesBridge extends FeedExpander $item = parent::parseItem($newsItem); $item['uri'] = str_replace('#xtor%3DRSS-8', '', $item['uri']); $article = getSimpleHTMLDOMCached($item['uri']); - //$item['content'] = $this->extractArticleContent($article); + $item['content'] = $this->extractArticleContent($article); $author = $this->extractAuthor($article); if (!empty($author)) { $item['author'] = $author; @@ -100,7 +100,7 @@ class FuturaSciencesBridge extends FeedExpander private function extractArticleContent($article) { - $contents = $article->find('section.article-text', 1); + $contents = $article->find('div.article-text', 0); foreach ($contents->find('img') as $img) { if (!empty($img->getAttribute('data-src'))) { From 52d3cce59dae1b5b8adb5497450e396473ebf6b2 Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Wed, 9 Aug 2023 22:40:24 +0200 Subject: [PATCH 064/716] bridges: add context to detectParameters (#3607) * bridges: add context to detectParameters Some bridges did not return the context parameter but they used it in the parameters * bridges: add context to detectParameters Fix test for InstagramBridge --- bridges/BadDragonBridge.php | 2 + bridges/BandcampBridge.php | 3 + bridges/FacebookBridge.php | 3 + bridges/FreeTelechargerBridge.php | 151 +++++++++++++++--------------- bridges/FunkBridge.php | 1 + bridges/FurAffinityBridge.php | 4 + bridges/GithubIssueBridge.php | 4 + bridges/ImgsedBridge.php | 1 + bridges/InstagramBridge.php | 7 +- bridges/RedditBridge.php | 2 + bridges/SkimfeedBridge.php | 3 +- bridges/TrelloBridge.php | 10 +- bridges/TwitterBridge.php | 4 + 13 files changed, 115 insertions(+), 80 deletions(-) diff --git a/bridges/BadDragonBridge.php b/bridges/BadDragonBridge.php index 2260bbd6..d38e3408 100644 --- a/bridges/BadDragonBridge.php +++ b/bridges/BadDragonBridge.php @@ -138,6 +138,7 @@ class BadDragonBridge extends BridgeAbstract // Sale $regex = '/^(https?:\/\/)?bad-dragon\.com\/sales/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Sales'; return $params; } @@ -192,6 +193,7 @@ class BadDragonBridge extends BridgeAbstract isset($urlParams['noAccessories']) && $urlParams['noAccessories'] === '1' && $params['noAccessories'] = 'on'; + $params['context'] = 'Clearance'; return $params; } diff --git a/bridges/BandcampBridge.php b/bridges/BandcampBridge.php index c041b2b3..a9bd2ea1 100644 --- a/bridges/BandcampBridge.php +++ b/bridges/BandcampBridge.php @@ -397,6 +397,7 @@ class BandcampBridge extends BridgeAbstract // By tag $regex = '/^(https?:\/\/)?bandcamp\.com\/tag\/([^\/.&?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By tag'; $params['tag'] = urldecode($matches[2]); return $params; } @@ -404,6 +405,7 @@ class BandcampBridge extends BridgeAbstract // By band $regex = '/^(https?:\/\/)?([^\/.&?\n]+?)\.bandcamp\.com/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By band'; $params['band'] = urldecode($matches[2]); return $params; } @@ -411,6 +413,7 @@ class BandcampBridge extends BridgeAbstract // By album $regex = '/^(https?:\/\/)?([^\/.&?\n]+?)\.bandcamp\.com\/album\/([^\/.&?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By album'; $params['band'] = urldecode($matches[2]); $params['album'] = urldecode($matches[3]); return $params; diff --git a/bridges/FacebookBridge.php b/bridges/FacebookBridge.php index 9ccf6dcd..3e693a38 100644 --- a/bridges/FacebookBridge.php +++ b/bridges/FacebookBridge.php @@ -88,6 +88,7 @@ class FacebookBridge extends BridgeAbstract // By profile $regex = '/^(https?:\/\/)?(www\.)?facebook\.com\/profile\.php\?id\=([^\/?&\n]+)?(.*)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'User'; $params['u'] = urldecode($matches[3]); return $params; } @@ -95,6 +96,7 @@ class FacebookBridge extends BridgeAbstract // By group $regex = '/^(https?:\/\/)?(www\.)?facebook\.com\/groups\/([^\/?\n]+)?(.*)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Group'; $params['g'] = urldecode($matches[3]); return $params; } @@ -103,6 +105,7 @@ class FacebookBridge extends BridgeAbstract $regex = '/^(https?:\/\/)?(www\.)?facebook\.com\/([^\/?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = ''; $params['u'] = urldecode($matches[3]); return $params; } diff --git a/bridges/FreeTelechargerBridge.php b/bridges/FreeTelechargerBridge.php index b94ff835..8362b4ff 100644 --- a/bridges/FreeTelechargerBridge.php +++ b/bridges/FreeTelechargerBridge.php @@ -2,89 +2,90 @@ class FreeTelechargerBridge extends BridgeAbstract { - const NAME = 'Free-Telecharger'; - const URI = 'https://www.free-telecharger.live/'; - const DESCRIPTION = 'Suivi de série sur Free-Telecharger'; - const MAINTAINER = 'sysadminstory'; - const PARAMETERS = [ - 'Suivi de publication de série' => [ - 'url' => [ - 'name' => 'URL de la série', - 'type' => 'text', - 'required' => true, - 'title' => 'URL d\'une série sans le https://www.free-telecharger.live/', - 'pattern' => 'series.*\.html', - 'exampleValue' => 'series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html' - ], - ] - ]; - const CACHE_TIMEOUT = 3600; - public function collectData() - { - $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')); + const NAME = 'Free-Telecharger'; + const URI = 'https://www.free-telecharger.live/'; + const DESCRIPTION = 'Suivi de série sur Free-Telecharger'; + const MAINTAINER = 'sysadminstory'; + const PARAMETERS = [ + 'Suivi de publication de série' => [ + 'url' => [ + 'name' => 'URL de la série', + 'type' => 'text', + 'required' => true, + 'title' => 'URL d\'une série sans le https://www.free-telecharger.live/', + 'pattern' => 'series.*\.html', + 'exampleValue' => 'series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html' + ], + ] + ]; + const CACHE_TIMEOUT = 3600; + public function collectData() + { + $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')); - // Find all block content of the page - $blocks = $html->find('div[class=block1]'); + // Find all block content of the page + $blocks = $html->find('div[class=block1]'); - // Global Infos block - $infosBlock = $blocks[0]; - // Links block - $linksBlock = $blocks[2]; + // Global Infos block + $infosBlock = $blocks[0]; + // Links block + $linksBlock = $blocks[2]; - // Extract Global Show infos - $this->showTitle = trim($infosBlock->find('div[class=titre1]', 0)->find('font', 0)->plaintext); - $this->showTechDetails = trim($infosBlock->find('div[align=center]', 0)->find('b', 0)->plaintext); + // Extract Global Show infos + $this->showTitle = trim($infosBlock->find('div[class=titre1]', 0)->find('font', 0)->plaintext); + $this->showTechDetails = trim($infosBlock->find('div[align=center]', 0)->find('b', 0)->plaintext); - // Get Episodes names and links - $episodes = $linksBlock->find('div[id=link]', 0)->find('font[color=#ff6600]'); - $links = $linksBlock->find('div[id=link]', 0)->find('a'); + // Get Episodes names and links + $episodes = $linksBlock->find('div[id=link]', 0)->find('font[color=#ff6600]'); + $links = $linksBlock->find('div[id=link]', 0)->find('a'); - foreach ($episodes as $index => $episode) { - $item = []; // Create an empty item - $item['title'] = $this->showTitle . ' ' . $this->showTechDetails . ' - ' . ltrim(trim($episode->plaintext), '-'); - $item['uri'] = $links[$index]->href; - $item['content'] = '<a href="' . $item['uri'] . '">' . $item['title'] . '</a>'; - $item['uid'] = hash('md5', $item['uri']); + foreach ($episodes as $index => $episode) { + $item = []; // Create an empty item + $item['title'] = $this->showTitle . ' ' . $this->showTechDetails . ' - ' . ltrim(trim($episode->plaintext), '-'); + $item['uri'] = $links[$index]->href; + $item['content'] = '<a href="' . $item['uri'] . '">' . $item['title'] . '</a>'; + $item['uid'] = hash('md5', $item['uri']); - $this->items[] = $item; // Add this item to the list - } + $this->items[] = $item; // Add this item to the list + } + } + + public function getName() + { + switch ($this->queriedContext) { + case 'Suivi de publication de série': + return $this->showTitle . ' ' . $this->showTechDetails . ' - ' . self::NAME; + break; + default: + return self::NAME; + } + } + + public function getURI() + { + switch ($this->queriedContext) { + case 'Suivi de publication de série': + return self::URI . $this->getInput('url'); + break; + default: + return self::URI; + } + } + + public function detectParameters($url) + { + // Example: https://www.free-telecharger.live/series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html + + $params = []; + $regex = '/^https:\/\/www.*\.free-telecharger\.live\/(series.*\.html)/'; + if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Suivi de publication de série'; + $params['url'] = urldecode($matches[1]); + return $params; } - public function getName() - { - switch ($this->queriedContext) { - case 'Suivi de publication de série': - return $this->showTitle . ' ' . $this->showTechDetails . ' - ' . self::NAME; - break; - default: - return self::NAME; - } - } - - public function getURI() - { - switch ($this->queriedContext) { - case 'Suivi de publication de série': - return self::URI . $this->getInput('url'); - break; - default: - return self::URI; - } - } - - public function detectParameters($url) - { - // Example: https://www.free-telecharger.live/series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html - - $params = []; - $regex = '/^https:\/\/www.*\.free-telecharger\.live\/(series.*\.html)/'; - if (preg_match($regex, $url, $matches) > 0) { - $params['url'] = urldecode($matches[1]); - return $params; - } - - return null; - } + return null; + } } diff --git a/bridges/FunkBridge.php b/bridges/FunkBridge.php index 69be4928..df499035 100644 --- a/bridges/FunkBridge.php +++ b/bridges/FunkBridge.php @@ -64,6 +64,7 @@ class FunkBridge extends BridgeAbstract $regex = '/^https?:\/\/(?:www\.)?funk\.net\/channel\/([^\/]+).*$/'; if (preg_match($regex, $url, $urlMatches) > 0) { return [ + 'context' => 'Channel', 'channel' => $urlMatches[1] ]; } else { diff --git a/bridges/FurAffinityBridge.php b/bridges/FurAffinityBridge.php index c548ff65..6c2d7b52 100644 --- a/bridges/FurAffinityBridge.php +++ b/bridges/FurAffinityBridge.php @@ -603,6 +603,7 @@ class FurAffinityBridge extends BridgeAbstract // Single journal $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/journal\/(\d+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Single Journal'; $params['journal-id'] = urldecode($matches[3]); return $params; } @@ -610,6 +611,7 @@ class FurAffinityBridge extends BridgeAbstract // Journals $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/journals\/([^\/&?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Journals'; $params['username-journals'] = urldecode($matches[3]); return $params; } @@ -617,6 +619,7 @@ class FurAffinityBridge extends BridgeAbstract // Gallery folder $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/gallery\/([^\/&?\n]+)\/folder\/(\d+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Gallery Folder'; $params['username-folder'] = urldecode($matches[3]); $params['folder-id'] = urldecode($matches[4]); $params['full'] = 'on'; @@ -626,6 +629,7 @@ class FurAffinityBridge extends BridgeAbstract // Gallery (must be after gallery folder) $regex = '/^(https?:\/\/)?(www\.)?furaffinity.net\/(gallery|scraps|favorites)\/([^\/&?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Gallery'; $params['username-' . $matches[3]] = urldecode($matches[4]); $params['full'] = 'on'; return $params; diff --git a/bridges/GithubIssueBridge.php b/bridges/GithubIssueBridge.php index e4e995e3..7f56abbd 100644 --- a/bridges/GithubIssueBridge.php +++ b/bridges/GithubIssueBridge.php @@ -277,6 +277,7 @@ class GithubIssueBridge extends BridgeAbstract case 2: // Project issues [$user, $project] = $path_segments; $show_comments = 'off'; + $context = 'Project Issues'; break; case 3: // Project issues with issue comments if ($path_segments[2] !== static::URL_PATH) { @@ -284,15 +285,18 @@ class GithubIssueBridge extends BridgeAbstract } [$user, $project] = $path_segments; $show_comments = 'on'; + $context = 'Project Issues'; break; case 4: // Issue comments [$user, $project, /* issues */, $issue] = $path_segments; + $context = 'Issue comments'; break; default: return null; } return [ + 'context' => $context, 'u' => $user, 'p' => $project, 'c' => $show_comments ?? null, diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index 5fe3eee8..cf17acb4 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -267,6 +267,7 @@ HTML, $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})\/(reels\/|tagged\/|) |(www\.|)(imgsed.com)\/(stories\/|tagged\/|)([a-zA-Z0-9_\.]{1,30})\/)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Username'; // Extract detected domain using the regex $domain = $matches[8] ?? $matches[4]; if ($domain == 'imgsed.com') { diff --git a/bridges/InstagramBridge.php b/bridges/InstagramBridge.php index 71431906..0f644c4a 100644 --- a/bridges/InstagramBridge.php +++ b/bridges/InstagramBridge.php @@ -63,9 +63,9 @@ class InstagramBridge extends BridgeAbstract ]; const TEST_DETECT_PARAMETERS = [ - 'https://www.instagram.com/metaverse' => ['u' => 'metaverse'], - 'https://instagram.com/metaverse' => ['u' => 'metaverse'], - 'http://www.instagram.com/metaverse' => ['u' => 'metaverse'], + 'https://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'https://instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'http://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], ]; const USER_QUERY_HASH = '58b6785bea111c67129decbe6a448951'; @@ -323,6 +323,7 @@ class InstagramBridge extends BridgeAbstract $regex = '/^(https?:\/\/)?(www\.)?instagram\.com\/([^\/?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Username'; $params['u'] = urldecode($matches[3]); return $params; } diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 86d7884b..bd60243f 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -301,10 +301,12 @@ class RedditBridge extends BridgeAbstract if ($path[1] == 'r') { return [ + 'context' => 'single', 'r' => $path[2] ]; } elseif ($path[1] == 'user') { return [ + 'context' => 'user', 'u' => $path[2] ]; } else { diff --git a/bridges/SkimfeedBridge.php b/bridges/SkimfeedBridge.php index 19d4c823..a224cdf4 100644 --- a/bridges/SkimfeedBridge.php +++ b/bridges/SkimfeedBridge.php @@ -455,11 +455,12 @@ class SkimfeedBridge extends BridgeAbstract return null; } - foreach (self::PARAMETERS as $channels) { + foreach (self::PARAMETERS as $context => $channels) { foreach ($channels as $box_name => $box) { foreach ($box['values'] as $name => $channel_url) { if (static::URI . $channel_url === $url) { return [ + 'context' => $context, $box_name => $name, ]; } diff --git a/bridges/TrelloBridge.php b/bridges/TrelloBridge.php index ea7eb71b..a1b5cfb8 100644 --- a/bridges/TrelloBridge.php +++ b/bridges/TrelloBridge.php @@ -670,7 +670,15 @@ class TrelloBridge extends BridgeAbstract { $regex = '/^(https?:\/\/)?trello\.com\/([bc])\/([^\/?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { - return [$matches[2] => $matches[3]]; + if ($matches[2] == 'b') { + $context = 'Board'; + } else if ($matches[2] == 'c') { + $context = 'Card'; + } + return [ + 'context' => $context, + $matches[2] => $matches[3] + ]; } else { return null; } diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 1ba00c66..1f115be8 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -133,6 +133,7 @@ EOD // By keyword or hashtag (search) $regex = '/^(https?:\/\/)?(www\.)?twitter\.com\/search.*(\?|&)q=([^\/&?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By keyword or hashtag'; $params['q'] = urldecode($matches[4]); return $params; } @@ -140,6 +141,7 @@ EOD // By hashtag $regex = '/^(https?:\/\/)?(www\.)?twitter\.com\/hashtag\/([^\/?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By keyword or hashtag'; $params['q'] = urldecode($matches[3]); return $params; } @@ -147,6 +149,7 @@ EOD // By list $regex = '/^(https?:\/\/)?(www\.)?twitter\.com\/([^\/?\n]+)\/lists\/([^\/?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By list'; $params['user'] = urldecode($matches[3]); $params['list'] = urldecode($matches[4]); return $params; @@ -155,6 +158,7 @@ EOD // By username $regex = '/^(https?:\/\/)?(www\.)?twitter\.com\/([^\/?\n]+)/'; if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By username'; $params['u'] = urldecode($matches[3]); return $params; } From 11ea6aedfd06e1da1294883d382c063951ae3c2e Mon Sep 17 00:00:00 2001 From: Christian Schabesberger <chris.schabesberger@mailbox.org> Date: Thu, 10 Aug 2023 23:59:37 +0200 Subject: [PATCH 065/716] hide dpa articles in Nordbayern News (#3608) --- bridges/NordbayernBridge.php | 42 ++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/bridges/NordbayernBridge.php b/bridges/NordbayernBridge.php index b409e1e0..aa32f4ba 100644 --- a/bridges/NordbayernBridge.php +++ b/bridges/NordbayernBridge.php @@ -44,6 +44,12 @@ class NordbayernBridge extends BridgeAbstract 'type' => 'checkbox', 'exampleValue' => 'unchecked', 'title' => 'Hide all paywall articles on NN' + ], + 'hideDPA' => [ + 'name' => 'Hide dpa articles', + 'type' => 'checkbox', + 'exampleValue' => 'unchecked', + 'title' => 'Hide external articles from dpa' ] ]]; @@ -103,7 +109,7 @@ class NordbayernBridge extends BridgeAbstract return $teaser; } - private function handleArticle($link) + private function getArticle($link) { $item = []; $article = getSimpleHTMLDOM($link); @@ -142,15 +148,9 @@ class NordbayernBridge extends BridgeAbstract $item['content'] .= self::getUseFullContent($content); } - // exclude police reports if desired - if ( - $this->getInput('policeReports') || - !str_contains($item['content'], 'Hier geht es zu allen aktuellen Polizeimeldungen.') - ) { - $this->items[] = $item; - } $article->clear(); + return $item; } private function handleNewsblock($listSite) @@ -161,11 +161,31 @@ class NordbayernBridge extends BridgeAbstract $url = urljoin(self::URI, $url); // exclude nn+ articles if desired if ( - !$this->getInput('hideNNPlus') || - !str_contains($url, 'www.nn.de') + $this->getInput('hideNNPlus') && + str_contains($url, 'www.nn.de') ) { - self::handleArticle($url); + continue; } + + $item = self::getArticle($url); + + // exclude police reports if desired + if ( + !$this->getInput('policeReports') && + str_contains($item['content'], 'Hier geht es zu allen aktuellen Polizeimeldungen.') + ) { + continue; + } + + // exclude dpa articles + if ( + $this->getInput('hideDPA') && + str_contains($item['author'], 'dpa') + ) { + continue; + } + + $this->items[] = $item; } } From d55994643d5a5495d185bfef43ff52a59aea9a03 Mon Sep 17 00:00:00 2001 From: George Sokianos <walkero@gmail.com> Date: Fri, 11 Aug 2023 15:16:53 +0100 Subject: [PATCH 066/716] bridges: Added Ko-Fi.com bridge (#3609) * Added Ko-Fi.com bridge * Changed the exampleValue based on KoFiBridge-pr-context1 artifacts * Fixed "Undefined array key 0" error * fixed PHPCS issues --------- Co-authored-by: George Sokianos <George.Sokianos@hostelworld.com> --- bridges/KoFiBridge.php | 79 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 bridges/KoFiBridge.php diff --git a/bridges/KoFiBridge.php b/bridges/KoFiBridge.php new file mode 100644 index 00000000..c1600590 --- /dev/null +++ b/bridges/KoFiBridge.php @@ -0,0 +1,79 @@ +<?php + +class KoFiBridge extends BridgeAbstract +{ + const MAINTAINER = 'walkero'; + const NAME = 'Ko-Fi Bridge'; + const URI = 'https://ko-fi.com'; + const CACHE_TIMEOUT = 3600; // 1h + const DESCRIPTION = 'Returns the newest articles.'; + const FEED_URI = 'https://ko-fi.com/Feed/PersonalFeed?pageIndex=0&pageId='; + const PARAMETERS = [[ + 'pageId' => [ + 'name' => 'Page ID', + 'type' => 'text', + 'required' => true, + 'exampleValue' => 'walkero', + ] + ]]; + + public function collectData() + { + $limit = 0; + $html = getSimpleHTMLDOM(self::FEED_URI . $this->getPageId()); + foreach ($html->find('div.feeditem-unit') as $element) { + if ($limit < 10) { + $titleWrapper = $element->find('div.content-link-text'); + if (isset($titleWrapper[0])) { + $item = []; + $item['title'] = $element->find('div.content-link-text div')[0]->plaintext; + // $item['timestamp'] = strtotime($element->find('div.feeditem-time', 0)->plaintext); + $item['uri'] = self::URI . $element->find('div.fi-post-item-large a')[0]->href; + if (isset($element->find('div.fi-post-item-large div.content-link-post img')[0])) { + $item['enclosures'][] = $element->find('div.fi-post-item-large div.content-link-post img')[0]->src; + } + // $item['content'] = $element->find('div.content-link-text div#content-link', 0)->plaintext; + + $html = getSimpleHTMLDOM($item['uri']); + $feedItemTime = $html->find('div.feeditem-time', 0); + $feedItemTime->find('span', 0)->remove(); + $feedItemTime->find('div', 0)->remove(); + $item['timestamp'] = strtotime(trim($feedItemTime->plaintext)); + $item['content'] = $this->getFullContent($html); + $html->clear(); + + $this->items[] = $item; + $limit++; + } + } + } + $html->clear(); + } + + private function getFullContent($html) + { + foreach ($html->find('script[type="text/javascript"]') as $script) { + if (!empty($script->innertext)) { + if (strpos($script->innertext, 'shadowDom.innerHTML += \'') !== false) { + preg_match_all('/d\N+/i', $script->innertext, $aMatches); + foreach ($aMatches[0] as $match) { + if (strpos($match, 'article-body') !== false) { + break; + } + } + $fullPostHtml = str_get_html(mb_substr($match, 21, -3)); + // Get the first paragraph + return mb_substr($fullPostHtml->innertext, 0, mb_strpos($fullPostHtml->innertext, '</p>') + 4); + } + } + } + } + + private function getPageId() + { + $html = getSimpleHTMLDOM(self::URI . '/' . $this->getInput('pageId')); + $reportUrl = $html->find('div.modal-dialog div.mb a.btn')[1]->href; + $html->clear(); + return substr($reportUrl, strpos($reportUrl, '=') + 1); + } +} From ce72503df607cf7aa7d53bd10f38da97eb9bf17b Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Fri, 11 Aug 2023 19:02:38 +0200 Subject: [PATCH 067/716] fix(codeberg): change dom selectors for timestamp and content, fix #3610 (#3611) --- bridges/CodebergBridge.php | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bridges/CodebergBridge.php b/bridges/CodebergBridge.php index df8ef647..83775885 100644 --- a/bridges/CodebergBridge.php +++ b/bridges/CodebergBridge.php @@ -209,7 +209,12 @@ class CodebergBridge extends BridgeAbstract $item['title'] = $li->find('a.title', 0)->plaintext . ' (' . $number . ')'; $item['uri'] = $li->find('a.title', 0)->href; - $item['timestamp'] = $li->find('span.time-since', 0)->title; + + $time = $li->find('relative-time.time-since', 0); + if ($time) { + $item['timestamp'] = $time->datetime; + } + $item['author'] = $li->find('div.desc', 0)->find('a', 1)->plaintext; // Fetch issue page @@ -270,14 +275,23 @@ class CodebergBridge extends BridgeAbstract $item['title'] = $li->find('a.title', 0)->plaintext . ' (' . $number . ')'; $item['uri'] = $li->find('a.title', 0)->href; - $item['timestamp'] = $li->find('span.time-since', 0)->title; + + $time = $li->find('relative-time.time-since', 0); + if ($time) { + $item['timestamp'] = $time->datetime; + } + $item['author'] = $li->find('div.desc', 0)->find('a', 1)->plaintext; // Fetch pull request page $pullRequestPage = getSimpleHTMLDOMCached($item['uri'], 3600); $pullRequestPage = defaultLinkTo($pullRequestPage, self::URI); - $item['content'] = $pullRequestPage->find('ui.timeline', 0)->find('div.render-content.markup', 0); + $var = $pullRequestPage->find('ui.timeline', 0); + if ($var) { + $var1 = $var->find('div.render-content.markup', 0); + $item['content'] = $var1; + } foreach ($li->find('a.ui.label') as $label) { $item['categories'][] = $label->plaintext; From 7a1180c80f790d0c9baf3512aa4da2593e621288 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Tue, 15 Aug 2023 23:32:58 +0930 Subject: [PATCH 068/716] bridges: added Itaku.ee Bridge (#3615) * Fix php8.2 deprecated warning Fix php8.2 warning: `Deprecated: Creation of dynamic property is deprecated` * modified tag presentation * renamed to fit naming convention * undo commit * applied phpcbf and phpunit --- bridges/ItakuBridge.php | 699 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 699 insertions(+) create mode 100644 bridges/ItakuBridge.php diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php new file mode 100644 index 00000000..6b0ebcb2 --- /dev/null +++ b/bridges/ItakuBridge.php @@ -0,0 +1,699 @@ +<?php + +class ItakuBridge extends BridgeAbstract +{ + const NAME = 'Itaku.ee Bridge'; + const URI = 'https://itaku.ee'; + const CACHE_TIMEOUT = 900; // 15mn + const MAINTAINER = 'mruac'; + const DESCRIPTION = 'Bridges for Itaku.ee'; + const PARAMETERS = [ + 'Image Search' => [ + 'text' => [ + 'name' => 'Text to search', + 'title' => 'Search includes title, description and tags.', + 'type' => 'text', + 'exampleValue' => 'Text (incl. tags)' + ], + 'tags' => [ + 'name' => 'Tags to search', + 'title' => 'Space seperated tags to include in search. Prepend with "-" to exclude, "~" for optional.', + 'type' => 'text', + 'exampleValue' => 'tag1 -tag2 ~tag3' + ], + 'order' => [ + 'name' => 'Sort by', + 'type' => 'list', + 'values' => [ + 'Trending' => '-hotness_score', + 'Newest' => '-date_added', + 'Oldest' => 'date_added', + 'Top' => '-num_likes', + 'Bottom' => 'num_likes' + ], + 'defaultValue' => '-date_added' + ], + 'range' => [ + 'name' => 'Date range', + 'type' => 'list', + 'values' => [ + 'Today' => 'today', + 'Yesterday' => 'yesterday', + 'Past 3 days' => '3_days', + 'Past week' => 'week', + 'Past month' => '30_days', + 'Past year' => '365_days', + 'All time' => '' + ], + 'defaultValue' => 'All time' + ], + 'video_only' => [ + 'name' => 'Video only?', + 'type' => 'checkbox' + ], + 'rating_s' => [ + 'name' => 'Include SFW', + 'type' => 'checkbox' + ], + 'rating_q' => [ + 'name' => 'Include Questionable', + 'type' => 'checkbox' + ], + 'rating_e' => [ + 'name' => 'Include NSFW', + 'type' => 'checkbox' + ] + ], + 'Post Search' => [ + 'tags' => [ + 'name' => 'Tags to search', + 'title' => 'Space seperated tags to include in search. Prepend with "-" to exclude, "~" for optional.', + 'type' => 'text', + 'exampleValue' => 'tag1 -tag2 ~tag3' + ], + 'order' => [ + 'name' => 'Sort by', + 'type' => 'list', + 'values' => [ + 'Trending' => '-hotness_score', + 'Newest' => '-date_added', + 'Oldest' => 'date_added', + 'Top' => '-num_likes', + 'Bottom' => 'num_likes' + ], + 'defaultValue' => '-date_added' + ], + 'range' => [ + 'name' => 'Date range', + 'type' => 'list', + 'values' => [ + 'Today' => 'today', + 'Yesterday' => 'yesterday', + 'Past 3 days' => '3_days', + 'Past week' => 'week', + 'Past month' => '30_days', + 'Past year' => '365_days', + 'All time' => '' + ], + 'defaultValue' => 'All time' + ], + 'text_only' => [ + 'name' => 'Only include posts with text?', + 'type' => 'checkbox' + ], + 'rating_s' => [ + 'name' => 'Include SFW', + 'type' => 'checkbox' + ], + 'rating_q' => [ + 'name' => 'Include Questionable', + 'type' => 'checkbox' + ], + 'rating_e' => [ + 'name' => 'Include NSFW', + 'type' => 'checkbox' + ] + ], + 'User profile' => [ + 'user' => [ + 'name' => 'Username', + 'type' => 'text', + 'required' => true + ], + 'user_id' => [ + 'name' => 'User ID', + 'type' => 'number', + 'title' => 'User ID, if known.' + ], + 'reshares' => [ + 'name' => 'Include reshares', + 'type' => 'checkbox' + ], + 'rating_s' => [ + 'name' => 'Include SFW', + 'type' => 'checkbox' + ], + 'rating_q' => [ + 'name' => 'Include Questionable', + 'type' => 'checkbox' + ], + 'rating_e' => [ + 'name' => 'Include NSFW', + 'type' => 'checkbox' + ] + ], + 'Home feed' => [ + 'order' => [ + 'name' => 'Sort by', + 'type' => 'list', + 'values' => [ + 'Trending' => '-hotness_score', + 'Newest' => '-date_added' + ], + 'defaultValue' => '-date_added' + ], + 'range' => [ + 'name' => 'Date range', + 'type' => 'list', + 'values' => [ + 'Today' => 'today', + 'Yesterday' => 'yesterday', + 'Past 3 days' => '3_days', + 'Past week' => 'week', + 'Past month' => '30_days', + 'Past year' => '365_days', + 'All time' => '' + ], + 'defaultValue' => 'All time' + ], + 'reshares' => [ + 'name' => 'Include reshares', + 'type' => 'checkbox' + ], + 'rating_s' => [ + 'name' => 'Include SFW', + 'type' => 'checkbox' + ], + 'rating_q' => [ + 'name' => 'Include Questionable', + 'type' => 'checkbox' + ], + 'rating_e' => [ + 'name' => 'Include NSFW', + 'type' => 'checkbox' + ] + ] + ]; + + public function collectData() + { + if ($this->queriedContext === 'Image Search') { + $opt = [ + 'text' => $this->getInput('text'), + 'optional_tags' => [], + 'negative_tags' => [], + 'required_tags' => [], + 'order' => $this->getInput('order'), + 'range' => $this->getInput('range'), + 'video_only' => $this->getInput('video_only'), + 'rating_s' => $this->getInput('rating_s'), + 'rating_q' => $this->getInput('rating_q'), + 'rating_e' => $this->getInput('rating_e') + ]; + + $tag_arr = explode(' ', $this->getInput('tags')); + foreach ($tag_arr as $str) { + switch ($str[0]) { + case '-': + $opt['negative_tags'][] = substr($str, 1); + break; + + case '~': + $opt['optional_tags'][] = substr($str, 1); + break; + + default: + $opt['required_tags'][] = substr($str, 1); + break; + } + } + + $data = $this->getImagesSearch($opt); + + foreach ($data['results'] as $record) { + $item = $this->getImage($record['id']); + $this->addItem($item); + } + } + + if ($this->queriedContext === 'Post Search') { + $opt = [ + 'optional_tags' => [], + 'negative_tags' => [], + 'required_tags' => [], + 'order' => $this->getInput('order'), + 'range' => $this->getInput('range'), + 'text_only' => $this->getInput('text_only'), + 'rating_s' => $this->getInput('rating_s'), + 'rating_q' => $this->getInput('rating_q'), + 'rating_e' => $this->getInput('rating_e') + ]; + + $tag_arr = explode(' ', $this->getInput('tags')); + foreach ($tag_arr as $str) { + switch ($str[0]) { + case '-': + $opt['negative_tags'][] = substr($str, 1); + break; + + case '~': + $opt['optional_tags'][] = substr($str, 1); + break; + + default: + $opt['required_tags'][] = substr($str, 1); + break; + } + } + + $data = $this->getPostsSearch($opt); + + foreach ($data['results'] as $record) { + $item = $this->getPost($record['id'], $record); + $this->addItem($item); + } + } + + if ( + $this->queriedContext === 'User profile' + || $this->queriedContext === 'Home feed' + ) { + $opt = [ + 'reshares' => $this->getInput('reshares'), + 'rating_s' => $this->getInput('rating_s'), + 'rating_q' => $this->getInput('rating_q'), + 'rating_e' => $this->getInput('rating_e') + ]; + + if ($this->queriedContext === 'User profile') { + $opt['order'] = '-date_added'; + $opt['range'] = ''; + $user_id = $this->getInput('user_id') ?? $this->getOwnerID($this->getInput('user')); + + $data = $this->getFeed( + $opt, + $user_id + ); + } + + if ($this->queriedContext === 'Home feed') { + $opt['order'] = $this->getInput('order'); + $opt['range'] = $this->getInput('range'); + $data = $this->getFeed($opt); + } + + foreach ($data['results'] as $record) { + switch ($record['content_type']) { + case 'reshare': + //get type of reshare and its id + $id = $record['content_object']['content_object']['id']; + switch ($record['content_object']['content_type']) { + case 'galleryimage': + $item = $this->getImage($id); + $item['title'] = "{$record['owner_username']} shared: {$item['title']}"; + break; + + case 'commission': + $item = $this->getCommission($id, $record['content_object']['content_object']); + $item['title'] = "{$record['owner_username']} shared: {$item['title']}"; + break; + + case 'post': + $item = $this->getPost($id, $record['content_object']['content_object']); + $item['title'] = "{$record['owner_username']} shared: {$item['title']}"; + break; + }; + break; + case 'galleryimage': + $item = $this->getImage($record['content_object']['id']); + break; + + case 'commission': + $item = $this->getCommission($record['content_object']['id'], $record['content_object']); + break; + + case 'post': + $item = $this->getPost($record['content_object']['id'], $record['content_object']); + break; + } + + $this->addItem($item); + } + } + } + + public function getName() + { + return self::NAME; + } + + public function getURI() + { + return self::URI; + } + + private function getImagesSearch(array $opt) + { + $url = self::URI . "/api/galleries/images/?by_following=false&date_range={$opt['range']}&ordering={$opt['order']}&is_video={$opt['video_only']}"; + $url .= "&text={$opt['text']}&visibility=PUBLIC&visibility=PROFILE_ONLY&page=1&page_size=30&format=json"; + + if (sizeof($opt['optional_tags']) > 0) { + foreach ($opt['optional_tags'] as $tag) { + $url .= "&optional_tags=$tag"; + } + } + if (sizeof($opt['negative_tags']) > 0) { + foreach ($opt['negative_tags'] as $tag) { + $url .= "&negative_tags=$tag"; + } + } + if (sizeof($opt['required_tags']) > 0) { + foreach ($opt['required_tags'] as $tag) { + $url .= "&required_tags=$tag"; + } + } + if ($opt['rating_s']) { + $url .= '&maturity_rating=SFW'; + } + if ($opt['rating_q']) { + $url .= '&maturity_rating=Questionable'; + } + if ($opt['rating_e']) { + $url .= '&maturity_rating=NSFW'; + } + + return $this->getData($url, false, true); + } + + + private function getPostsSearch(array $opt) + { + $url = self::URI . "/api/posts/?by_following=false&date_range={$opt['range']}&ordering={$opt['order']}"; + $url .= '&visibility=PUBLIC&visibility=PROFILE_ONLY&page=1&page_size=30&format=json'; + + if (sizeof($opt['optional_tags']) > 0) { + foreach ($opt['optional_tags'] as $tag) { + $url .= "&optional_tags=$tag"; + } + } + if (sizeof($opt['negative_tags']) > 0) { + foreach ($opt['negative_tags'] as $tag) { + $url .= "&negative_tags=$tag"; + } + } + if (sizeof($opt['required_tags']) > 0) { + foreach ($opt['required_tags'] as $tag) { + $url .= "&required_tags=$tag"; + } + } + if ($opt['rating_s']) { + $url .= '&maturity_rating=SFW'; + } + if ($opt['rating_q']) { + $url .= '&maturity_rating=Questionable'; + } + if ($opt['rating_e']) { + $url .= '&maturity_rating=NSFW'; + } + + return $this->getData($url, false, true); + } + + private function getFeed(array $opt, $ownerID = null) + { + $url = self::URI . "/api/feed/?date_range={$opt['range']}&ordering={$opt['order']}&page=1&page_size=30&format=json"; + + if (is_null($ownerID)) { + $url .= '&visibility=PUBLIC&by_following=false'; + } else { + $url .= "&owner={$ownerID}"; + } + + if (!$opt['reshares']) { + $url .= '&hide_reshares=true'; + } + if ($opt['rating_s']) { + $url .= '&maturity_rating=SFW'; + } + if ($opt['rating_q']) { + $url .= '&maturity_rating=Questionable'; + } + if ($opt['rating_e']) { + $url .= '&maturity_rating=NSFW'; + } + + return $this->getData($url, false, true); + } + + private function getOwnerID($username) + { + $url = self::URI . "/api/user_profiles/{$username}/?format=json"; + $data = $this->getData($url, true, true) + or returnServerError("Could not load $url"); + + return $data['owner']; + } + + private function getPost($id, array $metadata = null) + { + $uri = self::URI . '/posts/' . $id; + $url = self::URI . '/api/posts/' . $id . '/?format=json'; + $data = $metadata ?? $this->getData($url, true, true) + or returnServerError("Could not load $url"); + + $content_str = nl2br($data['content']); + $content = "<p>{$content_str}</p><br/>"; //TODO: Add link and itaku user mention detection and convert into links. + + if (array_key_exists('tags', $data) && sizeof($data['tags']) > 0) { + $tag_types = [ + 'ARTIST' => '', + 'COPYRIGHT' => '', + 'CHARACTER' => '', + 'SPECIES' => '', + 'GENERAL' => '', + 'META' => '' + ]; + foreach ($data['tags'] as $tag) { + $url = self::URI . '/tags/' . $tag['id']; + $str = "<a href=\"{$url}\">#{$tag['name']}</a> "; + $tag_types[$tag['tag_type']] .= $str; + } + + foreach ($tag_types as $type => $str) { + if (strlen($str) > 0) { + $content .= "🏷 <b>{$type}:</b> {$str}<br/>"; + } + } + } + + if (sizeof($data['folders']) > 0) { + $content .= '📁 In Folder(s): '; + foreach ($data['folders'] as $folder) { + $url = self::URI . '/profile/' . $data['owner_username'] . '/posts/' . $folder['id']; + $content .= "<a href=\"{$url}\">#{$folder['title']}</a> "; + } + } + + $content .= '<hr/>'; + if (sizeof($data['gallery_images']) > 0) { + foreach ($data['gallery_images'] as $media) { + $title = $media['title']; + $url = self::URI . '/images/' . $media['id']; + $src = $media['image_xl']; + $content .= '<p>'; + $content .= "<a href=\"{$url}\"><b>{$title}</b></a><br/>"; + if ($media['is_thumbnail_for_video']) { + $url = self::URI . '/api/galleries/images/' . $media['id'] . '/?format=json'; + $media_data = $this->getData($url, true, true) + or returnServerError("Could not load $url"); + $content .= "<video controls src=\"{$media_data['video']['video']}\" poster=\"{$media['image_xl']}\"/>"; + } else { + $content .= "<a href=\"{$url}\"><img src=\"{$src}\"></a>"; + } + $content .= '</p><br/>'; + } + } + + return [ + 'uri' => $uri, + 'title' => $data['title'], + 'timestamp' => $data['date_added'], + 'author' => $data['owner_username'], + 'content' => $content, + 'categories' => ['post'], + 'uid' => $uri + ]; + } + + private function getCommission($id, array $metadata = null) + { + $url = self::URI . '/api/commissions/' . $id . '/?format=json'; + $uri = self::URI . '/commissions/' . $id; + // Debug::log(var_dump($metadata)); + $data = $metadata ?? $this->getData($url, true, true) + or returnServerError("Could not load $url"); + + $content_str = nl2br($data['description']); + $content = "<p>{$content_str}</p><br>"; //TODO: Add link and itaku user mention detection and convert into links. + + if (array_key_exists('tags', $data) && sizeof($data['tags']) > 0) { + // $content .= "🏷 Tag(s): "; + $tag_types = [ + 'ARTIST' => '', + 'COPYRIGHT' => '', + 'CHARACTER' => '', + 'SPECIES' => '', + 'GENERAL' => '', + 'META' => '' + ]; + foreach ($data['tags'] as $tag) { + $url = self::URI . '/tags/' . $tag['id']; + $str = "<a href=\"{$url}\">#{$tag['name']}</a> "; + $tag_types[$tag['tag_type']] .= $str; + } + + foreach ($tag_types as $type => $str) { + if (strlen($str) > 0) { + $content .= "🏷 <b>{$type}:</b> {$str}<br/>"; + } + } + } + + if (array_key_exists('reference_gallery_sections', $data) && sizeof($data['reference_gallery_sections']) > 0) { + $content .= '📁 Example folder(s): '; + foreach ($data['folders'] as $folder) { + $url = self::URI . '/profile/' . $data['owner_username'] . '/gallery/' . $folder['id']; + $folder_name = $folder['title']; + if (!is_null($folder['group'])) { + $folder_name = $folder['group']['title'] . '/' . $folder_name; + } + $content .= "<a href=\"{$url}\">#{$folder_name}</a> "; + } + } + + $content .= '<hr/>'; + if (!is_null($data['thumbnail_detail'])) { + $content .= '<p>'; + $content .= "<a href=\"{$uri}\"><b>{$data['thumbnail_detail']['title']}</b></a><br/>"; + if ($data['thumbnail_detail']['is_thumbnail_for_video']) { + $url = self::URI . '/api/galleries/images/' . $data['thumbnail_detail']['id'] . '/?format=json'; + $media_data = $this->getData($url, true, true) + or returnServerError("Could not load $url"); + $content .= "<video controls src=\"{$media_data['video']['video']}\" poster=\"{$data['thumbnail_detail']['image_lg']}\"/>"; + } else { + $content .= "<a href=\"{$uri}\"><img src=\"{$data['thumbnail_detail']['image_lg']}\"></a>"; + } + + $content .= '</p>'; + } + + return [ + 'uri' => $uri, + 'title' => "{$data['comm_type']}: {$data['title']}", + 'timestamp' => $data['date_added'], + 'author' => $data['owner_username'], + 'content' => $content, + 'categories' => ['commission', $data['comm_type']], + 'uid' => $uri + ]; + } + + private function getImage($id /* array $metadata = null */) //$metadata disabled due to no essential information available in ./api/feed/ or ./api/galleries/images/ results. + { + $uri = self::URI . '/images/' . $id; + $url = self::URI . '/api/galleries/images/' . $id . '/?format=json'; + $data = /* $metadata ?? */ $this->getData($url, true, true) + or returnServerError("Could not load $url"); + + $content_str = nl2br($data['description']); + $content = "<p>{$content_str}</p><br/>"; //TODO: Add link and itaku user mention detection and convert into links. + + if (array_key_exists('tags', $data) && sizeof($data['tags']) > 0) { + // $content .= "🏷 Tag(s): "; + $tag_types = [ + 'ARTIST' => '', + 'COPYRIGHT' => '', + 'CHARACTER' => '', + 'SPECIES' => '', + 'GENERAL' => '', + 'META' => '' + ]; + foreach ($data['tags'] as $tag) { + $url = self::URI . '/tags/' . $tag['id']; + $str = "<a href=\"{$url}\">#{$tag['name']}</a> "; + $tag_types[$tag['tag_type']] .= $str; + } + + foreach ($tag_types as $type => $str) { + if (strlen($str) > 0) { + $content .= "🏷 <b>{$type}:</b> {$str}<br/>"; + } + } + } + + if (array_key_exists('sections', $data) && sizeof($data['sections']) > 0) { + $content .= '📁 In Folder(s): '; + foreach ($data['sections'] as $folder) { + $url = self::URI . '/profile/' . $data['owner_username'] . '/gallery/' . $folder['id']; + $folder_name = $folder['title']; + if (!is_null($folder['group'])) { + $folder_name = $folder['group']['title'] . '/' . $folder_name; + } + $content .= "<a href=\"{$url}\">#{$folder_name}</a> "; + } + } + + $content .= '<hr/>'; + + if (array_key_exists('is_thumbnail_for_video', $data)) { + $url = self::URI . '/api/galleries/images/' . $data['id'] . '/?format=json'; + $media_data = $this->getData($url, true, true) + or returnServerError("Could not load $url"); + $content .= "<video controls src=\"{$media_data['video']['video']}\" poster=\"{$data['image_xl']}\"/>"; + } else { + if (array_key_exists('video', $data) && is_null($data['video'])) { + $content .= "<a href=\"{$uri}\"><img src=\"{$data['image_xl']}\"></a>"; + } else { + $content .= "<video controls src=\"{$data['video']['video']}\" poster=\"{$data['image_xl']}\"/>"; + } + } + + return [ + 'uri' => $uri, + 'title' => $data['title'], + 'timestamp' => $data['date_added'], + 'author' => $data['owner_username'], + 'content' => $content, + 'categories' => ['image'], + 'uid' => $uri + ]; + } + + private function getData(string $url, bool $cache = false, bool $getJSON = false, array $httpHeaders = [], array $curlOptions = []) + { + // Debug::log($url); + if ($getJSON) { //get JSON object + if ($cache) { + $data = $this->loadCacheValue($url, 86400); // 24 hours + if (is_null($data)) { + $data = getContents($url, $httpHeaders, $curlOptions) or returnServerError("Could not load $url"); + $this->saveCacheValue($url, $data); + } + } else { + $data = getContents($url, $httpHeaders, $curlOptions) or returnServerError("Could not load $url"); + } + return json_decode($data, true); + } else { //get simpleHTMLDOM object + if ($cache) { + $html = getSimpleHTMLDOMCached($url, 86400); // 24 hours + } else { + $html = getSimpleHTMLDOM($url); + } + $html = defaultLinkTo($html, $url); + return $html; + } + } + + private function addItem($item) + { + if (is_null($item)) { + return; + } + + if (is_array($item) || is_object($item)) { + $this->items[] = $item; + } else { + returnServerError("Incorrectly parsed item. Check the code!\nType: " . gettype($item) . "\nprint_r(item:)\n" . var_dump($item)); + } + } +} From 28077155ca9343c6492fac36fc71078a6633e984 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Tue, 15 Aug 2023 23:34:09 +0930 Subject: [PATCH 069/716] [PatreonBridge] Resolve creator name in feed name (#3616) * resolve creators without custom url * hint how to enter creator with non-custom url * applied phpcbf --- bridges/PatreonBridge.php | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bridges/PatreonBridge.php b/bridges/PatreonBridge.php index b64102da..23dbf8ae 100644 --- a/bridges/PatreonBridge.php +++ b/bridges/PatreonBridge.php @@ -6,13 +6,13 @@ class PatreonBridge extends BridgeAbstract const URI = 'https://www.patreon.com/'; const CACHE_TIMEOUT = 300; // 5min const DESCRIPTION = 'Returns posts by creators on Patreon'; - const MAINTAINER = 'Roliga'; - const PARAMETERS = [ [ + const MAINTAINER = 'Roliga, mruac'; + const PARAMETERS = [[ 'creator' => [ 'name' => 'Creator', 'type' => 'text', 'required' => true, - 'exampleValue' => 'sanityinc', + 'exampleValue' => 'user?u=13425451', 'title' => 'Creator name as seen in their page URL' ] ]]; @@ -189,7 +189,13 @@ class PatreonBridge extends BridgeAbstract public function getName() { if (!is_null($this->getInput('creator'))) { - return $this->getInput('creator') . ' posts'; + $html = getSimpleHTMLDOMCached($this->getURI()); + if ($html) { + preg_match('#"name": "(.*)"#', $html->save(), $matches); + return 'Patreon posts from ' . stripcslashes($matches[1]); + } else { + return $this->getInput('creator') . 'posts from Patreon'; + } } return parent::getName(); From a1237d90f10539dcea9d3efdbff5dd5472b1928b Mon Sep 17 00:00:00 2001 From: Corentin Garcia <corenting@gmail.com> Date: Tue, 15 Aug 2023 16:16:06 +0200 Subject: [PATCH 070/716] [RainbowSixSiegeBridge] fix links, date and img tag (#3619) --- bridges/RainbowSixSiegeBridge.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bridges/RainbowSixSiegeBridge.php b/bridges/RainbowSixSiegeBridge.php index 73e2bdc4..77495a3c 100644 --- a/bridges/RainbowSixSiegeBridge.php +++ b/bridges/RainbowSixSiegeBridge.php @@ -9,7 +9,7 @@ class RainbowSixSiegeBridge extends BridgeAbstract const DESCRIPTION = 'Latest news about Rainbow Six Siege'; // API key to call Ubisoft API, extracted from the React frontend - const NIMBUS_API_KEY = '3u0FfSBUaTSew-2NVfAOSYWevVQHWtY9q3VM8Xx9Lto'; + const NIMBUS_API_KEY = '3b5a8be6dde511ec9d640242ac120002'; public function getIcon() { @@ -32,18 +32,23 @@ class RainbowSixSiegeBridge extends BridgeAbstract for ($i = 0; $i < count($json); $i++) { $jsonItem = $json[$i]; - $uri = 'https://www.ubisoft.com/en-us/game/rainbow-six/siege'; + $uri = 'https://www.ubisoft.com/en-us/game/rainbow-six/siege/news-updates'; $uri = $uri . $jsonItem['button']['buttonUrl']; - $thumbnail = '<img src="' . $jsonItem['thumbnail']['url'] . '" alt="Thumbnail">'; + $thumbnail = '<img src="' . $jsonItem['thumbnail']['url'] . '" alt="Thumbnail" />'; $content = $thumbnail . '<br />' . markdownToHtml($jsonItem['content']); $item = []; + + // The date string includes (Coordinated Universal Time) at the end + // so remove it to use strtotime + $date_str = str_replace('(Coordinated Universal Time)', '', $jsonItem['date']); + $item['timestamp'] = strtotime($date_str); + $item['uri'] = $uri; $item['id'] = $jsonItem['id']; $item['title'] = $jsonItem['title']; $item['content'] = $content; - $item['timestamp'] = strtotime($jsonItem['date']); $this->items[] = $item; } From 79e3f7f20474c2fda99640ffa69f94fd05d6d991 Mon Sep 17 00:00:00 2001 From: John S Long <john@128.io> Date: Sat, 19 Aug 2023 21:37:21 -0500 Subject: [PATCH 071/716] [MastodonBridge] Add support for excluding regular statuses (non-boosts/replies) (#3624) * [MastodonBridge]: add support for excluding posts (non-boosts/replies) * Update name of input * Fix lint failures --- bridges/MastodonBridge.php | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index 372838ca..855aae08 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -35,6 +35,11 @@ class MastodonBridge extends BridgeAbstract 'exampleValue' => '@sebsauvage@framapiaf.org', 'required' => true, ], + 'noregular' => [ + 'name' => 'Without regular statuses', + 'type' => 'checkbox', + 'title' => 'Hide regular statuses (i.e. non-boosts, replies, etc.)', + ], 'norep' => [ 'name' => 'Without replies', 'type' => 'checkbox', @@ -61,6 +66,10 @@ class MastodonBridge extends BridgeAbstract public function collectData() { + if ($this->getInput('norep') && $this->getInput('noboost') && $this->getInput('noregular')) { + throw new \Exception('replies, boosts, or regular statuses must be allowed'); + } + $user = $this->fetchAP($this->getURI()); if (!isset($user['outbox'])) { throw new \Exception('Unable to find the outbox'); @@ -115,6 +124,9 @@ class MastodonBridge extends BridgeAbstract if ($this->getInput('norep') && isset($content['inReplyTo'])) { return null; } + if ($this->getInput('noregular') && !isset($content['inReplyTo'])) { + return null; + } $item['title'] = ''; $item['author'] = $this->getInput('canusername'); $item = $this->parseObject($content, $item); @@ -123,6 +135,9 @@ class MastodonBridge extends BridgeAbstract if ($this->getInput('norep') && isset($content['object']['inReplyTo'])) { return null; } + if ($this->getInput('noregular') && !isset($content['object']['inReplyTo'])) { + return null; + } $item['title'] = ''; $item['author'] = $this->getInput('canusername'); $item = $this->parseObject($content['object'], $item); @@ -147,7 +162,7 @@ class MastodonBridge extends BridgeAbstract if (isset($object['name'])) { $item['title'] = $object['name']; - } else if (mb_strlen($strippedContent) > 75) { + } elseif (mb_strlen($strippedContent) > 75) { $contentSubstring = mb_substr($strippedContent, 0, mb_strpos(wordwrap($strippedContent, 75), "\n")); $item['title'] .= $contentSubstring . '...'; } else { From 959dd937b4b5c6bc9f37c659652900986c123f69 Mon Sep 17 00:00:00 2001 From: Eugene Molotov <eugene.molotov@yandex.ru> Date: Mon, 21 Aug 2023 07:53:54 +0500 Subject: [PATCH 072/716] [VkBridge] Using more universal regular expression to generate item title (#3627) --- bridges/VkBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 8c18f26a..90b2586e 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -453,7 +453,7 @@ class VkBridge extends BridgeAbstract { $content = explode('<br>', $content)[0]; $content = strip_tags($content); - preg_match('/^[:,"\w\ \p{L}\(\)\?#«»-]+/mu', htmlspecialchars_decode($content), $result); + preg_match('/.+?(?=[\.\n])/mu', htmlspecialchars_decode($content), $result); if (count($result) == 0) { return 'untitled'; } From c5cbab123120e243f091a20644b4de16d068667e Mon Sep 17 00:00:00 2001 From: veloute <21003408+veloute@users.noreply.github.com> Date: Tue, 22 Aug 2023 15:59:06 +0000 Subject: [PATCH 073/716] Update TheGuardianBridge.php (#3629) --- bridges/TheGuardianBridge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/bridges/TheGuardianBridge.php b/bridges/TheGuardianBridge.php index 2e14de7a..e05bde75 100644 --- a/bridges/TheGuardianBridge.php +++ b/bridges/TheGuardianBridge.php @@ -15,6 +15,7 @@ class TheGuardianBridge extends FeedExpander 'World News' => 'world/rss', 'US News' => '/us-news/rss', 'UK News' => '/uk-news/rss', + 'Australia News' => '/australia-news/rss', 'Europe News' => '/world/europe-news/rss', 'Asia News' => '/world/asia/rss', 'Tech' => '/uk/technology/rss', From 3ac861a86681b480fd194e102713044c485c57c0 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 22 Aug 2023 19:47:32 +0200 Subject: [PATCH 074/716] fix(twitch): Invalid argument supplied for foreach() at bridges/TwitchBridge.php line 115 (#3630) --- bridges/TwitchBridge.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index 8976174a..146fed3d 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -112,7 +112,9 @@ EOD; if (!is_null($video->game)) { $item['categories'][] = $video->game->displayName; } - foreach ($video->contentTags as $tag) { + + $contentTags = $video->contentTags ?? []; + foreach ($contentTags as $tag) { if (!$tag->isLanguageTag) { $item['categories'][] = $tag->localizedName; } From 54045be951cf7deab15535fccd93dc89998150d8 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 22 Aug 2023 20:06:16 +0200 Subject: [PATCH 075/716] fix(tpb): add missing cat (#3631) --- bridges/ThePirateBayBridge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/bridges/ThePirateBayBridge.php b/bridges/ThePirateBayBridge.php index 13095062..5b305c82 100644 --- a/bridges/ThePirateBayBridge.php +++ b/bridges/ThePirateBayBridge.php @@ -65,6 +65,7 @@ class ThePirateBayBridge extends BridgeAbstract '207' => 'HD Movies', '208' => 'HD TV-Shows', '209' => '3D', + '212' => 'UHD/4k TV-Shows', '299' => 'Other', '301' => 'Windows', '302' => 'Mac/Apple', From 7591b10219222f818d523f457fa6d01e1891260b Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Tue, 22 Aug 2023 20:44:36 +0200 Subject: [PATCH 076/716] [Core] New feature : User Interface to "Detect" Feed from an URL (#3436) * [Core] New feature : User Interface to "Detect" Feed from an URL Detect Action has been expanded to support returning a Feed in a JSON format instead of a Redirect. Existing usage of the Detect action will keep working as usual. Frontpage template has now a section to display the Feed detection result, and a button to start the Feed Detection. A new JS file contains the necessary JS (Ajax and Event management) to fill the Feed Detection section. * Coding policy fixes * [Core] New feature : User Interface to "Detect" Feed from an URL - Switch from old school XMLHttpRequest to fetch - Enhance UX of search results - Revert to it's original content - Switch to a new Action : FindfeedAction.php - Switch to template literals instead of string concatenation - FindFeed action could retrun multiple feeds - Results are sent with an absolute URL - Switch to Json::encode() helper function * [Core] New feature : User Interface to "Detect" Feed from an URL - Move specific JS code to rss-bridge.js - Change HTML tag for the button to have a consistant style with th rest of the page * [Core] New feature : User Interface to "Detect" Feed from an URL - If no context is sent, assume there is only one unnamed context - Find parameter name in global and currect context * fix * remove typo --------- Co-authored-by: Dag <me@dvikan.no> --- actions/FindfeedAction.php | 89 ++++++++++++++++++++++++++++++++++++ static/rss-bridge.js | 79 ++++++++++++++++++++++++++++++++ static/style.css | 35 ++++++++++++++ templates/frontpage.html.php | 9 ++++ 4 files changed, 212 insertions(+) create mode 100644 actions/FindfeedAction.php diff --git a/actions/FindfeedAction.php b/actions/FindfeedAction.php new file mode 100644 index 00000000..25fe4714 --- /dev/null +++ b/actions/FindfeedAction.php @@ -0,0 +1,89 @@ +<?php + +/** + * This action is used by the frontpage form search. + * It finds a bridge based off of a user input url. + * It uses bridges' detectParameters implementation. + */ +class FindfeedAction implements ActionInterface +{ + public function execute(array $request) + { + $targetURL = $request['url'] ?? null; + $format = $request['format'] ?? null; + + if (!$targetURL) { + return new Response('You must specify a url', 400); + } + if (!$format) { + return new Response('You must specify a format', 400); + } + + $bridgeFactory = new BridgeFactory(); + + $results = []; + foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) { + if (!$bridgeFactory->isEnabled($bridgeClassName)) { + continue; + } + + $bridge = $bridgeFactory->create($bridgeClassName); + + $bridgeParams = $bridge->detectParameters($targetURL); + + if ($bridgeParams === null) { + continue; + } + + // It's allowed to have no 'context' in a bridge (only a default context without any name) + // In this case, the reference to the parameters are found in the first element of the PARAMETERS array + + $context = $bridgeParams['context'] ?? 0; + + $bridgeData = []; + // Construct the array of parameters + foreach ($bridgeParams as $key => $value) { + // 'context' is a special case : it's a bridge parameters, there is no "name" for this parameter + if ($key == 'context') { + $bridgeData[$key]['name'] = 'Context'; + $bridgeData[$key]['value'] = $value; + } else { + $bridgeData[$key]['name'] = $this->getParameterName($bridge, $context, $key); + $bridgeData[$key]['value'] = $value; + } + } + + $bridgeParams['bridge'] = $bridgeClassName; + $bridgeParams['format'] = $format; + $content = [ + 'url' => get_home_page_url() . '?action=display&' . http_build_query($bridgeParams), + 'bridgeParams' => $bridgeParams, + 'bridgeData' => $bridgeData, + 'bridgeMeta' => [ + 'name' => $bridge::NAME, + 'description' => $bridge::DESCRIPTION, + 'parameters' => $bridge::PARAMETERS, + 'icon' => $bridge->getIcon(), + ], + ]; + $results[] = $content; + } + if ($results === []) { + return new Response(Json::encode(['message' => 'No bridge found for given url']), 404, ['content-type' => 'application/json']); + } + return new Response(Json::encode($results), 200, ['content-type' => 'application/json']); + } + + // Get parameter name in the actual context, or in the global parameter + private function getParameterName($bridge, $context, $key) + { + if (isset($bridge::PARAMETERS[$context][$key]['name'])) { + $name = $bridge::PARAMETERS[$context][$key]['name']; + } else if (isset($bridge::PARAMETERS['global'][$key]['name'])) { + $name = $bridge::PARAMETERS['global'][$key]['name']; + } else { + $name = 'Variable "' . $key . '" (No name provided)'; + } + return $name; + } +} diff --git a/static/rss-bridge.js b/static/rss-bridge.js index 498acd37..82069d8c 100644 --- a/static/rss-bridge.js +++ b/static/rss-bridge.js @@ -47,3 +47,82 @@ function rssbridge_toggle_bridge(){ bridge.getElementsByClassName('showmore-box')[0].checked = true; } } + +var rssbridge_feed_finder = (function() { + /* + * Code for "Find feed by URL" feature + */ + + // Start the Feed search + async function rssbridge_feed_search(event) { + const input = document.getElementById('searchfield'); + let content = input.value; + if (content) { + const findfeedresults = document.getElementById('findfeedresults'); + findfeedresults.innerHTML = 'Searching for matching feeds ...'; + let baseurl = window.location.protocol + window.location.pathname; + let url = baseurl + '?action=findfeed&format=Html&url=' + content; + const response = await fetch(url); + if (response.ok) { + const data = await response.json(); + rss_bridge_feed_display_found_feed(data); + } else { + rss_bridge_feed_display_feed_search_fail(); + } + } else { + rss_bridge_feed_display_find_feed_empty(); + } + } + + // Display the found feeds + function rss_bridge_feed_display_found_feed(obj) { + const findfeedresults = document.getElementById('findfeedresults'); + + let content = 'Found Feed(s) :'; + + // Let's go throug every Feed found + for (const element of obj) { + content += `<div class="search-result"> + <div class="icon"> + <img src="${element.bridgeMeta.icon}" width="60" /> + </div> + <div class="content"> + <h2><a href="${element.url}">${element.bridgeMeta.name}</a></h2> + <p> + <span class="description"><a href="${element.url}">${element.bridgeMeta.description}</a></span> + </p> + <div> + <ul>`; + + // Now display every Feed parameter + for (const param in element.bridgeData) { + content += `<li>${element.bridgeData[param].name} : ${element.bridgeData[param].value}</li>`; + } + content += `</div> + </div> + </div>`; + } + content += '<p><div class="alert alert-info" role="alert">This feed may be only one of the possible feeds. You may find more feeds using one of the bridges with different parameters, for example.</div></p>'; + findfeedresults.innerHTML = content; + } + + // Display an error if no feed were found + function rss_bridge_feed_display_feed_search_fail() { + const findfeedresults = document.getElementById('findfeedresults'); + findfeedresults.innerHTML = 'No Feed found !<div class="alert alert-info" role="alert">Not every bridge supports feed detection. You can check below within the bridge parameters to create a feed.</div>'; + } + + // Empty the Found Feed section + function rss_bridge_feed_display_find_feed_empty() { + const findfeedresults = document.getElementById('findfeedresults'); + findfeedresults.innerHTML = ''; + } + + // Add Event to 'Detect Feed" button + var rssbridge_feed_finder = function() { + const button = document.getElementById('findfeed'); + button.addEventListener("click", rssbridge_feed_search); + button.addEventListener("keyup", rssbridge_feed_search); + }; + return rssbridge_feed_finder; +}()); diff --git a/static/style.css b/static/style.css index a83e25e4..a9d5933a 100644 --- a/static/style.css +++ b/static/style.css @@ -453,3 +453,38 @@ button { color: #d8d3cb; } } + +/* find-feed */ +.search-result { + background-color: #f0f0f0; + border-radius: 5px; + padding: 15px; + display: flex; + position: relative; + text-align: left; +} +@media (prefers-color-scheme: dark) { + .search-result { + background-color: #202325; + } +} +.search-result h2 { + color: #288cfc; +} + +.search-result a { + text-decoration: none; + color: #248afa; +} +.search-result .icon { + margin: 0 15px 0 0; +} +.search-result span { + margin-right: 10px; +} +.search-result .description { + font-size: 110%; + margin-right: 0 !important; + margin-top: 5px !important; +} +/* end find-feed */ diff --git a/templates/frontpage.html.php b/templates/frontpage.html.php index 63f4a2ab..99e2ffd9 100644 --- a/templates/frontpage.html.php +++ b/templates/frontpage.html.php @@ -2,6 +2,7 @@ <script> document.addEventListener('DOMContentLoaded', rssbridge_toggle_bridge); document.addEventListener('DOMContentLoaded', rssbridge_list_search); + document.addEventListener('DOMContentLoaded', rssbridge_feed_finder); </script> <section class="searchbar"> @@ -15,6 +16,14 @@ onkeyup="rssbridge_list_search()" value="" > + <button + type="button" + id="findfeed" + name="findfeed" + />Find Feed from URL</button> + <section id="findfeedresults"> + </section> + </section> <?= raw($bridges) ?> From eb4ff7099f07859a2a3078c50a7d5366ca722bb1 Mon Sep 17 00:00:00 2001 From: Lars Stegman <LarsStegman@users.noreply.github.com> Date: Tue, 22 Aug 2023 21:28:16 +0200 Subject: [PATCH 077/716] CSS Selector Bridge 2 (#3626) * [CssSelector2Bridge] Implement CSS Selector bridge 2 * [CssSelector2Bridge] Fix author not being loaded * [CssSelector2Bridge] Remove unneeded time nullcheck * Fix linting * Fix failing test * Implement PR fixes * Update bridges/CssSelector2Bridge.php Co-authored-by: ORelio <ORelio@users.noreply.github.com> * Rename bridge and fix syntax error for php7 --------- Co-authored-by: ORelio <ORelio@users.noreply.github.com> --- bridges/CssSelectorComplexBridge.php | 458 +++++++++++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100644 bridges/CssSelectorComplexBridge.php diff --git a/bridges/CssSelectorComplexBridge.php b/bridges/CssSelectorComplexBridge.php new file mode 100644 index 00000000..4d44f853 --- /dev/null +++ b/bridges/CssSelectorComplexBridge.php @@ -0,0 +1,458 @@ +<?php + +class CssSelectorComplexBridge extends BridgeAbstract +{ + const MAINTAINER = 'Lars Stegman'; + const NAME = 'CSS Selector Complex Bridge'; + const URI = 'https://github.com/RSS-Bridge/rss-bridge/'; + const DESCRIPTION = <<<EOT + Convert any site to RSS feed using CSS selectors (Advanced Users). The bridge first selects + the element describing the article entries. It then extracts the links to the articles from + these elements. It then, depending on the setting "Load article from page", either parses + the selected elements, or downloads the page for each article and parses those. Parsing the + elements or page is done using the provided selectors. + EOT; + const PARAMETERS = [ + [ + 'home_page' => [ + 'name' => 'Site URL: Page with latest articles', + 'exampleValue' => 'https://example.com/blog/', + 'required' => true + ], + 'cookie' => [ + 'name' => '[Optional] Cookie', + 'title' => <<<EOT + Use when the website does not send the page contents, unless a static cookie is included. + EOT, + 'exampleValue' => 'sessionId=deadb33f' + ], + 'title_cleanup' => [ + 'name' => '[Optional] Text to remove from feed title', + 'title' => <<<EOT + Text to remove from the feed title, which is read from the article list page. + EOT, + 'exampleValue' => ' | BlogName', + ], + 'entry_element_selector' => [ + 'name' => 'Selector for article entry elements', + 'title' => <<<EOT + This bridge works using CSS selectors, e.g. "div.article" will match all + <div class="article">...</div> on home page, each one being treated as a feed item. + + Use the URL selector option to select the `a` element with the + `href` to the article link. If this option is not configured, the first encountered + `a` element is used. + EOT, + 'exampleValue' => 'div.article', + 'required' => true + ], + 'url_selector' => [ + 'name' => '[Optional] Selector for link elements', + 'title' => <<<EOT + The selector to find `a` elements in the entry element. If empty, + the first encountered `a` element is used. The `href` property + is used to create entries in the feed. + EOT, + 'exampleValue' => 'a.article', + 'defaultValue' => 'a' + ], + 'url_pattern' => [ + 'name' => '[Optional] Pattern for site URLs to keep in feed', + 'title' => 'Optionally filter items by applying a regular expression on their URL', + 'exampleValue' => '/blog/article/.*', + ], + 'limit' => self::LIMIT, + 'use_article_pages' => [ + 'name' => 'Load article from page', + 'title' => <<<EOT + If true, the article page is load and parsed to get the article contents using + the css selectors. (Slower!) + Otherwise, the element selected by the article entry selector is used. + EOT, + 'type' => 'checkbox' + ], + 'article_page_content_selector' => [ + 'name' => '[Optional] Selector to select article element', + 'title' => 'Extract the article from its page using the provided selector', + 'exampleValue' => 'article.content', + ], + 'content_cleanup' => [ + 'name' => '[Optional] Content cleanup: selector for items to remove', + 'title' => 'Selector for unnecessary elements to remove inside article contents.', + 'exampleValue' => 'div.ads, div.comments', + ], + 'title_selector' => [ + 'name' => '[Optional] Selector for the article title', + 'title' => 'Selector to select the article title', + 'defaultValue' => 'h1' + ], + 'category_selector' => [ + 'name' => '[Optional] Categories', + 'title' => <<<EOT + Selector to extract the catgories the article has + EOT, + 'exampleValue' => 'span.category, #main-category' + ], + 'author_selector' => [ + 'name' => '[Optional] Author', + 'title' => <<<EOT + Selector to extract the author of the article. If multiple elements are selected + the first one is used. + EOT, + 'exampleValue' => 'span#author' + ], + 'time_selector' => [ + 'name' => '[Optional] Time selector', + 'title' => <<<EOT + Selector to extract the timestamp of the article. If the element + is an html5 `time` element, the value for the `datetime` attribute is used. + EOT, + ], + 'time_format' => [ + 'name' => '[Optional] Format string for parsing time', + 'title' => <<<EOT + The format to use to parse the timestamp. See + https://www.php.net/manual/en/datetimeimmutable.createfromformat.php + for the format specification. + EOT + ], + 'remove_styling' => [ + 'name' => '[Optional] Remove styling', + 'title' => 'Remove class and style attributes from the page elements', + 'type' => 'checkbox' + ] + ] + ]; + + private $feedName = ''; + + public function getURI() + { + $url = $this->getInput('home_page'); + if (empty($url)) { + $url = parent::getURI(); + } + return $url; + } + + public function getName() + { + if (!empty($this->feedName)) { + return $this->feedName; + } + return parent::getName(); + } + + protected function getHeaders() + { + $headers = []; + $cookie = $this->getInput('cookie'); + if (!empty($cookie)) { + $headers[] = 'Cookie: ' . $cookie; + } + + return $headers; + } + + public function collectData() + { + $url = $this->getInput('home_page'); + $headers = $this->getHeaders(); + + $entry_element_selector = $this->getInput('entry_element_selector'); + $url_selector = $this->getInput('url_selector'); + $url_pattern = $this->getInput('url_pattern'); + $limit = $this->getInput('limit') ?? 10; + + $use_article_pages = $this->getInput('use_article_pages'); + $article_page_content_selector = $this->getInput('article_page_content_selector'); + $content_cleanup = $this->getInput('content_cleanup'); + $title_selector = $this->getInput('title_selector'); + $title_cleanup = $this->getInput('title_cleanup'); + $time_selector = $this->getInput('time_selector'); + $time_format = $this->getInput('time_format'); + + $category_selector = $this->getInput('category_selector'); + $author_selector = $this->getInput('author_selector'); + $remove_styling = $this->getInput('remove_styling'); + + $html = defaultLinkTo(getSimpleHTMLDOM($url, $headers), $url); + $this->feedName = $this->getTitle($html, $title_cleanup); + $entry_elements = $this->htmlFindEntryElements($html, $entry_element_selector, $url_selector, $url_pattern, $limit); + + if (empty($entry_elements)) { + return; + } + + // Fetch the elements from the article pages. + if ($use_article_pages) { + if (empty($article_page_content_selector)) { + returnClientError('`Article selector` is required when `Load article page` is enabled'); + } + + foreach (array_keys($entry_elements) as $uri) { + $entry_elements[$uri] = $this->fetchArticleElementFromPage($uri, $article_page_content_selector); + } + } + + foreach ($entry_elements as $uri => $element) { + $entry = $this->parseEntryElement( + $element, + $title_selector, + $author_selector, + $category_selector, + $time_selector, + $time_format, + $content_cleanup, + $this->feedName, + $remove_styling + ); + + $entry['uri'] = $uri; + $this->items[] = $entry; + } + } + + /** + * Filter a list of URLs using a pattern and limit + * @param array $links List of URLs + * @param string $url_pattern Pattern to look for in URLs + * @param int $limit Optional maximum amount of URLs to return + * @return array Array of URLs + */ + protected function filterUrlList($links, $url_pattern, $limit = 0) + { + if (!empty($url_pattern)) { + $url_pattern = '/' . str_replace('/', '\/', $url_pattern) . '/'; + $links = array_filter($links, function ($url) { + return preg_match($url_pattern, $url) === 1; + }); + } + + if ($limit > 0 && count($links) > $limit) { + $links = array_slice($links, 0, $limit); + } + + return $links; + } + + /** + * Retrieve title from webpage URL or DOM + * @param string|object $page URL or DOM to retrieve title from + * @param string $title_cleanup optional string to remove from webpage title, e.g. " | BlogName" + * @return string Webpage title + */ + protected function getTitle($page, $title_cleanup) + { + if (is_string($page)) { + $page = getSimpleHTMLDOMCached($page); + } + $title = html_entity_decode($page->find('title', 0)->plaintext); + if (!empty($title)) { + $title = trim(str_replace($title_cleanup, '', $title)); + } + + return $title; + } + + /** + * Remove all elements from HTML content matching cleanup selector + * @param string|object $content HTML content as HTML object or string + * @return string|object Cleaned content (same type as input) + */ + protected function cleanArticleContent($content, $cleanup_selector, $remove_styling) + { + $string_convert = false; + if (is_string($content)) { + $string_convert = true; + $content = str_get_html($content); + } + + if (!empty($cleanup_selector)) { + foreach ($content->find($cleanup_selector) as $item_to_clean) { + $item_to_clean->outertext = ''; + } + } + + if ($remove_styling) { + foreach (['class', 'style'] as $attribute_to_remove) { + foreach ($content->find('[' . $attribute_to_remove . ']') as $item_to_clean) { + $item_to_clean->removeAttribute($attribute_to_remove); + } + } + } + + if ($string_convert) { + $content = $content->outertext; + } + return $content; + } + + + /** + * Retrieve first N link+element from webpage URL or DOM satisfying the specified criteria + * @param string|object $page URL or DOM to retrieve feed items from + * @param string $entry_selector DOM selector for matching HTML elements that contain article + * entries + * @param string $url_selector DOM selector for matching links + * @param string $url_pattern Optional filter to keep only links matching the pattern + * @param int $limit Optional maximum amount of URLs to return + * @return array of items { <uri> => <html-element> } + */ + protected function htmlFindEntryElements($page, $entry_selector, $url_selector, $url_pattern = '', $limit = 0) + { + if (is_string($page)) { + $page = getSimpleHTMLDOM($page); + } + + $entryElements = $page->find($entry_selector); + if (empty($entryElements)) { + returnClientError('No entry elements for entry selector'); + } + + // Extract URIs with the associated entry element + $links_with_elements = []; + foreach ($entryElements as $entry) { + $url_element = $entry->find($url_selector, 0); + if (is_null($url_element)) { + // No `a` element found in this entry + if ($entry->tag == 'a') { + $url_element = $entry; + } else { + continue; + } + } + + $links_with_elements[$url_element->href] = $entry; + } + + if (empty($links_with_elements)) { + returnClientError('The provided URL selector matches some elements, but they do not + contain links.'); + } + + // Filter using the URL pattern + $filtered_urls = $this->filterUrlList(array_keys($links_with_elements), $url_pattern, $limit); + + if (empty($filtered_urls)) { + returnClientError('No results for URL pattern'); + } + + $items = []; + foreach ($filtered_urls as $link) { + $items[$link] = $links_with_elements[$link]; + } + + return $items; + } + + + /** + * Retrieve article element from its URL using content selector and return the DOM element + * @param string $entry_url URL to retrieve article from + * @param string $content_selector HTML selector for extracting content, e.g. "article.content" + * @return article DOM element + */ + protected function fetchArticleElementFromPage($entry_url, $content_selector) + { + $entry_html = getSimpleHTMLDOMCached($entry_url); + $article_content = $entry_html->find($content_selector, 0); + + if (is_null($article_content)) { + returnClientError('Could not article content at URL: ' . $entry_url); + } + + $article_content = defaultLinkTo($article_content, $entry_url); + return $article_content; + } + + protected function parseTimeStrAsTimestamp($timeStr, $format) + { + $date = date_parse_from_format($format, $timeStr); + if ($date['error_count'] != 0) { + returnClientError('Error while parsing time string'); + } + + $timestamp = mktime( + $date['hour'], + $date['minute'], + $date['second'], + $date['month'], + $date['day'], + $date['year'] + ); + + if ($timestamp == false) { + returnClientError('Error while creating timestamp'); + } + + return $timestamp; + } + + /** + * Retrieve article content from its URL using content selector and return a feed item + * @param object $entry_html A DOM element containing the article + * @param string $title_selector A selector to the article title from the article + * @param string $author_selector A selector to find the article author + * @param string $time_selector A selector to get the article publication time. + * @param string $time_format The format to parse the time_selector. + * @param string $content_cleanup Optional selector for removing elements, e.g. "div.ads, + * div.comments" + * @param string $title_default Optional title to use when could not extract title reliably + * @param bool $remove_styling Whether to remove class and style attributes from the HTML + * @return array Entry data: uri, title, content + */ + protected function parseEntryElement( + $entry_html, + $title_selector = null, + $author_selector = null, + $category_selector = null, + $time_selector = null, + $time_format = null, + $content_cleanup = null, + $title_default = null, + $remove_styling = false + ) { + $article_content = convertLazyLoading($entry_html); + + if (is_null($title_selector)) { + $article_title = $title_default; + } else { + $article_title = trim($entry_html->find($title_selector, 0)->innertext); + } + + $author = null; + if (!is_null($author_selector) && $author_selector != '') { + $author = trim($entry_html->find($author_selector, 0)->innertext); + } + + $categories = []; + if (!is_null($category_selector && $category_selector != '')) { + $category_elements = $entry_html->find($category_selector); + foreach ($category_elements as $category_element) { + $categories[] = trim($category_element->innertext); + } + } + + $time = null; + if (!is_null($time_selector) && $time_selector != '') { + $time_element = $entry_html->find($time_selector, 0); + $time = $time_element->getAttribute('datetime'); + if (is_null($time)) { + $time = $time_element->innertext; + } + + $this->parseTimeStrAsTimestamp($time, $time_format); + } + + $article_content = $this->cleanArticleContent($article_content, $content_cleanup, $remove_styling); + + $item = []; + $item['title'] = $article_title; + $item['content'] = $article_content; + $item['categories'] = $categories; + $item['timestamp'] = $time; + $item['author'] = $author; + return $item; + } +} From 0325c2414a6b00311abb291d1189c97f120053bd Mon Sep 17 00:00:00 2001 From: t0stiman <18124323+t0stiman@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:34:35 +0200 Subject: [PATCH 078/716] fix carthrottlebridge (#3633) --- bridges/CarThrottleBridge.php | 51 ++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/bridges/CarThrottleBridge.php b/bridges/CarThrottleBridge.php index 95641573..5b95dd28 100644 --- a/bridges/CarThrottleBridge.php +++ b/bridges/CarThrottleBridge.php @@ -1,44 +1,45 @@ <?php -class CarThrottleBridge extends FeedExpander +class CarThrottleBridge extends BridgeAbstract { - const NAME = 'Car Throttle '; - const URI = 'https://www.carthrottle.com'; + const NAME = 'Car Throttle'; + const URI = 'https://www.carthrottle.com/'; const DESCRIPTION = 'Get the latest car-related news from Car Throttle.'; const MAINTAINER = 't0stiman'; public function collectData() { - $this->collectExpandableDatas('https://www.carthrottle.com/rss', 10); - } + $news = getSimpleHTMLDOMCached(self::URI . 'news') + or returnServerError('could not retrieve page'); - protected function parseItem($feedItem) - { - $item = parent::parseItem($feedItem); + $this->items[] = []; - //fetch page - $articlePage = getSimpleHTMLDOMCached($feedItem->link) - or returnServerError('Could not retrieve ' . $feedItem->link); + //for each post + foreach ($news->find('div.cmg-card') as $post) { + $item = []; - $subtitle = $articlePage->find('p.standfirst', 0); - $article = $articlePage->find('div.content_field', 0); + $titleElement = $post->find('div.title a.cmg-link')[0]; + $item['uri'] = self::URI . $titleElement->getAttribute('href'); + $item['title'] = $titleElement->innertext; - $item['content'] = str_get_html($subtitle . $article); + $articlePage = getSimpleHTMLDOMCached($item['uri']) + or returnServerError('could not retrieve page'); - //convert <iframe>s to <a>s. meant for embedded videos. - foreach ($item['content']->find('iframe') as $found) { - $iframeUrl = $found->getAttribute('src'); + $item['author'] = $articlePage->find('div.author div')[1]->innertext; - if ($iframeUrl) { - $found->outertext = '<a href="' . $iframeUrl . '">' . $iframeUrl . '</a>'; + $dinges = $articlePage->find('div.main-body')[0]; + //remove ads + foreach ($dinges->find('aside') as $ad) { + $ad->outertext = ''; + $dinges->save(); } - } - //remove scripts from the text - foreach ($item['content']->find('script') as $remove) { - $remove->outertext = ''; - } + $item['content'] = $articlePage->find('div.summary')[0] . + $articlePage->find('figure.main-image')[0] . + $dinges; - return $item; + //add the item to the list + array_push($this->items, $item); + } } } From 18a8a5127180d9c4a4318f88cf2ffbc441b0ed60 Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:13:44 +0200 Subject: [PATCH 079/716] [GitlabIssueBridge] Add support for GitLab Epics --- bridges/GitlabIssueBridge.php | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/bridges/GitlabIssueBridge.php b/bridges/GitlabIssueBridge.php index 59797d8d..27c19467 100644 --- a/bridges/GitlabIssueBridge.php +++ b/bridges/GitlabIssueBridge.php @@ -3,10 +3,10 @@ class GitlabIssueBridge extends BridgeAbstract { const MAINTAINER = 'Mynacol'; - const NAME = 'Gitlab Issue/Merge Request'; + const NAME = 'Gitlab Issue/Merge Request/Epic'; const URI = 'https://gitlab.com/'; const CACHE_TIMEOUT = 1800; // 30min - const DESCRIPTION = 'Returns comments of an issue/MR of a gitlab project'; + const DESCRIPTION = 'Returns comments of an issue/MR/Epic of a gitlab project'; const PARAMETERS = [ 'global' => [ @@ -43,6 +43,14 @@ class GitlabIssueBridge extends BridgeAbstract 'exampleValue' => '2099', 'required' => true ] + ], + 'Epic comments' => [ + 'i' => [ + 'name' => 'Epic number', + 'type' => 'number', + 'exampleValue' => '2099', + 'required' => true + ] ] ]; @@ -56,6 +64,9 @@ class GitlabIssueBridge extends BridgeAbstract case 'Merge Request comments': $name .= ' MR !' . $this->getInput('i'); break; + case 'Epic comments': + $name .= ' Epic &' . $this->getInput('i'); + break; default: return parent::getName(); } @@ -74,6 +85,9 @@ class GitlabIssueBridge extends BridgeAbstract case 'Merge Request comments': $uri .= '-/merge_requests'; break; + case 'Epic comments': + $uri .= '-/epics'; + break; default: return $uri; } @@ -107,8 +121,10 @@ class GitlabIssueBridge extends BridgeAbstract foreach ($comments as $value) { foreach ($value->notes as $comment) { $item = []; - $item['uri'] = $comment->noteable_note_url; - $item['uid'] = $item['uri']; + if ($comment->noteable_note_url !== null) { + $item['uri'] = $comment->noteable_note_url; + $item['uid'] = $item['uri']; + } // TODO fix invalid timestamps (fdroid bot) $item['timestamp'] = $comment->created_at ?? $comment->updated_at ?? $comment->last_edited_at; From c3b5b382ba1d64e7613b0c9249be053168c4e1f7 Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:33:17 +0200 Subject: [PATCH 080/716] [ZeitBridge] Remove broken paywall workaround Clean up spoofing Google Bot as this workaround doesn't work anymore. --- bridges/ZeitBridge.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index a9e4bcf2..b294e9fb 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -66,8 +66,6 @@ class ZeitBridge extends FeedExpander $item['enclosures'] = []; $headers = [ - 'User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', - 'X-Forwarded-For: 66.249.66.1', 'Cookie: zonconsent=' . date('Y-m-d\TH:i:s.v\Z'), ]; From 999d5dce405165ce0a025259405943f7a9d8753f Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:54:02 +0200 Subject: [PATCH 081/716] [HeiseBridge] Remove archive link for heise+ archive.ph is also not able to provide the full content of paywalled heise+ articles. --- bridges/HeiseBridge.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index 08c43dc5..dfda311c 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -137,10 +137,8 @@ class HeiseBridge extends FeedExpander if (strpos($item['uri'], 'https://www.heise.de') !== 0) { return $item; } - - // abort on heise+ articles and link to archive.ph for full-text content + // abort on heise+ articles if ($sessioncookie == '' && str_starts_with($item['title'], 'heise+ |')) { - $item['uri'] = 'https://archive.ph/?run=1&url=' . urlencode($item['uri']); return $item; } From 14607c07f682a2766777b909e732af653c400fee Mon Sep 17 00:00:00 2001 From: Mynacol <Mynacol@users.noreply.github.com> Date: Sun, 27 Aug 2023 13:14:21 +0200 Subject: [PATCH 082/716] [GitlabIssueBridge] Fix example values for MR These values are used for testing and PR artifacts, but https://gitlab.com/fdroid/fdroidclient currently has no MR !2099, leading to a HTTP 404 error. This just uses issue #1 and MR !1. To support epics, the specified repository is ignored. --- bridges/GitlabIssueBridge.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bridges/GitlabIssueBridge.php b/bridges/GitlabIssueBridge.php index 27c19467..f900335f 100644 --- a/bridges/GitlabIssueBridge.php +++ b/bridges/GitlabIssueBridge.php @@ -18,12 +18,12 @@ class GitlabIssueBridge extends BridgeAbstract ], 'u' => [ 'name' => 'User/Organization name', - 'exampleValue' => 'fdroid', + 'exampleValue' => 'gitlab-org', 'required' => true ], 'p' => [ 'name' => 'Project name', - 'exampleValue' => 'fdroidclient', + 'exampleValue' => 'gitlab-foss', 'required' => true ] @@ -32,7 +32,7 @@ class GitlabIssueBridge extends BridgeAbstract 'i' => [ 'name' => 'Issue number', 'type' => 'number', - 'exampleValue' => '2099', + 'exampleValue' => '1', 'required' => true ] ], @@ -40,7 +40,7 @@ class GitlabIssueBridge extends BridgeAbstract 'i' => [ 'name' => 'Merge Request number', 'type' => 'number', - 'exampleValue' => '2099', + 'exampleValue' => '1', 'required' => true ] ], @@ -48,7 +48,7 @@ class GitlabIssueBridge extends BridgeAbstract 'i' => [ 'name' => 'Epic number', 'type' => 'number', - 'exampleValue' => '2099', + 'exampleValue' => '1', 'required' => true ] ] @@ -86,7 +86,7 @@ class GitlabIssueBridge extends BridgeAbstract $uri .= '-/merge_requests'; break; case 'Epic comments': - $uri .= '-/epics'; + $uri = 'https://' . $host . '/groups/' . $this->getInput('u') . '/-/epics'; break; default: return $uri; From 9e33a15b93de43a1f8ef3fc705748c59d8dbeed7 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Wed, 30 Aug 2023 00:34:10 +0930 Subject: [PATCH 083/716] [FurAffinityBridge] Fix if search result contains hidden submission (#3637) * Reverts to preview submission if full is hidden * Reverts to preview submission if full is hidden * revert * added fallback to higher res preview if SWF * amend --- bridges/FurAffinityBridge.php | 87 ++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/bridges/FurAffinityBridge.php b/bridges/FurAffinityBridge.php index 6c2d7b52..977fbb6b 100644 --- a/bridges/FurAffinityBridge.php +++ b/bridges/FurAffinityBridge.php @@ -892,7 +892,7 @@ class FurAffinityBridge extends BridgeAbstract $item = []; $submissionURL = $figure->find('b u a', 0)->href; - $imgURL = 'https:' . $figure->find('b u a img', 0)->src; + $imgURL = $figure->find('b u a img', 0)->src; $item['uri'] = $submissionURL; $item['title'] = html_entity_decode( @@ -900,52 +900,43 @@ class FurAffinityBridge extends BridgeAbstract ); $item['author'] = $figure->find('figcaption p a[href*=/user/]', 0)->title; + $item['content'] = "<a href=\"$submissionURL\"> <img src=\"{$imgURL}\" referrerpolicy=\"no-referrer\"/></a>"; + if ($this->getInput('full') === true) { $submissionHTML = $this->getFASimpleHTMLDOM($submissionURL, $cache); + if (!$this->isHiddenSubmission($submissionHTML)) { + $stats = $submissionHTML->find('.stats-container', 0); + $popupDate = $stats->find('.popup_date', 0); + if ($popupDate) { + $item['timestamp'] = strtotime($popupDate->title); + } - $stats = $submissionHTML->find('.stats-container', 0); - $popupDate = $stats->find('.popup_date', 0); - if ($popupDate) { - $item['timestamp'] = strtotime($popupDate->title); + $var = $submissionHTML->find('.actions a[href^=https://d.facdn]', 0); + if ($var) { + $item['enclosures'] = [$var->href]; + } + + foreach ($stats->find('#keywords a') as $keyword) { + $item['categories'][] = $keyword->plaintext; + } + + $previewSrc = $submissionHTML->find('#submissionImg', 0); + if ($previewSrc) { + $imgURL = 'https:' . $previewSrc->{'data-preview-src'}; + } else { + $imgURL = $submissionHTML->find('[property="og:image"]', 0)->{'content'}; + } + + $description = $submissionHTML->find('div.submission-description', 0); + if ($description) { + $this->setReferrerPolicy($description); + $description = trim($description->innertext); + } else { + $description = ''; + } + + $item['content'] = "<a href=\"$submissionURL\"> <img src=\"{$imgURL}\" referrerpolicy=\"no-referrer\"/></a><p>{$description}</p>"; } - - $var = $submissionHTML->find('.actions a[href^=https://d.facdn]', 0); - if ($var) { - $item['enclosures'] = [$var->href]; - } - - foreach ($stats->find('#keywords a') as $keyword) { - $item['categories'][] = $keyword->plaintext; - } - - $previewSrc = $submissionHTML->find('#submissionImg', 0) - ->{'data-preview-src'}; - if ($previewSrc) { - $imgURL = 'https:' . $previewSrc; - } - - $description = $submissionHTML->find('div.submission-description', 0); - if ($description) { - $this->setReferrerPolicy($description); - $description = trim($description->innertext); - } else { - $description = ''; - } - - $item['content'] = <<<EOD -<a href="$submissionURL"> - <img src="{$imgURL}" referrerpolicy="no-referrer" /> -</a> -<p> -{$description} -</p> -EOD; - } else { - $item['content'] = <<<EOD -<a href="$submissionURL"> - <img src="$imgURL" referrerpolicy="no-referrer" /> -</a> -EOD; } $this->items[] = $item; @@ -964,4 +955,14 @@ EOD; $img->referrerpolicy = 'no-referrer'; } } + + private function isHiddenSubmission($html) + { + //Disabled accounts prevents their userpage, gallery, favorites and journals from being viewed. + //Submissions can require maturity limit or logged-in account. + $system_message = $html->find('.section-body.alignleft', 0); + $system_message = $system_message ? $system_message->plaintext : ''; + + return str_contains($system_message, 'System Message'); + } } From f0ec797f4bceb25b396e75499ea13419ba2b0292 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Wed, 30 Aug 2023 00:35:37 +0930 Subject: [PATCH 084/716] [FurAffinityBridge] Option for instance host to add custom cookie (#3638) * added custom cookie config * appease phpunit --- bridges/FurAffinityBridge.php | 45 +++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/bridges/FurAffinityBridge.php b/bridges/FurAffinityBridge.php index 977fbb6b..087c3ded 100644 --- a/bridges/FurAffinityBridge.php +++ b/bridges/FurAffinityBridge.php @@ -6,7 +6,18 @@ class FurAffinityBridge extends BridgeAbstract const URI = 'https://www.furaffinity.net'; const CACHE_TIMEOUT = 300; // 5min const DESCRIPTION = 'Returns posts from various sections of FurAffinity'; - const MAINTAINER = 'Roliga'; + const MAINTAINER = 'Roliga, mruac'; + const CONFIGURATION = [ + 'aCookie' => [ + 'required' => false, + 'defaultValue' => 'ca6e4566-9d81-4263-9444-653b142e35f8' + + ], + 'bCookie' => [ + 'required' => false, + 'defaultValue' => '4ce65691-b50f-4742-a990-bf28d6de16ee' + ] + ]; const PARAMETERS = [ 'Search' => [ 'q' => [ @@ -594,7 +605,7 @@ class FurAffinityBridge extends BridgeAbstract * This was aquired by creating a new user on FA then * extracting the cookie from the browsers dev console. */ - const FA_AUTH_COOKIE = 'b=4ce65691-b50f-4742-a990-bf28d6de16ee; a=ca6e4566-9d81-4263-9444-653b142e35f8'; + private $FA_AUTH_COOKIE; public function detectParameters($url) { @@ -662,7 +673,14 @@ class FurAffinityBridge extends BridgeAbstract . '\'s Folder ' . $this->getInput('folder-id'); default: - return parent::getName(); + $name = parent::getName(); + if ($this->getOption('aCookie') !== null) { + $username = $this->loadCacheValue('username'); + if ($username !== null) { + $name = $username . '\'s ' . parent::getName(); + } + } + return $name; } } @@ -741,6 +759,7 @@ class FurAffinityBridge extends BridgeAbstract public function collectData() { + $this->FA_AUTH_COOKIE = 'b=' . $this->getOption('bCookie') . '; a=' . $this->getOption('aCookie'); switch ($this->queriedContext) { case 'Search': $data = [ @@ -806,19 +825,19 @@ class FurAffinityBridge extends BridgeAbstract $header = [ 'Host: ' . parse_url(self::URI, PHP_URL_HOST), 'Content-Type: application/x-www-form-urlencoded', - 'Cookie: ' . self::FA_AUTH_COOKIE + 'Cookie: ' . $this->FA_AUTH_COOKIE ]; $html = getSimpleHTMLDOM($this->getURI(), $header, $opts); $html = defaultLinkTo($html, $this->getURI()); - + $this->saveLoggedInUser($html); return $html; } private function getFASimpleHTMLDOM($url, $cache = false) { $header = [ - 'Cookie: ' . self::FA_AUTH_COOKIE + 'Cookie: ' . $this->FA_AUTH_COOKIE ]; if ($cache) { @@ -826,12 +845,24 @@ class FurAffinityBridge extends BridgeAbstract } else { $html = getSimpleHTMLDOM($url, $header); } - + $this->saveLoggedInUser($html); $html = defaultLinkTo($html, $url); return $html; } + private function saveLoggedInUser($html) + { + $current_user = $html->find('#my-username', 0); + if ($current_user !== null) { + preg_match('/^(?:My FA \( |~)(.*?)(?: \)|)$/', trim($current_user->plaintext), $matches); + $current_user = $current_user ? $matches[1] : null; + if ($current_user !== null) { + $this->saveCacheValue('username', $current_user); + } + } + } + private function itemsFromJournalList($html, $limit) { foreach ($html->find('table[id^=jid:]') as $journal) { From 52c59caf2f5ed83745a4a0e62dae868eed44279a Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Tue, 29 Aug 2023 17:08:18 +0200 Subject: [PATCH 085/716] [Core] Find Feed : fix CSS for search results (#3643) There was no margin at the bottom of the search result : in case of two ore more results, the two blocks had no space between them. --- static/style.css | 1 + 1 file changed, 1 insertion(+) diff --git a/static/style.css b/static/style.css index a9d5933a..b835c98d 100644 --- a/static/style.css +++ b/static/style.css @@ -462,6 +462,7 @@ button { display: flex; position: relative; text-align: left; + margin-bottom: 15px; } @media (prefers-color-scheme: dark) { .search-result { From 9707586ee8d92b153fd7b969e94ec883bfc2223b Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Wed, 30 Aug 2023 00:39:05 +0930 Subject: [PATCH 086/716] [PatreonBridge] Extend the presentation of parsed posts (#3617) * extend post presentation * applied phpcbf note: phpcs does not like long null coalescing chains * resolved phpcs --- bridges/PatreonBridge.php | 162 +++++++++++++++++++++++++++++++------- 1 file changed, 135 insertions(+), 27 deletions(-) diff --git a/bridges/PatreonBridge.php b/bridges/PatreonBridge.php index 23dbf8ae..3d19cd81 100644 --- a/bridges/PatreonBridge.php +++ b/bridges/PatreonBridge.php @@ -34,13 +34,18 @@ class PatreonBridge extends BridgeAbstract 'attachments', 'user_defined_tags', //'campaign', - //'poll.choices', + 'poll.choices', //'poll.current_user_responses.user', //'poll.current_user_responses.choice', //'poll.current_user_responses.poll', //'access_rules.tier.null', - //'images.null', - //'audio.null' + 'images.null', + 'audio.null', + // 'user.null', + 'attachments.null', + 'audio_preview.null', + 'poll.choices.null' + // 'poll.current_user_responses.null' ]), 'fields' => [ 'post' => implode(',', [ @@ -50,7 +55,7 @@ class PatreonBridge extends BridgeAbstract //'current_user_can_delete', //'current_user_can_view', //'current_user_has_liked', - //'embed', + 'embed', 'image', //'is_paid', //'like_count', @@ -58,9 +63,9 @@ class PatreonBridge extends BridgeAbstract //'patreon_url', //'patron_count', //'pledge_url', - //'post_file', - //'post_metadata', - //'post_type', + // 'post_file', + // 'post_metadata', + 'post_type', 'published_at', 'teaser_text', //'thumbnail_url', @@ -68,11 +73,24 @@ class PatreonBridge extends BridgeAbstract //'upgrade_url', 'url', //'was_posted_by_campaign_owner' + // 'content_teaser_text', + // 'current_user_can_report', + 'thumbnail', + // 'video_preview' ]), 'user' => implode(',', [ //'image_url', 'full_name', //'url' + ]), + 'media' => implode(',', [ + 'id', + 'image_urls', + 'download_url', + 'metadata', + 'file_name', + 'mimetype', + 'size_bytes' ]) ], 'filter' => [ @@ -97,41 +115,129 @@ class PatreonBridge extends BridgeAbstract $posts, 'user', $post->relationships->user->data->id - ); + )->attributes; $item['author'] = $user->full_name; - $image = $post->attributes->image ?? null; - if ($image) { - $logo = sprintf( - '<p><a href="%s"><img src="%s" /></a></p>', - $post->attributes->url, - $image->thumb_url ?? $image->url ?? $this->getURI() - ); - $item['content'] .= $logo; + //image, video, audio, link (featured post content) + switch ($post->attributes->post_type) { + case 'audio_file': + //check if download_url is null before assigning $audio + $id = $post->relationships->audio->data->id ?? null; + if (isset($id)) { + $audio = $this->findInclude($posts, 'media', $id)->attributes ?? null; + } + if (!isset($audio->download_url)) { //if not unlocked + $id = $post->relationships->audio_preview->data->id ?? null; + if (isset($id)) { + $audio = $this->findInclude($posts, 'media', $id)->attributes ?? null; + } + } + $thumbnail = $post->attributes->thumbnail->large ?? $post->attributes->thumbnail->url; + $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url; + $thumbnail = $thumbnail ?? $post->attributes->image->url; + $audio_filename = $audio->file_name ?? $item['title']; + $download_url = $audio->download_url ?? $item['uri']; + $item['content'] .= "<p><a href\"{$download_url}\"><img src=\"{$thumbnail}\"><br/>🎧 {$audio_filename}</a><br/>"; + if ($download_url !== $item['uri']) { + $item['enclosures'][] = $download_url; + $item['content'] .= "<audio controls src=\"{$download_url}\"></audio>"; + } + $item['content'] .= '</p>'; + break; + + case 'video_embed': + $thumbnail = $post->attributes->thumbnail->large ?? $post->attributes->thumbnail->url; + $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url; + $thumbnail = $thumbnail ?? $post->attributes->image->url; + $item['content'] .= "<p><a href=\"{$item['uri']}\">🎬 {$item['title']}<br><img src=\"{$thumbnail}\"></a></p>"; + break; + + case 'video_external_file': + $thumbnail = $post->attributes->thumbnail->large ?? $post->attributes->thumbnail->url; + $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url; + $thumbnail = $thumbnail ?? $post->attributes->image->url; + $item['content'] .= "<p><a href=\"{$item['uri']}\">🎬 {$item['title']}<br><img src=\"{$thumbnail}\"></a></p>"; + break; + + case 'image_file': + $item['content'] .= '<p>'; + foreach ($post->relationships->images->data as $key => $image) { + $image = $this->findInclude($posts, 'media', $image->id)->attributes; + $image_fullres = $image->download_url ?? $image->image_urls->url ?? $image->image_urls->original; + $filename = $image->file_name ?? ''; + $image_url = $image->image_urls->url ?? $image->image_urls->original; + $item['enclosures'][] = $image_fullres; + $item['content'] .= "<a href=\"{$image_fullres}\">{$filename}<br/><img src=\"{$image_url}\"></a><br/><br/>"; + } + $item['content'] .= '</p>'; + break; + + case 'link': + //make it locked safe + if (isset($post->attributes->embed)) { + $embed = $post->attributes->embed; + $thumbnail = $post->attributes->image->large_url ?? $post->attributes->image->thumb_url ?? $post->attributes->image->url; + $item['content'] .= '<p><table>'; + $item['content'] .= "<tr><td><a href=\"{$embed->url}\"><img src=\"{$thumbnail}\"></a></td></tr>"; + $item['content'] .= "<tr><td><b>{$embed->subject}</b></td></tr>"; + $item['content'] .= "<tr><td>{$embed->description}</td></tr>"; + $item['content'] .= '</table></p><hr/>'; + } + break; } + //content of the post if (isset($post->attributes->content)) { $item['content'] .= $post->attributes->content; } elseif (isset($post->attributes->teaser_text)) { $item['content'] .= '<p>' - . $post->attributes->teaser_text - . '</p>'; + . $post->attributes->teaser_text; + if (strlen($post->attributes->teaser_text) === 140) { + $item['content'] .= '…'; + } + $item['content'] .= '</p>'; } + //post tags if (isset($post->relationships->user_defined_tags)) { $item['categories'] = []; foreach ($post->relationships->user_defined_tags->data as $tag) { - $attrs = $this->findInclude($posts, 'post_tag', $tag->id); + $attrs = $this->findInclude($posts, 'post_tag', $tag->id)->attributes; $item['categories'][] = $attrs->value; } } - if (isset($post->relationships->attachments)) { - $item['enclosures'] = []; - foreach ($post->relationships->attachments->data as $attachment) { - $attrs = $this->findInclude($posts, 'attachment', $attachment->id); - $item['enclosures'][] = $attrs->url; + //poll + if (isset($post->relationships->poll->data)) { + $poll = $this->findInclude($posts, 'poll', $post->relationships->poll->data->id); + $item['content'] .= "<p><table><tr><th><b>Poll: {$poll->attributes->question_text}</b></th></tr>"; + foreach ($poll->relationships->choices->data as $key => $poll_option) { + $poll_option = $this->findInclude($posts, 'poll_choice', $poll_option->id); + $poll_option_text = $poll_option->attributes->text_content ?? null; + if (isset($poll_option_text)) { + $item['content'] .= "<tr><td><a href=\"{$item['uri']}\">{$poll_option_text}</a></td></tr>"; + } } + $item['content'] .= '</table></p>'; + } + + + //post attachments + if ( + isset($post->relationships->attachments->data) && + sizeof($post->relationships->attachments->data) > 0 + ) { + $item['enclosures'] = []; + $item['content'] .= '<hr><p><b>Attachments:</b><ul>'; + foreach ($post->relationships->attachments->data as $attachment) { + $attrs = $this->findInclude($posts, 'attachment', $attachment->id)->attributes; + $filename = $attrs->name; + $n = strrpos($filename, '.'); + $ext = ($n === false) ? '' : substr($filename, $n); + $item['enclosures'][] = $attrs->url . '#' . $ext; + $item['content'] .= '<li><a href="' . $attrs->url . '">' . $filename . '</a></li>'; + } + $item['content'] .= '</ul></p>'; } $this->items[] = $item; @@ -139,14 +245,16 @@ class PatreonBridge extends BridgeAbstract } /* - * Searches the "included" array in an API response and returns attributes - * for the first match. + * Searches the "included" array in an API response and returns the result for the first match. + * A result will include attributes containing further details of the included object + * (e.g. an audio object), and an optional relationships object that links to more "included" + * objects. (e.g. a poll object with related poll_choice(s)) */ private function findInclude($data, $type, $id) { foreach ($data->included as $include) { if ($include->type === $type && $include->id === $id) { - return $include->attributes; + return $include; } } } From 4d05d0beff361ab6724b9411ef2271af37beb7f5 Mon Sep 17 00:00:00 2001 From: csisoap <33269526+csisoap@users.noreply.github.com> Date: Tue, 29 Aug 2023 22:14:34 +0700 Subject: [PATCH 087/716] [TwitterBridge] Add support for OAuth authorization. (#3628) * Update TwitterClient.php - Add OAuth authorization header. - Add new endpoint. * Update TwitterBridge.php - Make some changes to support new endpoint. * Update TwitterBridge.php * clean up, fix warning * fix warning * fix warning * remove oauth token * fix wrong twitter id when encounter reply post. * Update TwitterClient.php * fix wrong twitter id cause by previous commit * clear warning * attempt to clear warning * attempt to clear warning --- bridges/TwitterBridge.php | 21 ++++- lib/TwitterClient.php | 184 ++++++++++++++++++++++++++------------ 2 files changed, 146 insertions(+), 59 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 1f115be8..a5d09f8a 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -306,15 +306,22 @@ EOD } } + // Array of Tweet IDs + $tweetIds = []; // Filter out unwanted tweets foreach ($data->tweets as $tweet) { + if (isset($tweet->rest_id)) { + $tweetIds[] = $tweet->rest_id; + $tweet = $tweet->legacy; + } + if (!$tweet) { continue; } // Filter out retweets to remove possible duplicates of original tweet switch ($this->queriedContext) { case 'By keyword or hashtag': - if (isset($tweet->retweeted_status) && substr($tweet->full_text, 0, 4) === 'RT @') { + if ((isset($tweet->retweeted_status) || isset($tweet->retweeted_status_result)) && substr($tweet->full_text, 0, 4) === 'RT @') { continue 2; } break; @@ -351,9 +358,13 @@ EOD $item = []; $realtweet = $tweet; + $tweetId = (isset($tweetIds[$i]) ? $tweetIds[$i] : $realtweet->conversation_id_str); if (isset($tweet->retweeted_status)) { // Tweet is a Retweet, so set author based on original tweet and set realtweet for reference to the right content $realtweet = $tweet->retweeted_status; + } elseif (isset($tweet->retweeted_status_result)) { + $tweetId = $tweet->retweeted_status_result->result->rest_id; + $realtweet = $tweet->retweeted_status_result->result->legacy; } if (isset($realtweet->truncated) && $realtweet->truncated) { @@ -364,6 +375,10 @@ EOD } } + if (!$realtweet) { + $realtweet = $tweet; + } + switch ($this->queriedContext) { case 'By username': if ($this->getInput('norep') && isset($tweet->in_reply_to_status_id)) { @@ -372,7 +387,7 @@ EOD $item['username'] = $data->user_info->legacy->screen_name; $item['fullname'] = $data->user_info->legacy->name; $item['avatar'] = $data->user_info->legacy->profile_image_url_https; - $item['id'] = $realtweet->id_str; + $item['id'] = (isset($realtweet->id_str) ? $realtweet->id_str : $tweetId); break; case 'By list': case 'By list ID': @@ -391,7 +406,7 @@ EOD $item['timestamp'] = $realtweet->created_at; $item['uri'] = self::URI . $item['username'] . '/status/' . $item['id']; - $item['author'] = (isset($tweet->retweeted_status) ? 'RT: ' : '') + $item['author'] = ((isset($tweet->retweeted_status) || (isset($tweet->retweeted_status_result))) ? 'RT: ' : '') . $item['fullname'] . ' (@' . $item['username'] . ')'; diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 0c6b9535..d2a09fdd 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -18,6 +18,66 @@ class TwitterClient $this->data = $this->cache->loadData() ?? []; $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; + $this->tw_consumer_key = '3nVuSoBZnx6U4vzUxf5w'; + $this->tw_consumer_secret = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys'; + $this->oauth_token = ''; //Fill here + $this->oauth_token_secret = ''; //Fill here + } + + private function getOauthAuthorization( + $oauth_token, + $oauth_token_secret, + $method = 'GET', + $url = '', + $body = '', + $timestamp = null, + $oauth_nonce = null + ) { + if (!$url) { + return ''; + } + $method = strtoupper($method); + $parseUrl = parse_url($url); + $link = $parseUrl['scheme'] . '://' . $parseUrl['host'] . $parseUrl['path']; + parse_str($parseUrl['query'], $query_params); + if ($body) { + parse_str($body, $body_params); + $query_params = array_merge($query_params, $body_params); + } + $payload = [ + 'oauth_version' => '1.0', + 'oauth_signature_method' => 'HMAC-SHA1', + 'oauth_consumer_key' => $this->tw_consumer_key, + 'oauth_token' => $oauth_token, + 'oauth_nonce' => $oauth_nonce ? $oauth_nonce : implode('', array_fill(0, 3, strval(time()))), + 'oauth_timestamp' => $timestamp ? $timestamp : time(), + ]; + $payload = array_merge($payload, $query_params); + ksort($payload); + + $url_parts = parse_url($url); + $url_parts['query'] = http_build_query($payload, '', '&', PHP_QUERY_RFC3986); + $base_url = $url_parts['scheme'] . '://' . $url_parts['host'] . $url_parts['path']; + $signature_base_string = strtoupper($method) . '&' . rawurlencode($base_url) . '&' . rawurlencode($url_parts['query']); + $hmac_key = $this->tw_consumer_secret . '&' . $oauth_token_secret; + $hex_signature = hash_hmac('sha1', $signature_base_string, $hmac_key, true); + $signature = base64_encode($hex_signature); + + $header_params = [ + 'oauth_version' => '1.0', + 'oauth_token' => $oauth_token, + 'oauth_nonce' => $payload['oauth_nonce'], + 'oauth_timestamp' => $payload['oauth_timestamp'], + 'oauth_signature' => $signature, + 'oauth_consumer_key' => $this->tw_consumer_key, + 'oauth_signature_method' => 'HMAC-SHA1', + ]; + // ksort($header_params); + $header_values = []; + foreach ($header_params as $key => $value) { + $header_values[] = rawurlencode($key) . '="' . (is_int($value) ? $value : rawurlencode($value)) . '"'; + } + return 'OAuth realm="http://api.twitter.com/", ' . implode(', ', $header_values); } private function extractTweetAndUsersFromGraphQL($timeline) @@ -25,13 +85,24 @@ class TwitterClient if (isset($timeline->data->user)) { $result = $timeline->data->user->result; $instructions = $result->timeline_v2->timeline->instructions; - } else { - $result = $timeline->data->list->timeline_response; + } elseif (isset($timeline->data->user_result)) { + $result = $timeline->data->user_result->result->timeline_response; $instructions = $result->timeline->instructions; } + if (isset($result->__typename) && $result->__typename === 'UserUnavailable') { throw new \Exception('UserUnavailable'); } + + if (isset($timeline->data->list)) { + $result = $timeline->data->list->timeline_response; + $instructions = $result->timeline->instructions; + } + + if (!isset($result) && !isset($instructions)) { + throw new \Exception('Unable to fetch user/list timeline'); + } + $instructionTypes = [ 'TimelineAddEntries', 'TimelineClearCache', @@ -78,14 +149,14 @@ class TwitterClient if (!isset($entry->content->itemContent->tweet_results->result->legacy)) { continue; } - $tweets[] = $entry->content->itemContent->tweet_results->result->legacy; + $tweets[] = $entry->content->itemContent->tweet_results->result; $userIds[] = $entry->content->itemContent->tweet_results->result->core->user_results->result; } else { if (!isset($entry->content->content->tweetResult->result->legacy)) { continue; } - $tweets[] = $entry->content->content->tweetResult->result->legacy; + $tweets[] = $entry->content->content->tweetResult->result; $userIds[] = $entry->content->content->tweetResult->result->core->user_result->result; } @@ -117,19 +188,22 @@ class TwitterClient } } - try { - $timeline = $this->fetchTimelineUsingSearch($screenName); - } catch (HttpException $e) { - if ($e->getCode() === 403) { - $this->data['guest_token'] = null; - $this->fetchGuestToken(); - $timeline = $this->fetchTimelineUsingSearch($screenName); - } else { - throw $e; - } - } + $timeline = $this->fetchTimeline($userInfo->rest_id); + // try { + // // $timeline = $this->fetchTimelineUsingSearch($screenName); + // } catch (HttpException $e) { + // if ($e->getCode() === 403) { + // $this->data['guest_token'] = null; + // $this->fetchGuestToken(); + // // $timeline = $this->fetchTimelineUsingSearch($screenName); + // $timeline = $this->fetchTimeline($userInfo->rest_id); + // } else { + // throw $e; + // } + // } - $tweets = $this->extractTweetFromSearch($timeline); + // $tweets = $this->extractTweetFromSearch($timeline); + $tweets = $this->extractTweetAndUsersFromGraphQL($timeline)->tweets; return (object) [ 'user_info' => $userInfo, @@ -155,7 +229,7 @@ class TwitterClient throw $e; } } - } elseif ($operation === 'By list ID') { + } else if ($operation == 'By list ID') { $id = $query['listId']; } else { throw new \Exception('Unknown operation to make list tweets'); @@ -223,43 +297,40 @@ class TwitterClient private function fetchTimeline($userId) { $variables = [ - 'userId' => $userId, + 'autoplay_enabled' => true, 'count' => 40, - 'includePromotedContent' => true, - 'withQuickPromoteEligibilityTweetFields' => true, - 'withSuperFollowsUserFields' => true, - 'withDownvotePerspective' => false, - 'withReactionsMetadata' => false, - 'withReactionsPerspective' => false, - 'withSuperFollowsTweetFields' => true, - 'withVoice' => true, - 'withV2Timeline' => true, + 'includeEditControl' => true, + 'includeEditPerspective' => false, + 'includeHasBirdwatchNotes' => false, + 'includeTweetImpression' => true, + 'includeTweetVisibilityNudge' => true, + 'rest_id' => $userId ]; $features = [ - 'responsive_web_twitter_blue_verified_badge_is_enabled' => true, - 'responsive_web_graphql_exclude_directive_enabled' => false, - 'verified_phone_label_enabled' => false, - 'responsive_web_graphql_timeline_navigation_enabled' => true, - 'responsive_web_graphql_skip_user_profile_image_extensions_enabled' => false, + 'android_graphql_skip_api_media_color_palette' => true, + 'blue_business_profile_image_shape_enabled' => true, + 'creator_subscriptions_subscription_count_enabled' => true, + 'creator_subscriptions_tweet_preview_api_enabled' => true, + 'freedom_of_speech_not_reach_fetch_enabled' => true, 'longform_notetweets_consumption_enabled' => true, + 'longform_notetweets_inline_media_enabled' => true, + 'longform_notetweets_rich_text_read_enabled' => true, + 'subscriptions_verification_info_enabled' => true, + 'super_follow_badge_privacy_enabled' => true, + 'super_follow_exclusive_tweet_notifications_enabled' => true, + 'super_follow_tweet_api_enabled' => true, + 'super_follow_user_api_enabled' => true, + 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => true, 'tweetypie_unmention_optimization_enabled' => true, - 'vibe_api_enabled' => true, - 'responsive_web_edit_tweet_api_enabled' => true, - 'graphql_is_translatable_rweb_tweet_is_translatable_enabled' => true, - 'view_counts_everywhere_api_enabled' => true, - 'freedom_of_speech_not_reach_appeal_label_enabled' => false, - 'standardized_nudges_misinfo' => true, - 'tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled' => false, - 'interactive_text_enabled' => true, - 'responsive_web_text_conversations_enabled' => false, - 'responsive_web_enhance_cards_enabled' => false, + 'unified_cards_ad_metadata_container_dynamic_card_content_query_enabled' => true, ]; $url = sprintf( - 'https://twitter.com/i/api/graphql/WZT7sCTrLvSOaWOXLDsWbQ/UserTweets?variables=%s&features=%s', + 'https://api.twitter.com/graphql/3JNH4e9dq1BifLxAa3UMWg/UserWithProfileTweetsQueryV2?variables=%s&features=%s', urlencode(json_encode($variables)), urlencode(json_encode($features)) ); - $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + $oauth = $this->getOauthAuthorization($this->oauth_token, $this->oauth_token_secret, 'GET', $url); + $response = Json::decode(getContents($url, $this->createHttpHeaders($oauth)), false); return $response; } @@ -280,7 +351,8 @@ class TwitterClient 'https://api.twitter.com/1.1/search/tweets.json?%s', http_build_query($queryParam) ); - $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + $oauth = $this->getOauthAuthorization($this->oauth_token, $this->oauth_token_secret, 'GET', $url); + $response = Json::decode(getContents($url, $this->createHttpHeaders($oauth)), false); return $response; } @@ -348,11 +420,6 @@ class TwitterClient // Grab the first error message throw new \Exception(sprintf('From twitter api: "%s"', $response->errors[0]->message)); } - if (!isset($response->data->user_by_screen_name->list)) { - throw new \Exception( - sprintf('Unable to find list in twitter response for %s, %s', $screenName, $listSlug) - ); - } $listInfo = $response->data->user_by_screen_name->list; $this->data[$screenName . '-' . $listSlug] = $listInfo; @@ -412,23 +479,28 @@ class TwitterClient ]; $url = sprintf( - 'https://twitter.com/i/api/graphql/BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline?variables=%s&features=%s', + 'https://api.twitter.com/graphql/BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline?variables=%s&features=%s', urlencode(json_encode($variables)), urlencode(json_encode($features)) ); - $response = Json::decode(getContents($url, $this->createHttpHeaders()), false); + $oauth = $this->getOauthAuthorization($this->oauth_token, $this->oauth_token_secret, 'GET', $url); + $response = Json::decode(getContents($url, $this->createHttpHeaders($oauth)), false); return $response; } - private function createHttpHeaders(): array + private function createHttpHeaders($oauth = null): array { $headers = [ 'authorization' => sprintf('Bearer %s', $this->authorization), 'x-guest-token' => $this->data['guest_token'] ?? null, ]; - foreach ($headers as $key => $value) { - $headers[] = sprintf('%s: %s', $key, $value); + if (isset($oauth)) { + $headers['authorization'] = $oauth; + unset($headers['x-guest-token']); } - return $headers; + foreach ($headers as $key => $value) { + $headers2[] = sprintf('%s: %s', $key, $value); + } + return $headers2; } } From 64000a25264a579230b1dfba377bad368090fb57 Mon Sep 17 00:00:00 2001 From: R3dError <50834839+R3dError@users.noreply.github.com> Date: Wed, 30 Aug 2023 19:01:55 +0200 Subject: [PATCH 088/716] [NACSouthGermanyMediaLibraryBridge] Add new bridge (#3636) * Init nac south bridge * Rename bridge * Refactoring * Refactor * Fix formatting * Fix testing errors * Change constants * Update logo * Remove author omission in descriptions * Fix comment * Add maintainer * Rename bridge * Add technical note to bridge description --- bridges/NACSouthGermanyMediaLibraryBridge.php | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 bridges/NACSouthGermanyMediaLibraryBridge.php diff --git a/bridges/NACSouthGermanyMediaLibraryBridge.php b/bridges/NACSouthGermanyMediaLibraryBridge.php new file mode 100644 index 00000000..fff6c554 --- /dev/null +++ b/bridges/NACSouthGermanyMediaLibraryBridge.php @@ -0,0 +1,123 @@ +<?php + +class NACSouthGermanyMediaLibraryBridge extends BridgeAbstract +{ + private const BASE_URI = 'https://www.nak-sued.de'; + + const NAME = 'NAK Süd Mediathek'; + const DESCRIPTION = 'RSS Feed für die Runkfunkbeiträge der NAK Süd auf Bayern 2 und SWR 1. + (Technical note: This bridge might not work on certain server instances because of blacklisted IP ranges to the website.)'; + const URI = self::BASE_URI . '/mediathek'; + const MAINTAINER = 'R3dError'; + const CACHE_TIMEOUT = 7200; + + private const BAYERN2_ROOT_URI = self::BASE_URI . '/mediathek/rundfunksendungen-auf-bayern-2/aktuelle-sendungen'; + private const SWR1_ROOT_URI = self::BASE_URI . '/mediathek/rundfunksendungen-auf-swr1/aktuelle-sendungen'; + + private const MONTHS = [ + 'Januar' => 1, + 'Februar' => 2, + 'März' => 3, + 'April' => 4, + 'Mai' => 5, + 'Juni' => 6, + 'Juli' => 7, + 'August' => 8, + 'September' => 9, + 'Oktober' => 10, + 'November' => 11, + 'Dezember' => 12, + ]; + + public function getIcon() + { + return 'https://www.nak-stuttgart.de/static/themes/nak_sued/images/nak-logo.png'; + } + + private static function parseTimestamp($title) + { + if (preg_match('/([0-9]+)\.\s*([^\s]+)\s*([0-9]+)/', $title, $matches)) { + $day = $matches[1]; + $month = self::MONTHS[$matches[2]]; + $year = $matches[3]; + return $year . '-' . $month . '-' . $day; + } else { + return ''; + } + } + + private static function collectDataForSWR1($parent, $item) + { + # Find link + $sourceURI = $parent->find('a', 1)->href; + $item['enclosures'] = [self::BASE_URI . $sourceURI]; + + # Add time to timestamp + $item['timestamp'] .= ' 07:27'; + + # Find author + if (preg_match('/\((.*?)\)/', html_entity_decode($item['content']), $matches)) { + $item['author'] = $matches[1]; + } + + return $item; + } + + private static function collectDataForBayern2($parent, $item) + { + # Find link + $playerDom = getSimpleHTMLDOMCached(self::BASE_URI . $parent->find('a', 0)->href); + $sourceURI = $playerDom->find('source', 0)->src; + $item['enclosures'] = [self::BASE_URI . $sourceURI]; + + # Add time to timestamp + $item['timestamp'] .= ' 06:45'; + + return $item; + } + + private function collectDataInList($pageURI, $customizeItemCall) + { + $page = getSimpleHTMLDOM(self::BASE_URI . $pageURI); + + foreach ($page->find('div.grids') as $parent) { + # Find title + $title = $parent->find('h2', 0)->plaintext; + + # Find content + $contentBlock = $parent->find('ul.contentlist', 0); + $content = ''; + foreach ($contentBlock->find('li') as $li) { + $content .= '<p>' . $li->plaintext . '</p>'; + } + + $item = [ + 'title' => $title, + 'content' => $content, + 'timestamp' => self::parseTimestamp($title), + ]; + $this->items[] = $customizeItemCall($parent, $item); + } + } + + private function collectDataFromAllPages($rootURI, $customizeItemCall) + { + $rootPage = getSimpleHTMLDOM($rootURI); + $pages = $rootPage->find('div#tabmenu', 0); + foreach ($pages->find('a') as $page) { + self::collectDataInList($page->href, [$this, $customizeItemCall]); + } + } + + public function collectData() + { + # Collect items + self::collectDataFromAllPages(self::BAYERN2_ROOT_URI, 'collectDataForBayern2'); + self::collectDataFromAllPages(self::SWR1_ROOT_URI, 'collectDataForSWR1'); + + # Sort items by decreasing timestamp + usort($this->items, function ($a, $b) { + return strtotime($b['timestamp']) <=> strtotime($a['timestamp']); + }); + } +} From 92b2bc5e119fa6a405017da9381b00500f65acbe Mon Sep 17 00:00:00 2001 From: Niehztog <Niehztog@users.noreply.github.com> Date: Sun, 3 Sep 2023 00:22:48 +0200 Subject: [PATCH 089/716] fixes extracting article images, article date/time and article author and item id (#3645) --- bridges/NiusBridge.php | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/bridges/NiusBridge.php b/bridges/NiusBridge.php index e5773a7d..05d0dacc 100644 --- a/bridges/NiusBridge.php +++ b/bridges/NiusBridge.php @@ -6,7 +6,7 @@ class NiusBridge extends XPathAbstract const URI = 'https://www.nius.de/news'; const DESCRIPTION = 'Die Stimme der Mehrheit'; const MAINTAINER = 'Niehztog'; - //const PARAMETERS = array(); + const CACHE_TIMEOUT = 3600; const FEED_SOURCE_URL = 'https://www.nius.de/news'; @@ -15,18 +15,30 @@ class NiusBridge extends XPathAbstract const XPATH_EXPRESSION_ITEM_CONTENT = './/h2[@class="title"]//node()'; const XPATH_EXPRESSION_ITEM_URI = './/a[1]/@href'; - const XPATH_EXPR_AUTHOR_PART1 = 'normalize-space(.//span[@class="author"]/text()[1])'; - const XPATH_EXPR_AUTHOR_PART2 = 'normalize-space(.//span[@class="author"]/text()[2])'; - const XPATH_EXPRESSION_ITEM_AUTHOR = 'substring-after(concat(' . self::XPATH_EXPR_AUTHOR_PART1 . ', " ", ' . self::XPATH_EXPR_AUTHOR_PART2 . '), " ")'; + const XPATH_EXPRESSION_ITEM_AUTHOR = 'normalize-space(.//span[@class="author"]/text()[3])'; - //const XPATH_EXPRESSION_ITEM_TIMESTAMP = './/td[3]'; - const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img[1]/@src'; + const XPATH_EXPRESSION_ITEM_TIMESTAMP = 'normalize-space(.//span[@class="author"]/text()[1])'; + const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img[@sizes]/@src'; const XPATH_EXPRESSION_ITEM_CATEGORIES = './/div[@class="subtitle"]/text()'; const SETTING_FIX_ENCODING = false; + protected function formatItemTimestamp($value) + { + return DateTimeImmutable::createFromFormat( + false !== strpos($value, ' Uhr') ? 'H:i \U\h\r' : 'd.m.y', + $value, + new DateTimeZone('Europe/Berlin') + )->format('U'); + } + protected function cleanMediaUrl($mediaUrl) { $result = preg_match('~https:\/\/www\.nius\.de\/_next\/image\?url=(.*)\?~', $mediaUrl, $matches); return $result ? $matches[1] : $mediaUrl; } + + protected function generateItemId(FeedItem $item) + { + return substr($item->getURI(), strrpos($item->getURI(), '/') + 1); + } } From b9fdd20f8f13514ad35102a6f49e382538563bd5 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Sun, 3 Sep 2023 07:53:36 +0930 Subject: [PATCH 090/716] [FurAffinityBridge] added doc for #3638 (#3646) * added custom cookie config * appease phpunit * added docs --- docs/10_Bridge_Specific/FurAffinityBridge.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/10_Bridge_Specific/FurAffinityBridge.md diff --git a/docs/10_Bridge_Specific/FurAffinityBridge.md b/docs/10_Bridge_Specific/FurAffinityBridge.md new file mode 100644 index 00000000..369f41cd --- /dev/null +++ b/docs/10_Bridge_Specific/FurAffinityBridge.md @@ -0,0 +1,13 @@ +FurAffinityBridge +=============== +By default this bridge will only return submissions that are rated "General" and are public. + +To unlock the ability to load submissions that require an account to view or are rated "Mature" and higher, you must set the following in `config.ini.php` with cookies from an existing FurAffinity account with the desired maturity ratings enabled in [Account Settings](https://www.furaffinity.net/controls/settings/). + +``` +[FurAffinityBridge] +aCookie = "your-a-cookie-value-here" ; from cookie "a" +bCookie = "your-b-cookie-value-here" ; from cookie "b" +``` + +To confirm the bridge is authenticated, the name of the authenticating account will be shown in the bridge's name once the bridge has been used at least once. (Example: `user's FurAffinity Bridge`) From 99b86c0e1c8972a6529cd470967e2258702238a8 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Mon, 4 Sep 2023 03:00:08 +0200 Subject: [PATCH 091/716] [GithubSearchBridge] repair bridge / handle new search ui (#3647) --- bridges/GithubSearchBridge.php | 90 ++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/bridges/GithubSearchBridge.php b/bridges/GithubSearchBridge.php index f477710f..0402a7c7 100644 --- a/bridges/GithubSearchBridge.php +++ b/bridges/GithubSearchBridge.php @@ -2,9 +2,10 @@ class GithubSearchBridge extends BridgeAbstract { - const MAINTAINER = 'corenting'; + const MAINTAINER = 'corenting, User123698745'; const NAME = 'Github Repositories Search'; - const URI = 'https://github.com/'; + const BASE_URI = 'https://github.com'; + const URI = self::BASE_URI . '/search'; const CACHE_TIMEOUT = 600; // 10min const DESCRIPTION = 'Returns a specified repositories search (sorted by recently updated)'; const PARAMETERS = [ [ @@ -18,60 +19,73 @@ class GithubSearchBridge extends BridgeAbstract public function collectData() { - $params = ['utf8' => '✓', - 'q' => urlencode($this->getInput('s')), - 's' => 'updated', - 'o' => 'desc', - 'type' => 'Repositories']; - $url = self::URI . 'search?' . http_build_query($params); + $html = getSimpleHTMLDOM(self::getURI()); - $html = getSimpleHTMLDOM($url); + $resultElement = $html->find('[data-testid="results-list"]', 0); + + foreach ($resultElement->children as $element) { + $titleElement = $element->find('.search-title', 0); + $descriptionElement = $element->find('div > .search-match', 0); + $topicElements = $element->find('a[href^="/topic"]'); + $languageElement = $element->find('li [aria-label$="language"]', 0); + $dateElement = $element->find('li [title*=" "]', 0); - foreach ($html->find('li.repo-list-item') as $element) { $item = []; + $item['uri'] = self::BASE_URI . $titleElement->find('a', 0)->href; + $item['title'] = trim($titleElement->plaintext); + $item['timestamp'] = strtotime($dateElement->attr['title']); - $uri = $element->find('.f4 a', 0)->href; - $uri = substr(self::URI, 0, -1) . $uri; - $item['uri'] = $uri; - - $title = $element->find('.f4', 0)->plaintext; - $item['title'] = $title; + $categories = []; // Description - if (count($element->find('p.mb-1')) != 0) { - $content = $element->find('p.mb-1', 0)->innertext; + $content = '<p>'; + if (isset($descriptionElement)) { + $content .= trim($descriptionElement->plaintext); } else { - $content = 'No description'; + $content .= 'No description'; } + $content .= '</p>'; - // Tags - $content = $content . '<br />'; - $tags = $element->find('a.topic-tag'); - $tags_array = []; - if (count($tags) != 0) { - $content = $content . 'Tags: '; - foreach ($tags as $tag_element) { - $tag_link = 'https://github.com' . $tag_element->href; - $tag_name = trim($tag_element->innertext); - $content = $content . '<a href="' . $tag_link . '">' . $tag_name . '</a> '; - array_push($tags_array, $tag_element->innertext); + // Topics + if (count($topicElements) > 0) { + $content .= '<p>'; + $content .= 'Topics: '; + foreach ($topicElements as $topicElement) { + $topicLink = self::BASE_URI . $topicElement->href; + $topicTitle = trim($topicElement->plaintext); + $content .= '<a href="' . $topicLink . '">' . $topicTitle . '</a> '; + $categories[] = $topicTitle; } + $content .= '</p>'; } // Programming language - if (count($element->find('span[itemprop=programmingLanguage]')) != 0) { - $language = $element->find('span[itemprop=programmingLanguage]', 0)->innertext; - - $content = $content . '<br />'; - $content = $content . 'Language: ' . $language; + if (isset($languageElement)) { + $content .= '<p>'; + $content .= 'Language: '; + $content .= trim($languageElement->plaintext); + $content .= '</p>'; } - $item['categories'] = $tags_array; $item['content'] = $content; - $date = $element->find('relative-time', 0)->datetime; - $item['timestamp'] = strtotime($date); + $item['categories'] = $categories; $this->items[] = $item; } } + + public function getURI() + { + $searchValue = $this->getInput('s'); + if (isset($searchValue)) { + $params = [ + 'q' => $searchValue, + 'type' => 'repositories', + 's' => 'updated', + 'o' => 'desc', + ]; + return self::URI . '?' . http_build_query($params); + } + return self::URI; + } } From 752098e0fa0988ab7f608832161c0f986bd80eb8 Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Tue, 5 Sep 2023 02:12:20 +0200 Subject: [PATCH 092/716] [Core] Fix Find Feed URL encoding (#3650) The URL entered by the user was not URL encoded for the find feed feature : this had lead to wrong content sent back to he server --- static/rss-bridge.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/rss-bridge.js b/static/rss-bridge.js index 82069d8c..9b6cce5d 100644 --- a/static/rss-bridge.js +++ b/static/rss-bridge.js @@ -56,7 +56,7 @@ var rssbridge_feed_finder = (function() { // Start the Feed search async function rssbridge_feed_search(event) { const input = document.getElementById('searchfield'); - let content = input.value; + let content = encodeURI(input.value); if (content) { const findfeedresults = document.getElementById('findfeedresults'); findfeedresults.innerHTML = 'Searching for matching feeds ...'; From 38b957398aec532f640550cddc9dd748133c3f27 Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Tue, 5 Sep 2023 02:12:47 +0200 Subject: [PATCH 093/716] [AutoJMBridge] Fix content extraction (#3649) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [AutoJMBridge] Fix content extraction - Website changed, bridge was updated accordingly - Added the function detectParameters - Added the test array for the detectParameters function * [AutoJMBridge] Fix test Fix content of the TEST_DETECT_PARAMETERS array * [AutoJMBridge] Update exaù^me value parameter Example value was not valid anymore, so it was updated --- bridges/AutoJMBridge.php | 41 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/bridges/AutoJMBridge.php b/bridges/AutoJMBridge.php index c9aaa660..98a86fcc 100644 --- a/bridges/AutoJMBridge.php +++ b/bridges/AutoJMBridge.php @@ -13,12 +13,20 @@ class AutoJMBridge extends BridgeAbstract 'type' => 'text', 'required' => true, 'title' => 'URL d\'une recherche avec filtre de véhicules sans le http://www.autojm.fr/', - 'exampleValue' => 'recherche?brands[]=peugeot&ranges[]=peugeot-nouvelle-308-2021-5p' + 'exampleValue' => 'recherche?brands[]=PEUGEOT&ranges[]=PEUGEOT 308' ], ] ]; + const CACHE_TIMEOUT = 3600; + const TEST_DETECT_PARAMETERS = [ + 'https://www.autojm.fr/recherche?brands%5B%5D=PEUGEOT&ranges%5B%5D=PEUGEOT%20308' + => ['url' => 'recherche?brands%5B%5D=PEUGEOT&ranges%5B%5D=PEUGEOT%20308', + 'context' => 'Afficher les offres de véhicules disponible sur la recheche AutoJM' + ] + ]; + public function getIcon() { return self::URI . 'favicon.ico'; @@ -35,6 +43,17 @@ class AutoJMBridge extends BridgeAbstract } } + public function getURI() + { + switch ($this->queriedContext) { + case 'Afficher les offres de véhicules disponible sur la recheche AutoJM': + return self::URI . $this->getInput('url'); + break; + default: + return self::URI; + } + } + public function collectData() { // Get the number of result for this search @@ -52,7 +71,7 @@ class AutoJMBridge extends BridgeAbstract $data = json_decode($json); $nb_results = $data->nbResults; - $total_pages = ceil($nb_results / 15); + $total_pages = ceil($nb_results / 14); // Limit the number of page to analyse to 10 for ($page = 1; $page <= $total_pages && $page <= 10; $page++) { @@ -66,8 +85,8 @@ class AutoJMBridge extends BridgeAbstract $image = $car->find('div[class=card-car__header__img]', 0)->find('img', 0)->src; // Decode HTML attribute JSON data $car_data = json_decode(html_entity_decode($car->{'data-layer'})); - $car_model = $car->{'data-title'} . ' ' . $car->{'data-suptitle'}; - $availability = $car->find('div[class=card-car__modalites]', 0)->find('div[class=col]', 0)->plaintext; + $car_model = $car_data->title; + $availability = $car->find('div[class*=card-car__modalites]', 0)->find('div[class=col]', 0)->plaintext; $warranty = $car->find('div[data-type=WarrantyCard]', 0)->plaintext; $discount_html = $car->find('div[class=subtext vehicle_reference_element]', 0); // Check if there is any discount info displayed @@ -132,4 +151,18 @@ class AutoJMBridge extends BridgeAbstract return $html; } + + public function detectParameters($url) + { + $params = []; + $regex = '/^(https?:\/\/)?(www\.|)autojm.fr\/(recherche\?.*|recherche\/[0-9]{1,10}\?.*)$/m'; + if (preg_match($regex, $url, $matches) > 0) { + $url = preg_replace('#(recherche|recherche/[0-9]{1,10})#', 'recherche', $matches[3]); + + $params['url'] = $url; + $params['context'] = 'Afficher les offres de véhicules disponible sur la recheche AutoJM'; + + return $params; + } + } } From 52b90e0873e56e55b39c4379748f54d55e46248e Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Wed, 6 Sep 2023 03:46:47 +0200 Subject: [PATCH 094/716] [Core] Fix Find Feed URL encoding (Really this time) (#3651) - the URL was only partially encoded because encodeURI() was used instead of encodeURIComponent() Now the whole URL is urlencoded, and the whole URL is passed as is in the GET parameter 'url' --- static/rss-bridge.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/rss-bridge.js b/static/rss-bridge.js index 9b6cce5d..2c45294c 100644 --- a/static/rss-bridge.js +++ b/static/rss-bridge.js @@ -56,7 +56,7 @@ var rssbridge_feed_finder = (function() { // Start the Feed search async function rssbridge_feed_search(event) { const input = document.getElementById('searchfield'); - let content = encodeURI(input.value); + let content = encodeURIComponent(input.value); if (content) { const findfeedresults = document.getElementById('findfeedresults'); findfeedresults.innerHTML = 'Searching for matching feeds ...'; From dbe37cc302bec84f622fbd65aa88690b0d946521 Mon Sep 17 00:00:00 2001 From: csisoap <33269526+csisoap@users.noreply.github.com> Date: Wed, 6 Sep 2023 21:14:11 +0700 Subject: [PATCH 095/716] [TwitterBridge] Filter out any promoted tweet (#3652) * Filter out any advertise tweet * Make some filter work, fix bug that may happen with tweet id list. * clear phpcs warning, ignore line length warning --- bridges/TwitterBridge.php | 84 ++++++++++++++++++++++----------------- lib/TwitterClient.php | 13 +++++- 2 files changed, 60 insertions(+), 37 deletions(-) diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index a5d09f8a..8470dcf7 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -217,7 +217,7 @@ EOD private function getFullText($id) { $url = sprintf( - 'https://cdn.syndication.twimg.com/tweet-result?id=%s&lang=en', + 'https://cdn.syndication.twimg.com/tweet-result?id=%s&lang=en&token=449yf2pc4g', $id ); @@ -306,55 +306,67 @@ EOD } } - // Array of Tweet IDs - $tweetIds = []; - // Filter out unwanted tweets - foreach ($data->tweets as $tweet) { - if (isset($tweet->rest_id)) { - $tweetIds[] = $tweet->rest_id; - $tweet = $tweet->legacy; - } - - if (!$tweet) { - continue; - } - // Filter out retweets to remove possible duplicates of original tweet - switch ($this->queriedContext) { - case 'By keyword or hashtag': - if ((isset($tweet->retweeted_status) || isset($tweet->retweeted_status_result)) && substr($tweet->full_text, 0, 4) === 'RT @') { - continue 2; - } - break; - } - $tweets[] = $tweet; - } - $hidePictures = $this->getInput('nopic'); $hidePinned = $this->getInput('nopinned'); if ($hidePinned) { $pinnedTweetId = null; - if ($user && $user->pinned_tweet_ids_str) { - $pinnedTweetId = $user->pinned_tweet_ids_str; + if ($data->user_info && $data->user_info->legacy->pinned_tweet_ids_str) { + $pinnedTweetId = $data->user_info->legacy->pinned_tweet_ids_str[0]; } } + // Array of Tweet IDs + $tweetIds = []; + // Filter out unwanted tweets + foreach ($data->tweets as $tweet) { + if (!$tweet) { + continue; + } + + if (isset($tweet->legacy)) { + $legacy_info = $tweet->legacy; + } else { + $legacy_info = $tweet; + } + + // Filter out retweets to remove possible duplicates of original tweet + switch ($this->queriedContext) { + case 'By keyword or hashtag': + // phpcs:ignore + if ((isset($legacy_info->retweeted_status) || isset($legacy_info->retweeted_status_result)) && substr($legacy_info->full_text, 0, 4) === 'RT @') { + continue 2; + } + break; + } + + // Skip own Retweets... + if (isset($legacy_info->retweeted_status) && $legacy_info->retweeted_status->user->id_str === $tweet->user->id_str) { + continue; + // phpcs:ignore + } elseif (isset($legacy_info->retweeted_status_result) && $tweet->retweeted_status_result->result->legacy->user_id_str === $legacy_info->user_id_str) { + continue; + } + + $tweetId = (isset($legacy_info->id_str) ? $legacy_info->id_str : $tweet->rest_id); + // Skip pinned tweet + if ($hidePinned && ($tweetId === $pinnedTweetId)) { + continue; + } + + if (isset($tweet->rest_id)) { + $tweetIds[] = $tweetId; + } + $rtweet = $legacy_info; + $tweets[] = $rtweet; + } + if ($this->queriedContext === 'By username') { $this->feedIconUrl = $data->user_info->legacy->profile_image_url_https ?? null; } $i = 0; foreach ($tweets as $tweet) { - // Skip own Retweets... - if (isset($tweet->retweeted_status) && $tweet->retweeted_status->user->id_str === $tweet->user->id_str) { - continue; - } - - // Skip pinned tweet - if ($hidePinned && $tweet->id_str === $pinnedTweetId) { - continue; - } - $item = []; $realtweet = $tweet; diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index d2a09fdd..20f21482 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -146,9 +146,14 @@ class TwitterClient } if (isset($timeline->data->user)) { - if (!isset($entry->content->itemContent->tweet_results->result->legacy)) { + if (!isset($entry->content->itemContent->tweet_results->result)) { continue; } + + if (isset($entry->content->itemContent->promotedMetadata)) { + continue; + } + $tweets[] = $entry->content->itemContent->tweet_results->result; $userIds[] = $entry->content->itemContent->tweet_results->result->core->user_results->result; @@ -156,6 +161,12 @@ class TwitterClient if (!isset($entry->content->content->tweetResult->result->legacy)) { continue; } + + // Filter out any advertise tweet + if (isset($entry->content->content->tweetPromotedMetadata)) { + continue; + } + $tweets[] = $entry->content->content->tweetResult->result; $userIds[] = $entry->content->content->tweetResult->result->core->user_result->result; From b3a784244808041a264c3bd548d2bdca9a143275 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Wed, 6 Sep 2023 23:46:25 +0930 Subject: [PATCH 096/716] [PixivBridge] Add cookie auth and options (#3653) * added cookie mgmt and support for issue https://github.com/RSS-Bridge/rss-bridge/issues/2759 * added image proxy option * + mature and ai options, + cookie doc * mention doc * check cookie is auth'd --- bridges/PixivBridge.php | 201 ++++++++++++++++++++++--- docs/10_Bridge_Specific/PixivBridge.md | 23 +++ 2 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 docs/10_Bridge_Specific/PixivBridge.md diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php index cf509855..5549c609 100644 --- a/bridges/PixivBridge.php +++ b/bridges/PixivBridge.php @@ -7,6 +7,17 @@ class PixivBridge extends BridgeAbstract const NAME = 'Pixiv Bridge'; const URI = 'https://www.pixiv.net/'; const DESCRIPTION = 'Returns the tag search from pixiv.net'; + const MAINTAINER = 'mruac'; + const CONFIGURATION = [ + 'cookie' => [ + 'required' => false, + 'defaultValue' => null + ], + 'proxy_url' => [ + 'required' => false, + 'defaultValue' => null + ] + ]; const PARAMETERS = [ @@ -23,11 +34,21 @@ class PixivBridge extends BridgeAbstract 'mode' => [ 'name' => 'Post Type', 'type' => 'list', - 'values' => ['All Works' => 'all', - 'Illustrations' => 'illustrations/', - 'Manga' => 'manga/', - 'Novels' => 'novels/'] + 'values' => [ + 'All Works' => 'all', + 'Illustrations' => 'illustrations/', + 'Manga' => 'manga/', + 'Novels' => 'novels/' + ] ], + 'mature' => [ + 'name' => 'Include R-18 works', + 'type' => 'checkbox' + ], + 'ai' => [ + 'name' => 'Include AI-Generated works', + 'type' => 'checkbox' + ] ], 'Tag' => [ 'tag' => [ @@ -76,7 +97,7 @@ class PixivBridge extends BridgeAbstract default: return parent::getName(); } - return 'Pixiv ' . $this->getKey('mode') . " from ${context} ${query}"; + return 'Pixiv ' . $this->getKey('mode') . " from {$context} {$query}"; } public function getURI() @@ -106,7 +127,7 @@ class PixivBridge extends BridgeAbstract break; case 'User': $uri = static::URI . 'ajax/user/' . $this->getInput('userid') - . '/profile/top'; + . '/profile/top'; break; default: returnClientError('Invalid Context'); @@ -116,18 +137,47 @@ class PixivBridge extends BridgeAbstract private function getDataFromJSON($json, $json_key) { - $json = $json['body'][$json_key]; + $key = $json_key; + if ( + $this->queriedContext === 'Tag' && + $this->getOption('cookie') !== null + ) { + switch ($json_key) { + case 'illust': + case 'manga': + $key = 'illustManga'; + break; + } + } + $json = $json['body'][$key]; // Tags context contains subkey - if ($this->queriedContext == 'Tag') { + if ($this->queriedContext === 'Tag') { $json = $json['data']; + if ($this->getOption('cookie') !== null) { + switch ($json_key) { + case 'illust': + $json = array_reduce($json, function ($acc, $i) { + if ($i['illustType'] === 0) { + $acc[] = $i; + }return $acc; + }, []); + break; + case 'manga': + $json = array_reduce($json, function ($acc, $i) { + if ($i['illustType'] === 1) { + $acc[] = $i; + }return $acc; + }, []); + break; + } + } } return $json; } private function collectWorksArray() { - $content = getContents($this->getSearchURI($this->getInput('mode'))); - $content = json_decode($content, true); + $content = $this->getData($this->getSearchURI($this->getInput('mode')), true, true); if ($this->getInput('mode') == 'all') { $total = []; foreach (self::JSON_KEY_MAP[$this->queriedContext] as $mode => $json_key) { @@ -144,14 +194,36 @@ class PixivBridge extends BridgeAbstract public function collectData() { + $this->checkOptions(); + $proxy_url = $this->getOption('proxy_url'); + $proxy_url = $proxy_url ? rtrim($proxy_url, '/') : null; + $content = $this->collectWorksArray(); $content = array_filter($content, function ($v, $k) { return !array_key_exists('isAdContainer', $v); }, ARRAY_FILTER_USE_BOTH); + // Sort by updateDate to get newest works usort($content, function ($a, $b) { return $b['updateDate'] <=> $a['updateDate']; }); + + //exclude AI generated works if unchecked. + if ($this->getInput('ai') !== true) { + $content = array_filter($content, function ($v) { + $isAI = $v['aiType'] === 2; + return !$isAI; + }); + } + + //exclude R-18 works if unchecked. + if ($this->getInput('mature') !== true) { + $content = array_filter($content, function ($v) { + $isMature = $v['xRestrict'] > 0; + return !$isMature; + }); + } + $content = array_slice($content, 0, $this->getInput('posts')); foreach ($content as $result) { @@ -168,12 +240,25 @@ class PixivBridge extends BridgeAbstract $item['author'] = $result['userName']; $item['timestamp'] = $result['updateDate']; $item['categories'] = $result['tags']; - $cached_image = $this->cacheImage( - $result['url'], - $result['id'], - array_key_exists('illustType', $result) - ); - $item['content'] = "<img src='" . $cached_image . "' />"; + + if ($proxy_url) { + //use proxy image host if set. + if ($this->getInput('fullsize')) { + $ajax_uri = static::URI . 'ajax/illust/' . $result['id']; + $imagejson = $this->getData($ajax_uri, true, true); + $img_url = preg_replace('/https:\/\/i\.pximg\.net/', $proxy_url, $imagejson['body']['urls']['original']); + } else { + $img_url = preg_replace('/https:\/\/i\.pximg\.net/', $proxy_url, $result['url']); + } + } else { + //else cache and use image. + $img_url = $this->cacheImage( + $result['url'], + $result['id'], + array_key_exists('illustType', $result) + ); + } + $item['content'] = "<img src='" . $img_url . "' />"; // Additional content items if (array_key_exists('pageCount', $result)) { @@ -188,6 +273,7 @@ class PixivBridge extends BridgeAbstract /** * todo: remove manual file cache + * See bridge specific documentation for alternative option. */ private function cacheImage($url, $illustId, $isImage) { @@ -209,19 +295,96 @@ class PixivBridge extends BridgeAbstract // Get fullsize URL if ($isImage && $this->getInput('fullsize')) { $ajax_uri = static::URI . 'ajax/illust/' . $illustId; - $imagejson = json_decode(getContents($ajax_uri), true); + $imagejson = $this->getData($ajax_uri, true, true); $url = $imagejson['body']['urls']['original']; } $headers = ['Referer: ' . static::URI]; try { - $illust = getContents($url, $headers); + $illust = $this->getData($url, true, false, $headers); } catch (Exception $e) { - $illust = getContents($thumbnailurl, $headers); // Original thumbnail + $illust = $this->getData($thumbnailurl, true, false, $headers); // Original thumbnail } file_put_contents($path, $illust); } return get_home_page_url() . 'cache/pixiv_img/' . preg_replace('/.*\//', '', $path); } + + private function checkOptions() + { + $proxy = $this->getOption('proxy_url'); + if ($proxy) { + if ( + !(strlen($proxy) > 0 && preg_match('/https?:\/\/.*/', $proxy)) + ) { + return returnServerError('Invalid proxy_url value set. The proxy must include the HTTP/S at the beginning of the url.'); + } + } + + $cookie = $this->getCookie(); + if ($cookie) { + $isAuth = $this->loadCacheValue('is_authenticated'); + if (!$isAuth) { + $res = $this->getData('https://www.pixiv.net/ajax/webpush', true, true) + or returnServerError('Invalid PHPSESSID cookie provided. Please check the 🍪 and try again.'); + if ($res['error'] === false) { + $this->saveCacheValue('is_authenticated', true); + } + } + } + } + + private function checkCookie(array $headers) + { + if (array_key_exists('set-cookie', $headers)) { + foreach ($headers['set-cookie'] as $value) { + if (str_starts_with($value, 'PHPSESSID=')) { + parse_str(strtr($value, ['&' => '%26', '+' => '%2B', ';' => '&']), $cookie); + if ($cookie['PHPSESSID'] != $this->getCookie()) { + $this->saveCacheValue('cookie', $cookie['PHPSESSID']); + } + break; + } + } + } + } + + private function getCookie() + { + // checks if cookie is set, if not initialise it with the cookie from the config + $value = $this->loadCacheValue('cookie', 2678400 /* 30 days + 1 day to let cookie chance to renew */); + if (!isset($value)) { + $value = $this->getOption('cookie'); + $this->saveCacheValue('cookie', $this->getOption('cookie')); + } + return $value; + } + + //Cache getContents by default + private function getData(string $url, bool $cache = true, bool $getJSON = false, array $httpHeaders = [], array $curlOptions = []) + { + $cookie_str = $this->getCookie(); + if ($cookie_str) { + $curlOptions[CURLOPT_COOKIE] = 'PHPSESSID=' . $cookie_str; + } + + if ($cache) { + $data = $this->loadCacheValue($url, 86400); // 24 hours + if (!$data) { + $data = getContents($url, $httpHeaders, $curlOptions, true) or returnServerError("Could not load $url"); + $this->saveCacheValue($url, $data); + } + } else { + $data = getContents($url, $httpHeaders, $curlOptions, true) or returnServerError("Could not load $url"); + } + + $this->checkCookie($data['headers']); + + if ($getJSON) { + return json_decode($data['content'], true); + } else { + return $data['content']; + } + } } diff --git a/docs/10_Bridge_Specific/PixivBridge.md b/docs/10_Bridge_Specific/PixivBridge.md new file mode 100644 index 00000000..b782a445 --- /dev/null +++ b/docs/10_Bridge_Specific/PixivBridge.md @@ -0,0 +1,23 @@ +PixivBridge +=============== + +# Image proxy +As Pixiv requires images to be loaded with the `Referer "https://www.pixiv.net/"` header set, caching or image proxy is required to use this bridge. + +To turn off image caching, set the `proxy_url` value in this bridge's configuration section of `config.ini.php` to the url of the proxy. The bridge will then use the proxy in this format (essentially replacing `https://i.pximg.net` with the proxy): + +Before: `https://i.pximg.net/img-original/img/0000/00/00/00/00/00/12345678_p0.png` + +After: `https://proxy.example.com/img-original/img/0000/00/00/00/00/00/12345678_p0.png` + +``` +proxy_url = "https://proxy.example.com" +``` + +# Authentication +Authentication is required to view and search R-18+ and non-public images. To enable this, set the following in this bridge's configuration in `config.ini.php`. + +``` +; from cookie "PHPSESSID". Recommend to get in incognito browser. +cookie = "00000000_hashedsessionidhere" +``` \ No newline at end of file From 586d707ae48b74b150eb101f2e9986d0bd18d89f Mon Sep 17 00:00:00 2001 From: July <phantop@tuta.io> Date: Sat, 9 Sep 2023 03:19:09 -0400 Subject: [PATCH 097/716] [ArsTechnicaBridge] Add new bridge (#3657) --- bridges/ArsTechnicaBridge.php | 71 +++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 bridges/ArsTechnicaBridge.php diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php new file mode 100644 index 00000000..1e3e6379 --- /dev/null +++ b/bridges/ArsTechnicaBridge.php @@ -0,0 +1,71 @@ +<?php + +class ArsTechnicaBridge extends FeedExpander +{ + const MAINTAINER = 'phantop'; + const NAME = 'Ars Technica'; + const URI = 'https://arstechnica.com/'; + const DESCRIPTION = 'Returns the latest articles from Ars Technica'; + const PARAMETERS = [[ + 'section' => [ + 'name' => 'Site section', + 'type' => 'list', + 'defaultValue' => 'index', + 'values' => [ + 'All' => 'index', + 'Apple' => 'apple', + 'Board Games' => 'cardboard', + 'Cars' => 'cars', + 'Features' => 'features', + 'Gaming' => 'gaming', + 'Information Technology' => 'technology-lab', + 'Science' => 'science', + 'Staff Blogs' => 'staff-blogs', + 'Tech Policy' => 'tech-policy', + 'Tech' => 'gadgets', + ] + ] + ]]; + + public function collectData() + { + $url = 'https://feeds.arstechnica.com/arstechnica/' . $this->getInput('section'); + $this->collectExpandableDatas($url); + } + + protected function parseItem($newItem) + { + $item = parent::parseItem($newItem); + + $item_html = getSimpleHTMLDOMCached($item['uri'] . '&'); + $item_html = defaultLinkTo($item_html, self::URI); + $item['content'] = $item_html->find('.amp-wp-article-content', 0); + + // remove various ars advertising + $item['content']->find('#social-left', 0)->remove(); + foreach ($item['content']->find('.ars-component-buy-box') as $ad) { + $ad->remove(); + } + foreach ($item['content']->find('i-amphtml-sizer') as $ad) { + $ad->remove(); + } + foreach ($item['content']->find('.sidebar') as $ad) { + $ad->remove(); + } + + foreach ($item['content']->find('a') as $link) { //remove amp redirect links + $url = $link->getAttribute('href'); + if (str_contains($url, 'go.redirectingat.com')) { + $url = extractFromDelimiters($url, 'url=', '&'); + $url = urldecode($url); + $link->setAttribute('href', $url); + } + } + + $item['content'] = backgroundToImg(str_replace('data-amp-original-style="background-image', 'style="background-image', $item['content'])); + + $item['uid'] = explode('=', $item['uri'])[1]; + + return $item; + } +} From 078091752ad2dbd6d4ca261d4b8e3ce333739e3a Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Sun, 10 Sep 2023 04:03:38 +0200 Subject: [PATCH 098/716] doc : Add documentation for the Findfeed action (#3659) * doc: Add documentation for the Findfeed action Added the documentation to the Findfeed action * doc: Add documentation for the Findfeed action - Complete documentation - fix typos --- docs/04_For_Developers/04_Actions.md | 94 +++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/docs/04_For_Developers/04_Actions.md b/docs/04_For_Developers/04_Actions.md index c346731a..74c8cbfb 100644 --- a/docs/04_For_Developers/04_Actions.md +++ b/docs/04_For_Developers/04_Actions.md @@ -1,8 +1,9 @@ -RSS-Bridge currently supports three 'actions' which it can operate: +RSS-Bridge currently supports four 'actions' which it can operate: 1) [Display](#display) (`?action=display`) 2) [Detect](#detect) (`?action=detect`) 3) [List](#list) (`?action=list`) +3) [FindFeed](#findfeed) (`?action=findfeed`) ## Display @@ -19,7 +20,7 @@ The `detect` action attempts to redirect the user to an appropriate `display` ac If an appropriate bridge is found, a `301 Moved Permanently` HTTP status code is returned with a relative location for a `display` action. If no appropriate bridge is found or a required parameter is missing, a `400 Bad Request` status code is returned. -The parameters for this action are listed bellow: +The parameters for this action are listed below: Parameter | Required | Description ----------|----------|------------ @@ -75,4 +76,91 @@ Parameter | Optional | Description ### `total` -This parameter represents the total number of bridges available to the current instance of RSS-Bridge. \ No newline at end of file +This parameter represents the total number of bridges available to the current instance of RSS-Bridge. + +## FindFeed + +The `findfeed` action attempts to list all available feeds based on a supplied URL for the active bridges of this instance. As bridges have to individually implement `detectParameters`, this it may not work for every bridge. + +If one or more bridges return a feed, a JSON data array structure is returned. If no feeds were found, a `404 Not Found` status code is returned. If a required parameter is missing, a `400 Bad Request` status code is returned. + +For each feed, the whole feed URL is sent in the `url` member, the feed specific bridge parameters metadata in the `bridgeData` member and the Bridge metadata in the `bridgeMeta` member. + +This example shows JSON data for the NASA Instagram account URL (`https://www.instagram.com/nasa/`) using the `Html` format : + +```JSON +[ + { + "url": "https://rssbridge.host/?action=display&context=Username&u=nasa&bridge=InstagramBridge&format=Html", + "bridgeParams": { + "context": "Username", + "u": "nasa", + "bridge": "InstagramBridge", + "format": "Html" + }, + "bridgeData": { + "context": { + "name": "Context", + "value": "Username" + }, + "u": { + "name": "username", + "value": "nasa" + } + }, + "bridgeMeta": { + "name": "Instagram Bridge", + "description": "Returns the newest images", + "parameters": { + "Username": { + "u": { + "name": "username", + "exampleValue": "aesoprockwins", + "required": true + } + }, + "Hashtag": { + "h": { + "name": "hashtag", + "exampleValue": "beautifulday", + "required": true + } + }, + "Location": { + "l": { + "name": "location", + "exampleValue": "london", + "required": true + } + }, + "global": { + "media_type": { + "name": "Media type", + "type": "list", + "required": false, + "values": { + "All": "all", + "Video": "video", + "Picture": "picture", + "Multiple": "multiple" + }, + "defaultValue": "all" + }, + "direct_links": { + "name": "Use direct media links", + "type": "checkbox" + } + } + }, + "icon": "https://www.instagram.com//favicon.ico" + } + } +] +``` + +The parameters for this action are listed below: + +Parameter | Required | Description +----------|----------|------------ +`url` | yes | Specifies the URL to attempt to find a feed from. The value of this should be URL encoded. +`format` | yes | Specifies the name of the format to use for the URL of the feeds. This is passed to the detected `display` action. Possible values are determined from the formats available to the current instance of RSS-Bridge. From a786bbd4e0d45a98446ed0384ed57705a17b89d3 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Sun, 10 Sep 2023 11:45:05 +0930 Subject: [PATCH 099/716] DisplayAction: defaultchecked fix (#3654) * . * attempt to fix #2943 https://github.com/RSS-Bridge/rss-bridge/issues/2943 * Revert "." This reverts commit c0b6ccfea6ce873e9c9ce7c3600b3a96d9911468. * lint * Revert "attempt to fix #2943" This reverts commit 9f1a66e48d636a543e2171df212acf9731744bd0. * moved fix to BridgeAbstract --- lib/BridgeAbstract.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index e58ddb91..2a6cd8ab 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -160,11 +160,7 @@ abstract class BridgeAbstract implements BridgeInterface switch ($type) { case 'checkbox': - if (!isset($properties['defaultValue'])) { - $this->inputs[$context][$name]['value'] = false; - } else { - $this->inputs[$context][$name]['value'] = $properties['defaultValue']; - } + $this->inputs[$context][$name]['value'] = $inputs[$context][$name]['value'] ?? false; break; case 'list': if (!isset($properties['defaultValue'])) { @@ -191,10 +187,14 @@ abstract class BridgeAbstract implements BridgeInterface foreach (static::PARAMETERS['global'] as $name => $properties) { if (isset($inputs[$name])) { $value = $inputs[$name]; - } elseif (isset($properties['defaultValue'])) { - $value = $properties['defaultValue']; } else { - continue; + if ($properties['type'] === 'checkbox') { + $value = false; + } elseif (isset($properties['defaultValue'])) { + $value = $properties['defaultValue']; + } else { + continue; + } } $this->inputs[$queriedContext][$name]['value'] = $value; } From 4b9f6f7e53e0b2e9aae59df2bbffc0bdd6805aea Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 10 Sep 2023 21:50:15 +0200 Subject: [PATCH 100/716] fix: rewrite and improve caching (#3594) --- README.md | 13 +- actions/ConnectivityAction.php | 32 +- actions/DetectAction.php | 2 +- actions/DisplayAction.php | 264 +++++++---------- actions/ListAction.php | 2 +- actions/SetBridgeCacheAction.php | 16 +- bridges/AO3Bridge.php | 6 +- bridges/BugzillaBridge.php | 2 +- bridges/ElloBridge.php | 17 +- bridges/InstagramBridge.php | 15 +- bridges/MastodonBridge.php | 2 +- bridges/RedditBridge.php | 22 ++ bridges/SoundcloudBridge.php | 15 +- bridges/SpotifyBridge.php | 12 +- bridges/TwitterBridge.php | 152 ---------- bridges/WordPressMadaraBridge.php | 2 +- bridges/YoutubeBridge.php | 243 ++++++++------- caches/ArrayCache.php | 52 ++++ caches/FileCache.php | 161 ++++------ caches/MemcachedCache.php | 98 ++----- caches/NullCache.php | 22 +- caches/SQLiteCache.php | 98 +++---- config.default.ini.php | 2 +- .../prepare_release/fetch_contributors.php | 9 +- docs/06_Helper_functions/index.md | 43 +-- docs/07_Cache_API/02_CacheInterface.md | 14 +- docs/index.md | 6 +- index.php | 4 + lib/BridgeAbstract.php | 19 +- lib/BridgeInterface.php | 2 + lib/CacheFactory.php | 24 +- lib/CacheInterface.php | 12 +- lib/Configuration.php | 7 +- lib/FeedExpander.php | 6 +- lib/FormatInterface.php | 10 +- lib/Logger.php | 17 +- lib/RssBridge.php | 98 ++++--- lib/TwitterClient.php | 18 +- lib/bootstrap.php | 2 +- lib/contents.php | 276 +++--------------- lib/error.php | 47 --- lib/http.php | 252 ++++++++++++++++ lib/utils.php | 29 +- tests/Actions/ListActionTest.php | 3 +- tests/CacheTest.php | 14 +- 45 files changed, 993 insertions(+), 1169 deletions(-) create mode 100644 caches/ArrayCache.php delete mode 100644 lib/error.php create mode 100644 lib/http.php diff --git a/README.md b/README.md index e0487e6b..dee69b85 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,10 @@ Alternatively find another Requires minimum PHP 7.4. +```shell +apt install nginx php-fpm php-mbstring php-simplexml php-curl +``` + ```shell cd /var/www composer create-project -v --no-dev rss-bridge/rss-bridge @@ -334,10 +338,11 @@ This is the feed item structure that bridges are expected to produce. ### Cache backends -* `file` -* `sqlite` -* `memcached` -* `null` +* `File` +* `SQLite` +* `Memcached` +* `Array` +* `Null` ### Licenses diff --git a/actions/ConnectivityAction.php b/actions/ConnectivityAction.php index 604b7806..3bc82a9d 100644 --- a/actions/ConnectivityAction.php +++ b/actions/ConnectivityAction.php @@ -34,7 +34,7 @@ class ConnectivityAction implements ActionInterface public function execute(array $request) { if (!Debug::isEnabled()) { - throw new \Exception('This action is only available in debug mode!'); + return new Response('This action is only available in debug mode!'); } $bridgeName = $request['bridge'] ?? null; @@ -43,7 +43,7 @@ class ConnectivityAction implements ActionInterface } $bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName); if (!$bridgeClassName) { - throw new \Exception(sprintf('Bridge not found: %s', $bridgeName)); + return new Response('Bridge not found', 404); } return $this->reportBridgeConnectivity($bridgeClassName); } @@ -54,29 +54,25 @@ class ConnectivityAction implements ActionInterface throw new \Exception('Bridge is not whitelisted!'); } - $retVal = [ - 'bridge' => $bridgeClassName, - 'successful' => false, - 'http_code' => 200, - ]; - $bridge = $this->bridgeFactory->create($bridgeClassName); $curl_opts = [ - CURLOPT_CONNECTTIMEOUT => 5 + CURLOPT_CONNECTTIMEOUT => 5, + CURLOPT_FOLLOWLOCATION => true, + ]; + $result = [ + 'bridge' => $bridgeClassName, + 'successful' => false, + 'http_code' => null, ]; try { - $reply = getContents($bridge::URI, [], $curl_opts, true); - - if ($reply['code'] === 200) { - $retVal['successful'] = true; - if (strpos(implode('', $reply['status_lines']), '301 Moved Permanently')) { - $retVal['http_code'] = 301; - } + $response = getContents($bridge::URI, [], $curl_opts, true); + $result['http_code'] = $response['code']; + if (in_array($response['code'], [200])) { + $result['successful'] = true; } } catch (\Exception $e) { - $retVal['successful'] = false; } - return new Response(Json::encode($retVal), 200, ['Content-Type' => 'text/json']); + return new Response(Json::encode($result), 200, ['content-type' => 'text/json']); } } diff --git a/actions/DetectAction.php b/actions/DetectAction.php index 6c9fa22d..49b7ced7 100644 --- a/actions/DetectAction.php +++ b/actions/DetectAction.php @@ -45,7 +45,7 @@ class DetectAction implements ActionInterface $bridgeParams['format'] = $format; $url = '?action=display&' . http_build_query($bridgeParams); - return new Response('', 301, ['Location' => $url]); + return new Response('', 301, ['location' => $url]); } throw new \Exception('No bridge found for given URL: ' . $targetURL); diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 7b2efec1..7c59b3d5 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -10,50 +10,41 @@ class DisplayAction implements ActionInterface return new Response('503 Service Unavailable', 503); } $this->cache = RssBridge::getCache(); - $this->cache->setScope('http'); - $this->cache->setKey($request); - // avg timeout of 20m - $timeout = 60 * 15 + rand(1, 60 * 10); + $cacheKey = 'http_' . json_encode($request); /** @var Response $cachedResponse */ - $cachedResponse = $this->cache->loadData($timeout); - if ($cachedResponse && !Debug::isEnabled()) { - //Logger::info(sprintf('Returning cached (http) response: %s', $cachedResponse->getBody())); + $cachedResponse = $this->cache->get($cacheKey); + if ($cachedResponse) { + $ifModifiedSince = $_SERVER['HTTP_IF_MODIFIED_SINCE'] ?? null; + $lastModified = $cachedResponse->getHeader('last-modified'); + if ($ifModifiedSince && $lastModified) { + $lastModified = new \DateTimeImmutable($lastModified); + $lastModifiedTimestamp = $lastModified->getTimestamp(); + $modifiedSince = strtotime($ifModifiedSince); + if ($lastModifiedTimestamp <= $modifiedSince) { + $modificationTimeGMT = gmdate('D, d M Y H:i:s ', $lastModifiedTimestamp); + return new Response('', 304, ['last-modified' => $modificationTimeGMT . 'GMT']); + } + } return $cachedResponse; } - $response = $this->createResponse($request); - if (in_array($response->getCode(), [429, 503])) { - //Logger::info(sprintf('Storing cached (http) response: %s', $response->getBody())); - $this->cache->setScope('http'); - $this->cache->setKey($request); - $this->cache->saveData($response); - } - return $response; - } - - private function createResponse(array $request) - { - $bridgeFactory = new BridgeFactory(); - $formatFactory = new FormatFactory(); $bridgeName = $request['bridge'] ?? null; - $format = $request['format'] ?? null; - + if (!$bridgeName) { + return new Response('Missing bridge param', 400); + } + $bridgeFactory = new BridgeFactory(); $bridgeClassName = $bridgeFactory->createBridgeClassName($bridgeName); if (!$bridgeClassName) { - throw new \Exception(sprintf('Bridge not found: %s', $bridgeName)); + return new Response('Bridge not found', 404); } + $format = $request['format'] ?? null; if (!$format) { - throw new \Exception('You must specify a format!'); + return new Response('You must specify a format!', 400); } if (!$bridgeFactory->isEnabled($bridgeClassName)) { - throw new \Exception('This bridge is not whitelisted'); + return new Response('This bridge is not whitelisted', 400); } - $format = $formatFactory->create($format); - - $bridge = $bridgeFactory->create($bridgeClassName); - $bridge->loadConfiguration(); - $noproxy = $request['_noproxy'] ?? null; if ( Configuration::getConfig('proxy', 'url') @@ -64,147 +55,100 @@ class DisplayAction implements ActionInterface define('NOPROXY', true); } - $cacheTimeout = $request['_cache_timeout'] ?? null; - if (Configuration::getConfig('cache', 'custom_timeout') && $cacheTimeout) { - $cacheTimeout = (int) $cacheTimeout; - } else { - // At this point the query argument might still be in the url but it won't be used - $cacheTimeout = $bridge->getCacheTimeout(); + $bridge = $bridgeFactory->create($bridgeClassName); + $formatFactory = new FormatFactory(); + $format = $formatFactory->create($format); + + $response = $this->createResponse($request, $bridge, $format); + + if ($response->getCode() === 200) { + $ttl = $request['_cache_timeout'] ?? null; + if (Configuration::getConfig('cache', 'custom_timeout') && $ttl) { + $ttl = (int) $ttl; + } else { + $ttl = $bridge->getCacheTimeout(); + } + $this->cache->set($cacheKey, $response, $ttl); } - // Remove parameters that don't concern bridges - $bridge_params = array_diff_key( - $request, - array_fill_keys( - [ - 'action', - 'bridge', - 'format', - '_noproxy', - '_cache_timeout', - '_error_time' - ], - '' - ) - ); + if (in_array($response->getCode(), [429, 503])) { + $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); // average 20m + } - // Remove parameters that don't concern caches - $cache_params = array_diff_key( - $request, - array_fill_keys( - [ - 'action', - 'format', - '_noproxy', - '_cache_timeout', - '_error_time' - ], - '' - ) - ); - - $this->cache->setScope(''); - $this->cache->setKey($cache_params); + if ($response->getCode() === 500) { + $this->cache->set($cacheKey, $response, 60 * 15); + } + if (rand(1, 100) === 2) { + $this->cache->prune(); + } + return $response; + } + private function createResponse(array $request, BridgeInterface $bridge, FormatInterface $format) + { $items = []; $infos = []; - $feed = $this->cache->loadData($cacheTimeout); - - if ($feed && !Debug::isEnabled()) { - if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) { - $modificationTime = $this->cache->getTime(); - // The client wants to know if the feed has changed since its last check - $modifiedSince = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']); - if ($modificationTime <= $modifiedSince) { - $modificationTimeGMT = gmdate('D, d M Y H:i:s ', $modificationTime); - return new Response('', 304, ['Last-Modified' => $modificationTimeGMT . 'GMT']); + try { + $bridge->loadConfiguration(); + // Remove parameters that don't concern bridges + $bridgeData = array_diff_key($request, array_fill_keys(['action', 'bridge', 'format', '_noproxy', '_cache_timeout', '_error_time'], '')); + $bridge->setDatas($bridgeData); + $bridge->collectData(); + $items = $bridge->getItems(); + if (isset($items[0]) && is_array($items[0])) { + $feedItems = []; + foreach ($items as $item) { + $feedItems[] = new FeedItem($item); + } + $items = $feedItems; + } + $infos = [ + 'name' => $bridge->getName(), + 'uri' => $bridge->getURI(), + 'donationUri' => $bridge->getDonationURI(), + 'icon' => $bridge->getIcon() + ]; + } catch (\Exception $e) { + $errorOutput = Configuration::getConfig('error', 'output'); + $reportLimit = Configuration::getConfig('error', 'report_limit'); + if ($e instanceof HttpException) { + // Reproduce (and log) these responses regardless of error output and report limit + if ($e->getCode() === 429) { + Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); + return new Response('429 Too Many Requests', 429); + } + if ($e->getCode() === 503) { + Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); + return new Response('503 Service Unavailable', 503); } } - - if (isset($feed['items']) && isset($feed['extraInfos'])) { - foreach ($feed['items'] as $item) { - $items[] = new FeedItem($item); - } - $infos = $feed['extraInfos']; + Logger::error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]); + $errorCount = 1; + if ($reportLimit > 1) { + $errorCount = $this->logBridgeError($bridge->getName(), $e->getCode()); } - } else { - try { - $bridge->setDatas($bridge_params); - $bridge->collectData(); - $items = $bridge->getItems(); - if (isset($items[0]) && is_array($items[0])) { - $feedItems = []; - foreach ($items as $item) { - $feedItems[] = new FeedItem($item); - } - $items = $feedItems; - } - $infos = [ - 'name' => $bridge->getName(), - 'uri' => $bridge->getURI(), - 'donationUri' => $bridge->getDonationURI(), - 'icon' => $bridge->getIcon() - ]; - } catch (\Exception $e) { - $errorOutput = Configuration::getConfig('error', 'output'); - $reportLimit = Configuration::getConfig('error', 'report_limit'); - if ($e instanceof HttpException) { - // Reproduce (and log) these responses regardless of error output and report limit - if ($e->getCode() === 429) { - Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e))); - return new Response('429 Too Many Requests', 429); - } - if ($e->getCode() === 503) { - Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e))); - return new Response('503 Service Unavailable', 503); - } - // Might want to cache other codes such as 504 Gateway Timeout - } - if (in_array($errorOutput, ['feed', 'none'])) { - Logger::error(sprintf('Exception in DisplayAction(%s): %s', $bridgeClassName, create_sane_exception_message($e)), ['e' => $e]); - } - $errorCount = 1; - if ($reportLimit > 1) { - $errorCount = $this->logBridgeError($bridge->getName(), $e->getCode()); - } - // Let clients know about the error if we are passed the report limit - if ($errorCount >= $reportLimit) { - if ($errorOutput === 'feed') { - // Render the exception as a feed item - $items[] = $this->createFeedItemFromException($e, $bridge); - } elseif ($errorOutput === 'http') { - // Rethrow so that the main exception handler in RssBridge.php produces an HTTP 500 - throw $e; - } elseif ($errorOutput === 'none') { - // Do nothing (produces an empty feed) - } else { - // Do nothing, unknown error output? Maybe throw exception or validate in Configuration.php - } + // Let clients know about the error if we are passed the report limit + if ($errorCount >= $reportLimit) { + if ($errorOutput === 'feed') { + // Render the exception as a feed item + $items[] = $this->createFeedItemFromException($e, $bridge); + } elseif ($errorOutput === 'http') { + return new Response(render(__DIR__ . '/../templates/error.html.php', ['e' => $e]), 500); + } elseif ($errorOutput === 'none') { + // Do nothing (produces an empty feed) } } - - // Unfortunately need to set scope and key again because they might be modified - $this->cache->setScope(''); - $this->cache->setKey($cache_params); - $this->cache->saveData([ - 'items' => array_map(function (FeedItem $item) { - return $item->toArray(); - }, $items), - 'extraInfos' => $infos - ]); - $this->cache->purgeCache(); } $format->setItems($items); $format->setExtraInfos($infos); - $newModificationTime = $this->cache->getTime(); - $format->setLastModified($newModificationTime); - $headers = []; - if ($newModificationTime) { - $headers['Last-Modified'] = gmdate('D, d M Y H:i:s ', $newModificationTime) . 'GMT'; - } - $headers['Content-Type'] = $format->getMimeType() . '; charset=' . $format->getCharset(); + $now = time(); + $format->setLastModified($now); + $headers = [ + 'last-modified' => gmdate('D, d M Y H:i:s ', $now) . 'GMT', + 'content-type' => $format->getMimeType() . '; charset=' . $format->getCharset(), + ]; return new Response($format->stringify(), 200, $headers); } @@ -234,9 +178,8 @@ class DisplayAction implements ActionInterface private function logBridgeError($bridgeName, $code) { - $this->cache->setScope('error_reporting'); - $this->cache->setkey([$bridgeName . '_' . $code]); - $report = $this->cache->loadData(); + $cacheKey = 'error_reporting_' . $bridgeName . '_' . $code; + $report = $this->cache->get($cacheKey); if ($report) { $report = Json::decode($report); $report['time'] = time(); @@ -248,7 +191,8 @@ class DisplayAction implements ActionInterface 'count' => 1, ]; } - $this->cache->saveData(Json::encode($report)); + $ttl = 86400 * 5; + $this->cache->set($cacheKey, Json::encode($report), $ttl); return $report['count']; } diff --git a/actions/ListAction.php b/actions/ListAction.php index 6ce7e33e..9025bf6e 100644 --- a/actions/ListAction.php +++ b/actions/ListAction.php @@ -37,6 +37,6 @@ class ListAction implements ActionInterface ]; } $list->total = count($list->bridges); - return new Response(Json::encode($list), 200, ['Content-Type' => 'application/json']); + return new Response(Json::encode($list), 200, ['content-type' => 'application/json']); } } diff --git a/actions/SetBridgeCacheAction.php b/actions/SetBridgeCacheAction.php index 416f2378..a8e712d4 100644 --- a/actions/SetBridgeCacheAction.php +++ b/actions/SetBridgeCacheAction.php @@ -19,7 +19,10 @@ class SetBridgeCacheAction implements ActionInterface $authenticationMiddleware = new ApiAuthenticationMiddleware(); $authenticationMiddleware($request); - $key = $request['key'] or returnClientError('You must specify key!'); + $key = $request['key'] ?? null; + if (!$key) { + returnClientError('You must specify key!'); + } $bridgeFactory = new BridgeFactory(); @@ -40,13 +43,10 @@ class SetBridgeCacheAction implements ActionInterface $value = $request['value']; $cache = RssBridge::getCache(); - $cache->setScope(get_class($bridge)); - if (!is_array($key)) { - // not sure if $key is an array when it comes in from request - $key = [$key]; - } - $cache->setKey($key); - $cache->saveData($value); + + $cacheKey = get_class($bridge) . '_' . $key; + $ttl = 86400 * 3; + $cache->set($cacheKey, $value, $ttl); header('Content-Type: text/plain'); echo 'done'; diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php index 57e12fbd..e30c6b70 100644 --- a/bridges/AO3Bridge.php +++ b/bridges/AO3Bridge.php @@ -33,6 +33,7 @@ class AO3Bridge extends BridgeAbstract ], ] ]; + private $title; public function collectData() { @@ -94,11 +95,12 @@ class AO3Bridge extends BridgeAbstract $url = self::URI . "/works/$id/navigate"; $httpClient = RssBridge::getHttpClient(); + $version = 'v0.0.1'; $response = $httpClient->request($url, [ - 'useragent' => 'rss-bridge bot (https://github.com/RSS-Bridge/rss-bridge)', + 'useragent' => "rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)", ]); - $html = \str_get_html($response['body']); + $html = \str_get_html($response->getBody()); $html = defaultLinkTo($html, self::URI); $this->title = $html->find('h2 a', 0)->plaintext; diff --git a/bridges/BugzillaBridge.php b/bridges/BugzillaBridge.php index 9b4d1adc..c2dc8d40 100644 --- a/bridges/BugzillaBridge.php +++ b/bridges/BugzillaBridge.php @@ -159,7 +159,7 @@ class BugzillaBridge extends BridgeAbstract protected function getUser($user) { // Check if the user endpoint is available - if ($this->loadCacheValue($this->instance . 'userEndpointClosed', 86400)) { + if ($this->loadCacheValue($this->instance . 'userEndpointClosed')) { return $user; } diff --git a/bridges/ElloBridge.php b/bridges/ElloBridge.php index 4cc1858b..9017bc11 100644 --- a/bridges/ElloBridge.php +++ b/bridges/ElloBridge.php @@ -114,18 +114,17 @@ class ElloBridge extends BridgeAbstract private function getAPIKey() { $cache = RssBridge::getCache(); - $cache->setScope('ElloBridge'); - $cache->setKey(['key']); - $key = $cache->loadData(); + $cacheKey = 'ElloBridge_key'; + $apiKey = $cache->get($cacheKey); - if ($key == null) { - $keyInfo = getContents(self::URI . 'api/webapp-token') or - returnServerError('Unable to get token.'); - $key = json_decode($keyInfo)->token->access_token; - $cache->saveData($key); + if (!$apiKey) { + $keyInfo = getContents(self::URI . 'api/webapp-token') or returnServerError('Unable to get token.'); + $apiKey = json_decode($keyInfo)->token->access_token; + $ttl = 60 * 60 * 20; + $cache->set($cacheKey, $apiKey, $ttl); } - return $key; + return $apiKey; } public function getName() diff --git a/bridges/InstagramBridge.php b/bridges/InstagramBridge.php index 0f644c4a..9a846fb1 100644 --- a/bridges/InstagramBridge.php +++ b/bridges/InstagramBridge.php @@ -99,23 +99,22 @@ class InstagramBridge extends BridgeAbstract } $cache = RssBridge::getCache(); - $cache->setScope('InstagramBridge'); - $cache->setKey([$username]); - $key = $cache->loadData(); + $cacheKey = 'InstagramBridge_' . $username; + $pk = $cache->get($cacheKey); - if ($key == null) { + if (!$pk) { $data = $this->getContents(self::URI . 'web/search/topsearch/?query=' . $username); foreach (json_decode($data)->users as $user) { if (strtolower($user->user->username) === strtolower($username)) { - $key = $user->user->pk; + $pk = $user->user->pk; } } - if ($key == null) { + if (!$pk) { returnServerError('Unable to find username in search result.'); } - $cache->saveData($key); + $cache->set($cacheKey, $pk); } - return $key; + return $pk; } public function collectData() diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index 855aae08..81401be9 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -100,7 +100,7 @@ class MastodonBridge extends BridgeAbstract // We fetch the boosted content. try { $rtContent = $this->fetchAP($content['object']); - $rtUser = $this->loadCacheValue($rtContent['attributedTo'], 86400); + $rtUser = $this->loadCacheValue($rtContent['attributedTo']); if (!isset($rtUser)) { // We fetch the author, since we cannot always assume the format of the URL. $user = $this->fetchAP($rtContent['attributedTo']); diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index bd60243f..196f7d20 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -72,8 +72,30 @@ class RedditBridge extends BridgeAbstract ] ] ]; + private CacheInterface $cache; + + public function __construct() + { + $this->cache = RssBridge::getCache(); + } public function collectData() + { + $cacheKey = 'reddit_rate_limit'; + if ($this->cache->get($cacheKey)) { + throw new HttpException('429 Too Many Requests', 429); + } + try { + $this->collectDataInternal(); + } catch (HttpException $e) { + if ($e->getCode() === 429) { + $this->cache->set($cacheKey, true, 60 * 16); + throw $e; + } + } + } + + private function collectDataInternal(): void { $user = false; $comments = false; diff --git a/bridges/SoundcloudBridge.php b/bridges/SoundcloudBridge.php index 0bd9a2b0..5664761b 100644 --- a/bridges/SoundcloudBridge.php +++ b/bridges/SoundcloudBridge.php @@ -36,7 +36,7 @@ class SoundCloudBridge extends BridgeAbstract private $feedTitle = null; private $feedIcon = null; - private $cache = null; + private CacheInterface $cache; private $clientIdRegex = '/client_id.*?"(.+?)"/'; private $widgetRegex = '/widget-.+?\.js/'; @@ -44,8 +44,6 @@ class SoundCloudBridge extends BridgeAbstract public function collectData() { $this->cache = RssBridge::getCache(); - $this->cache->setScope('SoundCloudBridge'); - $this->cache->setKey(['client_id']); $res = $this->getUser($this->getInput('u')); @@ -121,11 +119,9 @@ HTML; private function getClientID() { - $this->cache->setScope('SoundCloudBridge'); - $this->cache->setKey(['client_id']); - $clientID = $this->cache->loadData(); + $clientID = $this->cache->get('SoundCloudBridge_client_id'); - if ($clientID == null) { + if (!$clientID) { return $this->refreshClientID(); } else { return $clientID; @@ -151,10 +147,7 @@ HTML; if (preg_match($this->clientIdRegex, $widgetJS, $matches)) { $clientID = $matches[1]; - $this->cache->setScope('SoundCloudBridge'); - $this->cache->setKey(['client_id']); - $this->cache->saveData($clientID); - + $this->cache->set('SoundCloudBridge_client_id', $clientID); return $clientID; } } diff --git a/bridges/SpotifyBridge.php b/bridges/SpotifyBridge.php index 7b7e2b1d..eb847f3d 100644 --- a/bridges/SpotifyBridge.php +++ b/bridges/SpotifyBridge.php @@ -279,10 +279,9 @@ class SpotifyBridge extends BridgeAbstract private function fetchAccessToken() { $cache = RssBridge::getCache(); - $cacheKey = sprintf('%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); - $cache->setScope('SpotifyBridge'); - $cache->setKey([$cacheKey]); - $token = $cache->loadData(3600); + $cacheKey = sprintf('SpotifyBridge:%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); + + $token = $cache->get($cacheKey); if ($token) { $this->token = $token; } else { @@ -294,9 +293,8 @@ class SpotifyBridge extends BridgeAbstract ]); $data = Json::decode($json); $this->token = $data['access_token']; - $cache->setScope('SpotifyBridge'); - $cache->setKey([$cacheKey]); - $cache->saveData($this->token); + + $cache->set($cacheKey, $this->token, 3600); } } diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index 8470dcf7..b9586150 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -594,156 +594,4 @@ EOD; { return (intval($tweet1['id']) < intval($tweet2['id']) ? 1 : -1); } - - //The aim of this function is to get an API key and a guest token - //This function takes 2 requests, and therefore is cached - private function getApiKey($forceNew = 0) - { - $r_cache = RssBridge::getCache(); - $scope = 'TwitterBridge'; - $r_cache->setScope($scope); - $r_cache->setKey(['refresh']); - $data = $r_cache->loadData(); - - $refresh = null; - if ($data === null) { - $refresh = time(); - $r_cache->saveData($refresh); - } else { - $refresh = $data; - } - - $cacheFactory = new CacheFactory(); - - $cache = RssBridge::getCache(); - $cache->setScope($scope); - $cache->setKey(['api_key']); - $data = $cache->loadData(); - - $apiKey = null; - if ($forceNew || $data === null || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY) { - $twitterPage = getContents('https://twitter.com'); - - $jsLink = false; - $jsMainRegexArray = [ - '/(https:\/\/abs\.twimg\.com\/responsive-web\/web\/main\.[^\.]+\.js)/m', - '/(https:\/\/abs\.twimg\.com\/responsive-web\/web_legacy\/main\.[^\.]+\.js)/m', - '/(https:\/\/abs\.twimg\.com\/responsive-web\/client-web\/main\.[^\.]+\.js)/m', - '/(https:\/\/abs\.twimg\.com\/responsive-web\/client-web-legacy\/main\.[^\.]+\.js)/m', - ]; - foreach ($jsMainRegexArray as $jsMainRegex) { - if (preg_match_all($jsMainRegex, $twitterPage, $jsMainMatches, PREG_SET_ORDER, 0)) { - $jsLink = $jsMainMatches[0][0]; - break; - } - } - if (!$jsLink) { - returnServerError('Could not locate main.js link'); - } - - $jsContent = getContents($jsLink); - $apiKeyRegex = '/([a-zA-Z0-9]{59}%[a-zA-Z0-9]{44})/m'; - preg_match_all($apiKeyRegex, $jsContent, $apiKeyMatches, PREG_SET_ORDER, 0); - $apiKey = $apiKeyMatches[0][0]; - $cache->saveData($apiKey); - } else { - $apiKey = $data; - } - - $gt_cache = RssBridge::getCache(); - $gt_cache->setScope($scope); - $gt_cache->setKey(['guest_token']); - $guestTokenUses = $gt_cache->loadData(); - - $guestToken = null; - if ( - $forceNew || $guestTokenUses === null || !is_array($guestTokenUses) || count($guestTokenUses) != 2 - || $guestTokenUses[0] <= 0 || (time() - $refresh) > self::GUEST_TOKEN_EXPIRY - ) { - $guestToken = $this->getGuestToken($apiKey); - if ($guestToken === null) { - if ($guestTokenUses === null) { - returnServerError('Could not parse guest token'); - } else { - $guestToken = $guestTokenUses[1]; - } - } else { - $gt_cache->saveData([self::GUEST_TOKEN_USES, $guestToken]); - $r_cache->saveData(time()); - } - } else { - $guestTokenUses[0] -= 1; - $gt_cache->saveData($guestTokenUses); - $guestToken = $guestTokenUses[1]; - } - - $this->apiKey = $apiKey; - $this->guestToken = $guestToken; - $this->authHeaders = [ - 'authorization: Bearer ' . $apiKey, - 'x-guest-token: ' . $guestToken, - ]; - - return [$apiKey, $guestToken]; - } - - // Get a guest token. This is different to an API key, - // and it seems to change more regularly than the API key. - private function getGuestToken($apiKey) - { - $headers = [ - 'authorization: Bearer ' . $apiKey, - ]; - $opts = [ - CURLOPT_POST => 1, - ]; - - try { - $pageContent = getContents('https://api.twitter.com/1.1/guest/activate.json', $headers, $opts, true); - $guestToken = json_decode($pageContent['content'])->guest_token; - } catch (Exception $e) { - $guestToken = null; - } - return $guestToken; - } - - /** - * Tries to make an API call to twitter. - * @param $api string API entry point - * @param $params array additional URI parmaeters - * @return object json data - */ - private function makeApiCall($api, $params) - { - $uri = self::API_URI . $api . '?' . http_build_query($params); - - $retries = 1; - $retry = 0; - do { - $retry = 0; - - try { - $result = getContents($uri, $this->authHeaders, [], true); - } catch (HttpException $e) { - switch ($e->getCode()) { - case 401: - // fall-through - case 403: - if ($retries) { - $retries--; - $retry = 1; - $this->getApiKey(1); - continue 2; - } - // fall-through - default: - throw $e; - } - } - } while ($retry); - - $data = json_decode($result['content']); - - return $data; - } } diff --git a/bridges/WordPressMadaraBridge.php b/bridges/WordPressMadaraBridge.php index c5ff54b5..4325075c 100644 --- a/bridges/WordPressMadaraBridge.php +++ b/bridges/WordPressMadaraBridge.php @@ -117,7 +117,7 @@ The default URI shows the Madara demo page.'; protected function getMangaInfo($url) { $url_cache = 'TitleInfo_' . preg_replace('/[^\w]/', '.', rtrim($url, '/')); - $cache = $this->loadCacheValue($url_cache, 86400); + $cache = $this->loadCacheValue($url_cache); if (isset($cache)) { return $cache; } diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 54a38d98..8e3ac540 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -77,6 +77,138 @@ class YoutubeBridge extends BridgeAbstract private $channel_name = ''; // This took from repo BetterVideoRss of VerifiedJoseph. const URI_REGEX = '/(https?:\/\/(?:www\.)?(?:[a-zA-Z0-9-.]{2,256}\.[a-z]{2,20})(\:[0-9]{2 ,4})?(?:\/[a-zA-Z0-9@:%_\+.,~#"\'!?&\/\/=\-*]+|\/)?)/ims'; //phpcs:ignore + private CacheInterface $cache; + + public function __construct() + { + $this->cache = RssBridge::getCache(); + } + + private function collectDataInternal() + { + $xml = ''; + $html = ''; + $url_feed = ''; + $url_listing = ''; + + if ($this->getInput('u')) { + /* User and Channel modes */ + $this->request = $this->getInput('u'); + $url_feed = self::URI . 'feeds/videos.xml?user=' . urlencode($this->request); + $url_listing = self::URI . 'user/' . urlencode($this->request) . '/videos'; + } elseif ($this->getInput('c')) { + $this->request = $this->getInput('c'); + $url_feed = self::URI . 'feeds/videos.xml?channel_id=' . urlencode($this->request); + $url_listing = self::URI . 'channel/' . urlencode($this->request) . '/videos'; + } elseif ($this->getInput('custom')) { + $this->request = $this->getInput('custom'); + $url_listing = self::URI . urlencode($this->request) . '/videos'; + } + + if (!empty($url_feed) || !empty($url_listing)) { + $this->feeduri = $url_listing; + if (!empty($this->getInput('custom'))) { + $html = $this->ytGetSimpleHTMLDOM($url_listing); + $jsonData = $this->getJSONData($html); + $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl; + $this->iconURL = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; + } + if (!$this->skipFeeds()) { + $html = $this->ytGetSimpleHTMLDOM($url_feed); + $this->ytBridgeParseXmlFeed($html); + } else { + if (empty($this->getInput('custom'))) { + $html = $this->ytGetSimpleHTMLDOM($url_listing); + $jsonData = $this->getJSONData($html); + } + $channel_id = ''; + if (isset($jsonData->contents)) { + $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId; + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1]; + $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents; + // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; + $this->parseJSONListing($jsonData); + } else { + returnServerError('Unable to get data from YouTube. Username/Channel: ' . $this->request); + } + } + $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); + } elseif ($this->getInput('p')) { + /* playlist mode */ + // TODO: this mode makes a lot of excess video query requests. + // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration" + // This cache will be used to find out, which videos to fetch + // to make feed of 15 items or more, if there a lot of videos published on that date. + $this->request = $this->getInput('p'); + $url_feed = self::URI . 'feeds/videos.xml?playlist_id=' . urlencode($this->request); + $url_listing = self::URI . 'playlist?list=' . urlencode($this->request); + $html = $this->ytGetSimpleHTMLDOM($url_listing); + $jsonData = $this->getJSONData($html); + // TODO: this method returns only first 100 video items + // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0]; + $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; + $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; + $item_count = count($jsonData); + + if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) { + $this->ytBridgeParseXmlFeed($xml); + } else { + $this->parseJSONListing($jsonData); + } + $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); + usort($this->items, function ($item1, $item2) { + if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) { + $item1['timestamp'] = strtotime($item1['timestamp']); + $item2['timestamp'] = strtotime($item2['timestamp']); + } + return $item2['timestamp'] - $item1['timestamp']; + }); + } elseif ($this->getInput('s')) { + /* search mode */ + $this->request = $this->getInput('s'); + $url_listing = self::URI + . 'results?search_query=' + . urlencode($this->request) + . '&sp=CAI%253D'; + + $html = $this->ytGetSimpleHTMLDOM($url_listing); + + $jsonData = $this->getJSONData($html); + $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; + $jsonData = $jsonData->sectionListRenderer->contents; + foreach ($jsonData as $data) { + // Search result includes some ads, have to filter them + if (isset($data->itemSectionRenderer->contents[0]->videoRenderer)) { + $jsonData = $data->itemSectionRenderer->contents; + break; + } + } + $this->parseJSONListing($jsonData); + $this->feeduri = $url_listing; + $this->feedName = 'Search: ' . $this->request; + } else { + /* no valid mode */ + returnClientError("You must either specify either:\n - YouTube + username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); + } + } + + public function collectData() + { + $cacheKey = 'youtube_rate_limit'; + if ($this->cache->get($cacheKey)) { + throw new HttpException('429 Too Many Requests', 429); + } + try { + $this->collectDataInternal(); + } catch (HttpException $e) { + if ($e->getCode() === 429) { + $this->cache->set($cacheKey, true, 60 * 16); + throw $e; + } + } + } private function ytBridgeQueryVideoInfo($vid, &$author, &$desc, &$time) { @@ -153,7 +285,8 @@ class YoutubeBridge extends BridgeAbstract $item['timestamp'] = $time; $item['uri'] = self::URI . 'watch?v=' . $vid; if (!$thumbnail) { - $thumbnail = '0'; // Fallback to default thumbnail if there aren't any provided. + // Fallback to default thumbnail if there aren't any provided. + $thumbnail = '0'; } $thumbnailUri = str_replace('/www.', '/img.', self::URI) . 'vi/' . $vid . '/' . $thumbnail . '.jpg'; $item['content'] = '<a href="' . $item['uri'] . '"><img src="' . $thumbnailUri . '" /></a><br />' . $desc; @@ -315,111 +448,6 @@ class YoutubeBridge extends BridgeAbstract } } - public function collectData() - { - $xml = ''; - $html = ''; - $url_feed = ''; - $url_listing = ''; - - if ($this->getInput('u')) { /* User and Channel modes */ - $this->request = $this->getInput('u'); - $url_feed = self::URI . 'feeds/videos.xml?user=' . urlencode($this->request); - $url_listing = self::URI . 'user/' . urlencode($this->request) . '/videos'; - } elseif ($this->getInput('c')) { - $this->request = $this->getInput('c'); - $url_feed = self::URI . 'feeds/videos.xml?channel_id=' . urlencode($this->request); - $url_listing = self::URI . 'channel/' . urlencode($this->request) . '/videos'; - } elseif ($this->getInput('custom')) { - $this->request = $this->getInput('custom'); - $url_listing = self::URI . urlencode($this->request) . '/videos'; - } - - if (!empty($url_feed) || !empty($url_listing)) { - $this->feeduri = $url_listing; - if (!empty($this->getInput('custom'))) { - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl; - $this->iconURL = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; - } - if (!$this->skipFeeds()) { - $html = $this->ytGetSimpleHTMLDOM($url_feed); - $this->ytBridgeParseXmlFeed($html); - } else { - if (empty($this->getInput('custom'))) { - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - } - $channel_id = ''; - if (isset($jsonData->contents)) { - $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId; - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1]; - $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents; - // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; - $this->parseJSONListing($jsonData); - } else { - returnServerError('Unable to get data from YouTube. Username/Channel: ' . $this->request); - } - } - $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); - } elseif ($this->getInput('p')) { /* playlist mode */ - // TODO: this mode makes a lot of excess video query requests. - // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration" - // This cache will be used to find out, which videos to fetch - // to make feed of 15 items or more, if there a lot of videos published on that date. - $this->request = $this->getInput('p'); - $url_feed = self::URI . 'feeds/videos.xml?playlist_id=' . urlencode($this->request); - $url_listing = self::URI . 'playlist?list=' . urlencode($this->request); - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - // TODO: this method returns only first 100 video items - // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0]; - $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; - $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; - $item_count = count($jsonData); - - if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) { - $this->ytBridgeParseXmlFeed($xml); - } else { - $this->parseJSONListing($jsonData); - } - $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); // feedName will be used by getName() - usort($this->items, function ($item1, $item2) { - if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) { - $item1['timestamp'] = strtotime($item1['timestamp']); - $item2['timestamp'] = strtotime($item2['timestamp']); - } - return $item2['timestamp'] - $item1['timestamp']; - }); - } elseif ($this->getInput('s')) { /* search mode */ - $this->request = $this->getInput('s'); - $url_listing = self::URI - . 'results?search_query=' - . urlencode($this->request) - . '&sp=CAI%253D'; - - $html = $this->ytGetSimpleHTMLDOM($url_listing); - - $jsonData = $this->getJSONData($html); - $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; - $jsonData = $jsonData->sectionListRenderer->contents; - foreach ($jsonData as $data) { // Search result includes some ads, have to filter them - if (isset($data->itemSectionRenderer->contents[0]->videoRenderer)) { - $jsonData = $data->itemSectionRenderer->contents; - break; - } - } - $this->parseJSONListing($jsonData); - $this->feeduri = $url_listing; - $this->feedName = 'Search: ' . $this->request; // feedName will be used by getName() - } else { /* no valid mode */ - returnClientError("You must either specify either:\n - YouTube - username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); - } - } - private function skipFeeds() { return ($this->getInput('duration_min') || $this->getInput('duration_max')); @@ -438,14 +466,13 @@ class YoutubeBridge extends BridgeAbstract public function getName() { - // Name depends on queriedContext: switch ($this->queriedContext) { case 'By username': case 'By channel id': case 'By custom name': case 'By playlist Id': case 'Search result': - return htmlspecialchars_decode($this->feedName) . ' - YouTube'; // We already know it's a bridge, right? + return htmlspecialchars_decode($this->feedName) . ' - YouTube'; default: return parent::getName(); } diff --git a/caches/ArrayCache.php b/caches/ArrayCache.php new file mode 100644 index 00000000..efce4f35 --- /dev/null +++ b/caches/ArrayCache.php @@ -0,0 +1,52 @@ +<?php + +declare(strict_types=1); + +class ArrayCache implements CacheInterface +{ + private array $data = []; + + public function get(string $key, $default = null) + { + $item = $this->data[$key] ?? null; + if (!$item) { + return $default; + } + $expiration = $item['expiration']; + if ($expiration === 0 || $expiration > time()) { + return $item['value']; + } + $this->delete($key); + return $default; + } + + public function set(string $key, $value, int $ttl = null): void + { + $this->data[$key] = [ + 'key' => $key, + 'value' => $value, + 'expiration' => $ttl === null ? 0 : time() + $ttl, + ]; + } + + public function delete(string $key): void + { + unset($this->data[$key]); + } + + public function clear(): void + { + $this->data = []; + } + + public function prune(): void + { + foreach ($this->data as $key => $item) { + $expiration = $item['expiration']; + if ($expiration === 0 || $expiration > time()) { + continue; + } + $this->delete($key); + } + } +} diff --git a/caches/FileCache.php b/caches/FileCache.php index 6e150cb4..1495971a 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -1,13 +1,10 @@ <?php -/** - * @link https://www.php.net/manual/en/function.clearstatcache.php - */ +declare(strict_types=1); + class FileCache implements CacheInterface { private array $config; - protected string $scope; - protected string $key; public function __construct(array $config = []) { @@ -23,125 +20,89 @@ class FileCache implements CacheInterface $this->config['path'] = rtrim($this->config['path'], '/') . '/'; } - public function getConfig() + public function get(string $key, $default = null) { - return $this->config; + $cacheFile = $this->createCacheFile($key); + if (!file_exists($cacheFile)) { + return $default; + } + $item = unserialize(file_get_contents($cacheFile)); + if ($item === false) { + Logger::warning(sprintf('Failed to unserialize: %s', $cacheFile)); + $this->delete($key); + return $default; + } + $expiration = $item['expiration']; + if ($expiration === 0 || $expiration > time()) { + return $item['value']; + } + $this->delete($key); + return $default; } - public function loadData(int $timeout = 86400) + public function set($key, $value, int $ttl = null): void { - clearstatcache(); - if (!file_exists($this->getCacheFile())) { - return null; - } - $modificationTime = filemtime($this->getCacheFile()); - if (time() - $timeout < $modificationTime) { - $data = unserialize(file_get_contents($this->getCacheFile())); - if ($data === false) { - Logger::warning(sprintf('Failed to unserialize: %s', $this->getCacheFile())); - // Intentionally not throwing an exception - return null; - } - return $data; - } - // It's a good idea to delete the expired item here, but commented out atm - // unlink($this->getCacheFile()); - return null; - } - - public function saveData($data): void - { - $bytes = file_put_contents($this->getCacheFile(), serialize($data), LOCK_EX); + $item = [ + 'key' => $key, + 'value' => $value, + 'expiration' => $ttl === null ? 0 : time() + $ttl, + ]; + $cacheFile = $this->createCacheFile($key); + $bytes = file_put_contents($cacheFile, serialize($item), LOCK_EX); if ($bytes === false) { - throw new \Exception(sprintf('Failed to write to: %s', $this->getCacheFile())); + // Consider just logging the error here + throw new \Exception(sprintf('Failed to write to: %s', $cacheFile)); } } - public function getTime(): ?int + public function delete(string $key): void { - clearstatcache(); - $cacheFile = $this->getCacheFile(); - if (file_exists($cacheFile)) { - $time = filemtime($cacheFile); - if ($time !== false) { - return $time; - } - return null; - } - - return null; + unlink($this->createCacheFile($key)); } - public function purgeCache(int $timeout = 86400): void + public function clear(): void + { + foreach (scandir($this->config['path']) as $filename) { + $cacheFile = $this->config['path'] . $filename; + $excluded = ['.' => true, '..' => true, '.gitkeep' => true]; + if (isset($excluded[$filename]) || !is_file($cacheFile)) { + continue; + } + unlink($cacheFile); + } + } + + public function prune(): void { if (! $this->config['enable_purge']) { return; } - - $cachePath = $this->getScope(); - if (!file_exists($cachePath)) { - return; - } - $cacheIterator = new \RecursiveIteratorIterator( - new \RecursiveDirectoryIterator($cachePath), - \RecursiveIteratorIterator::CHILD_FIRST - ); - - foreach ($cacheIterator as $cacheFile) { - $basename = $cacheFile->getBasename(); - $excluded = [ - '.' => true, - '..' => true, - '.gitkeep' => true, - ]; - if (isset($excluded[$basename])) { + foreach (scandir($this->config['path']) as $filename) { + $cacheFile = $this->config['path'] . $filename; + $excluded = ['.' => true, '..' => true, '.gitkeep' => true]; + if (isset($excluded[$filename]) || !is_file($cacheFile)) { continue; - } elseif ($cacheFile->isFile()) { - $filepath = $cacheFile->getPathname(); - if (filemtime($filepath) < time() - $timeout) { - // todo: sometimes this file doesn't exists - unlink($filepath); - } } - } - } - - public function setScope(string $scope): void - { - $this->scope = $this->config['path'] . trim($scope, " \t\n\r\0\x0B\\\/") . '/'; - } - - public function setKey(array $key): void - { - $this->key = json_encode($key); - } - - private function getScope() - { - if (is_null($this->scope)) { - throw new \Exception('Call "setScope" first!'); - } - - if (!is_dir($this->scope)) { - if (mkdir($this->scope, 0755, true) !== true) { - throw new \Exception('mkdir: Unable to create file cache folder'); + $item = unserialize(file_get_contents($cacheFile)); + if ($item === false) { + unlink($cacheFile); + continue; } + $expiration = $item['expiration']; + if ($expiration === 0 || $expiration > time()) { + continue; + } + unlink($cacheFile); } - - return $this->scope; } - private function getCacheFile() + private function createCacheFile(string $key): string { - return $this->getScope() . $this->getCacheName(); + return $this->config['path'] . hash('md5', $key) . '.cache'; } - private function getCacheName() + public function getConfig() { - if (is_null($this->key)) { - throw new \Exception('Call "setKey" first!'); - } - - return hash('md5', $this->key) . '.cache'; + return $this->config; } } diff --git a/caches/MemcachedCache.php b/caches/MemcachedCache.php index dcb572c7..78035435 100644 --- a/caches/MemcachedCache.php +++ b/caches/MemcachedCache.php @@ -1,70 +1,36 @@ <?php +declare(strict_types=1); + class MemcachedCache implements CacheInterface { - private string $scope; - private string $key; - private $conn; - private $expiration = 0; + private \Memcached $conn; - public function __construct() + public function __construct(string $host, int $port) { - if (!extension_loaded('memcached')) { - throw new \Exception('"memcached" extension not loaded. Please check "php.ini"'); + $this->conn = new \Memcached(); + // This call does not actually connect to server yet + if (!$this->conn->addServer($host, $port)) { + throw new \Exception('Unable to add memcached server'); } - - $section = 'MemcachedCache'; - $host = Configuration::getConfig($section, 'host'); - $port = Configuration::getConfig($section, 'port'); - - if (empty($host) && empty($port)) { - throw new \Exception('Configuration for ' . $section . ' missing.'); - } - if (empty($host)) { - throw new \Exception('"host" param is not set for ' . $section); - } - if (empty($port)) { - throw new \Exception('"port" param is not set for ' . $section); - } - if (!ctype_digit($port)) { - throw new \Exception('"port" param is invalid for ' . $section); - } - - $port = intval($port); - - if ($port < 1 || $port > 65535) { - throw new \Exception('"port" param is invalid for ' . $section); - } - - $conn = new \Memcached(); - $conn->addServer($host, $port) or returnServerError('Could not connect to memcached server'); - $this->conn = $conn; } - public function loadData(int $timeout = 86400) + public function get(string $key, $default = null) { - $value = $this->conn->get($this->getCacheKey()); + $value = $this->conn->get($key); if ($value === false) { - return null; + return $default; } - if (time() - $timeout < $value['time']) { - return $value['data']; - } - return null; + return $value; } - public function saveData($data): void + public function set(string $key, $value, $ttl = null): void { - $value = [ - 'data' => $data, - 'time' => time(), - ]; - $result = $this->conn->set($this->getCacheKey(), $value, $this->expiration); + $expiration = $ttl === null ? 0 : time() + $ttl; + $result = $this->conn->set($key, $value, $expiration); if ($result === false) { Logger::warning('Failed to store an item in memcached', [ - 'scope' => $this->scope, - 'key' => $this->key, - 'expiration' => $this->expiration, + 'key' => $key, 'code' => $this->conn->getLastErrorCode(), 'message' => $this->conn->getLastErrorMessage(), 'number' => $this->conn->getLastErrorErrno(), @@ -73,38 +39,18 @@ class MemcachedCache implements CacheInterface } } - public function getTime(): ?int + public function delete(string $key): void { - $value = $this->conn->get($this->getCacheKey()); - if ($value === false) { - return null; - } - return $value['time']; + $this->conn->delete($key); } - public function purgeCache(int $timeout = 86400): void + public function clear(): void { - // Note: does not purges cache right now - // Just sets cache expiration and leave cache purging for memcached itself - $this->expiration = $timeout; + $this->conn->flush(); } - public function setScope(string $scope): void + public function prune(): void { - $this->scope = $scope; - } - - public function setKey(array $key): void - { - $this->key = json_encode($key); - } - - private function getCacheKey() - { - if (is_null($this->key)) { - throw new \Exception('Call "setKey" first!'); - } - - return 'rss_bridge_cache_' . hash('md5', $this->scope . $this->key . 'A'); + // memcached manages pruning on its own } } diff --git a/caches/NullCache.php b/caches/NullCache.php index fe43fe06..2549b117 100644 --- a/caches/NullCache.php +++ b/caches/NullCache.php @@ -4,28 +4,24 @@ declare(strict_types=1); class NullCache implements CacheInterface { - public function setScope(string $scope): void + public function get(string $key, $default = null) + { + return $default; + } + + public function set(string $key, $value, int $ttl = null): void { } - public function setKey(array $key): void + public function delete(string $key): void { } - public function loadData(int $timeout = 86400) + public function clear(): void { } - public function saveData($data): void - { - } - - public function getTime(): ?int - { - return null; - } - - public function purgeCache(int $timeout = 86400): void + public function prune(): void { } } diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index 92235862..beb33e88 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -1,10 +1,10 @@ <?php +declare(strict_types=1); + class SQLiteCache implements CacheInterface { private \SQLite3 $db; - private string $scope; - private string $key; private array $config; public function __construct(array $config) @@ -31,85 +31,77 @@ class SQLiteCache implements CacheInterface $this->db->exec("CREATE TABLE storage ('key' BLOB PRIMARY KEY, 'value' BLOB, 'updated' INTEGER)"); } $this->db->busyTimeout($config['timeout']); + // https://www.sqlite.org/pragma.html#pragma_journal_mode + $this->db->exec('PRAGMA journal_mode = wal'); + // https://www.sqlite.org/pragma.html#pragma_synchronous + $this->db->exec('PRAGMA synchronous = NORMAL'); } - public function loadData(int $timeout = 86400) + public function get(string $key, $default = null) { + $cacheKey = $this->createCacheKey($key); $stmt = $this->db->prepare('SELECT value, updated FROM storage WHERE key = :key'); - $stmt->bindValue(':key', $this->getCacheKey()); + $stmt->bindValue(':key', $cacheKey); $result = $stmt->execute(); if (!$result) { - return null; + return $default; } $row = $result->fetchArray(\SQLITE3_ASSOC); if ($row === false) { - return null; + return $default; } - $value = $row['value']; - $modificationTime = $row['updated']; - if (time() - $timeout < $modificationTime) { - $data = unserialize($value); - if ($data === false) { - Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($value, 0, 100))); - return null; + $expiration = $row['updated']; + if ($expiration === 0 || $expiration > time()) { + $blob = $row['value']; + $value = unserialize($blob); + if ($value === false) { + Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100))); + // delete? + return $default; } - return $data; + return $value; } - // It's a good idea to delete expired cache items. - // However I'm seeing lots of SQLITE_BUSY errors so commented out for now - // $stmt = $this->db->prepare('DELETE FROM storage WHERE key = :key'); - // $stmt->bindValue(':key', $this->getCacheKey()); - // $stmt->execute(); - return null; + // delete? + return $default; } - - public function saveData($data): void + public function set(string $key, $value, int $ttl = null): void { - $blob = serialize($data); - + $cacheKey = $this->createCacheKey($key); + $blob = serialize($value); + $expiration = $ttl === null ? 0 : time() + $ttl; $stmt = $this->db->prepare('INSERT OR REPLACE INTO storage (key, value, updated) VALUES (:key, :value, :updated)'); - $stmt->bindValue(':key', $this->getCacheKey()); + $stmt->bindValue(':key', $cacheKey); $stmt->bindValue(':value', $blob, \SQLITE3_BLOB); - $stmt->bindValue(':updated', time()); - $stmt->execute(); - } - - public function getTime(): ?int - { - $stmt = $this->db->prepare('SELECT updated FROM storage WHERE key = :key'); - $stmt->bindValue(':key', $this->getCacheKey()); + $stmt->bindValue(':updated', $expiration); $result = $stmt->execute(); - if ($result) { - $row = $result->fetchArray(\SQLITE3_ASSOC); - if ($row !== false) { - return $row['updated']; - } - } - return null; + // Unclear whether we should $result->finalize(); here? } - public function purgeCache(int $timeout = 86400): void + public function delete(string $key): void + { + $key = $this->createCacheKey($key); + $stmt = $this->db->prepare('DELETE FROM storage WHERE key = :key'); + $stmt->bindValue(':key', $key); + $result = $stmt->execute(); + } + + public function prune(): void { if (!$this->config['enable_purge']) { return; } - $stmt = $this->db->prepare('DELETE FROM storage WHERE updated < :expired'); - $stmt->bindValue(':expired', time() - $timeout); - $stmt->execute(); + $stmt = $this->db->prepare('DELETE FROM storage WHERE updated <= :now'); + $stmt->bindValue(':now', time()); + $result = $stmt->execute(); } - public function setScope(string $scope): void + public function clear(): void { - $this->scope = $scope; + $this->db->query('DELETE FROM storage'); } - public function setKey(array $key): void + private function createCacheKey($key) { - $this->key = json_encode($key); - } - - private function getCacheKey() - { - return hash('sha1', $this->scope . $this->key, true); + return hash('sha1', $key, true); } } diff --git a/config.default.ini.php b/config.default.ini.php index d0c508f4..52786aef 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -55,7 +55,7 @@ max_filesize = 20 [cache] -; Cache type: file, sqlite, memcached, null +; Cache type: file, sqlite, memcached, array, null type = "file" ; Allow users to specify custom timeout for specific requests. diff --git a/contrib/prepare_release/fetch_contributors.php b/contrib/prepare_release/fetch_contributors.php index cfe2c5b2..dd99229f 100644 --- a/contrib/prepare_release/fetch_contributors.php +++ b/contrib/prepare_release/fetch_contributors.php @@ -15,14 +15,17 @@ while ($next) { /* Collect all contributors */ 'User-Agent' => 'RSS-Bridge', ]; $httpClient = new CurlHttpClient(); - $result = $httpClient->request($url, ['headers' => $headers]); + $response = $httpClient->request($url, ['headers' => $headers]); - foreach (json_decode($result['body']) as $contributor) { + $json = $response->getBody(); + $json_decode = Json::decode($json, false); + foreach ($json_decode as $contributor) { $contributors[] = $contributor; } // Extract links to "next", "last", etc... - $links = explode(',', $result['headers']['link'][0]); + $link1 = $response->getHeader('link'); + $links = explode(',', $link1); $next = false; // Check if there is a link with 'rel="next"' diff --git a/docs/06_Helper_functions/index.md b/docs/06_Helper_functions/index.md index 2f0c513c..31a13953 100644 --- a/docs/06_Helper_functions/index.md +++ b/docs/06_Helper_functions/index.md @@ -5,10 +5,12 @@ The `getInput` function is used to receive a value for a parameter, specified in $this->getInput('your input name here'); ``` -`getInput` will either return the value for your parameter or `null` if the parameter is unknown or not specified. +`getInput` will either return the value for your parameter +or `null` if the parameter is unknown or not specified. # getKey -The `getKey` function is used to receive the key name to a selected list value given the name of the list, specified in `const PARAMETERS` +The `getKey` function is used to receive the key name to a selected list +value given the name of the list, specified in `const PARAMETERS` Is able to work with multidimensional list arrays. ```PHP @@ -34,7 +36,8 @@ $this->getKey('country'); // if the selected value was "ve", this function will return "Venezuela" ``` -`getKey` will either return the key name for your parameter or `null` if the parameter is unknown or not specified. +`getKey` will either return the key name for your parameter or `null` if the parameter +is unknown or not specified. # getContents The `getContents` function uses [cURL](https://secure.php.net/manual/en/book.curl.php) to acquire data from the specified URI while respecting the various settings defined at a global level by RSS-Bridge (i.e., proxy host, user agent, etc.). This function accepts a few parameters: @@ -53,33 +56,29 @@ $html = getContents($url, $header, $opts); ``` # getSimpleHTMLDOM -The `getSimpleHTMLDOM` function is a wrapper for the [simple_html_dom](https://simplehtmldom.sourceforge.io/) [file_get_html](https://simplehtmldom.sourceforge.io/docs/1.9/api/file_get_html/) function in order to provide context by design. +The `getSimpleHTMLDOM` function is a wrapper for the +[simple_html_dom](https://simplehtmldom.sourceforge.io/) [file_get_html](https://simplehtmldom.sourceforge.io/docs/1.9/api/file_get_html/) function in order to provide context by design. ```PHP $html = getSimpleHTMLDOM('your URI'); ``` # getSimpleHTMLDOMCached -The `getSimpleHTMLDOMCached` function does the same as the [`getSimpleHTMLDOM`](#getsimplehtmldom) function, except that the content received for the given URI is stored in a cache and loaded from cache on the next request if the specified cache duration was not reached. Use this function for data that is very unlikely to change between consecutive requests to **RSS-Bridge**. This function allows to specify the cache duration with the second parameter (default is 24 hours / 86400 seconds). +The `getSimpleHTMLDOMCached` function does the same as the +[`getSimpleHTMLDOM`](#getsimplehtmldom) function, +except that the content received for the given URI is stored in a cache +and loaded from cache on the next request if the specified cache duration +was not reached. + +Use this function for data that is very unlikely to change between consecutive requests to **RSS-Bridge**. +This function allows to specify the cache duration with the second parameter. ```PHP $html = getSimpleHTMLDOMCached('your URI', 86400); // Duration 24h ``` -**Notice:** Due to the current implementation a value greater than 86400 seconds (24 hours) will not work as the cache is purged every 24 hours automatically. - -# returnError -**Notice:** Whenever possible make use of [`returnClientError`](#returnclienterror) or [`returnServerError`](#returnservererror) - -The `returnError` function aborts execution of the current bridge and returns the given error message with the provided error number: - -```PHP -returnError('Your error message', 404); -``` - -Check the [list of error codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) for applicable error numbers. - # returnClientError -The `returnClientError` function aborts execution of the current bridge and returns the given error message with error code **400**: +The `returnClientError` function aborts execution of the current bridge +and returns the given error message with error code **400**: ```PHP returnClientError('Your error message') @@ -94,10 +93,12 @@ The `returnServerError` function aborts execution of the current bridge and retu returnServerError('Your error message') ``` -Use this function when a problem occurs that has nothing to do with the parameters provided by the user. (like: Host service gone missing, empty data received, etc...) +Use this function when a problem occurs that has nothing to do with the parameters provided by the user. +(like: Host service gone missing, empty data received, etc...) # defaultLinkTo -Automatically replaces any relative URL in a given string or DOM object (i.e. the one returned by [getSimpleHTMLDOM](#getsimplehtmldom)) with an absolute URL. +Automatically replaces any relative URL in a given string or DOM object +(i.e. the one returned by [getSimpleHTMLDOM](#getsimplehtmldom)) with an absolute URL. ```php defaultLinkTo ( mixed $content, string $server ) : object diff --git a/docs/07_Cache_API/02_CacheInterface.md b/docs/07_Cache_API/02_CacheInterface.md index 61127a0d..3e71237d 100644 --- a/docs/07_Cache_API/02_CacheInterface.md +++ b/docs/07_Cache_API/02_CacheInterface.md @@ -3,16 +3,14 @@ See `CacheInterface`. ```php interface CacheInterface { - public function setScope(string $scope): void; + public function get(string $key, $default = null); - public function setKey(array $key): void; + public function set(string $key, $value, int $ttl = null): void; - public function loadData(); + public function delete(string $key): void; - public function saveData($data): void; + public function clear(): void; - public function getTime(): ?int; - - public function purgeCache(int $seconds): void; + public function prune(): void; } -``` \ No newline at end of file +``` diff --git a/docs/index.md b/docs/index.md index 71fa9f37..c370cb1b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,8 @@ -**RSS-Bridge** is free and open source software for generating Atom or RSS feeds from websites which don't have one. It is written in PHP and intended to run on a Web server. See the [Screenshots](01_General/04_Screenshots.md) for a quick introduction to **RSS-Bridge** +RSS-Bridge is a web application. + +It generates web feeds for websites that don't have one. + +Officially hosted instance: https://rss-bridge.org/bridge01/ - You want to know more about **RSS-Bridge**? Check out our **[project goals](01_General/01_Project-goals.md)**. diff --git a/index.php b/index.php index 538f1c6e..9181c0b0 100644 --- a/index.php +++ b/index.php @@ -1,5 +1,9 @@ <?php +if (version_compare(\PHP_VERSION, '7.4.0') === -1) { + exit('RSS-Bridge requires minimum PHP version 7.4.0!'); +} + require_once __DIR__ . '/lib/bootstrap.php'; $rssBridge = new RssBridge(); diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 2a6cd8ab..36a77669 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -116,6 +116,10 @@ abstract class BridgeAbstract implements BridgeInterface */ private array $configuration = []; + public function __construct() + { + } + /** {@inheritdoc} */ public function getItems() { @@ -410,15 +414,13 @@ abstract class BridgeAbstract implements BridgeInterface /** * Loads a cached value for the specified key * - * @param int $timeout Cache duration (optional) * @return mixed Cached value or null if the key doesn't exist or has expired */ - protected function loadCacheValue(string $key, int $timeout = 86400) + protected function loadCacheValue(string $key) { $cache = RssBridge::getCache(); - $cache->setScope($this->getShortName()); - $cache->setKey([$key]); - return $cache->loadData($timeout); + $cacheKey = $this->getShortName() . '_' . $key; + return $cache->get($cacheKey); } /** @@ -426,12 +428,11 @@ abstract class BridgeAbstract implements BridgeInterface * * @param mixed $value Value to cache */ - protected function saveCacheValue(string $key, $value) + protected function saveCacheValue(string $key, $value, $ttl = 86400) { $cache = RssBridge::getCache(); - $cache->setScope($this->getShortName()); - $cache->setKey([$key]); - $cache->saveData($value); + $cacheKey = $this->getShortName() . '_' . $key; + $cache->set($cacheKey, $value, $ttl); } public function getShortName(): string diff --git a/lib/BridgeInterface.php b/lib/BridgeInterface.php index 977ad7f6..63bc7b70 100644 --- a/lib/BridgeInterface.php +++ b/lib/BridgeInterface.php @@ -57,6 +57,8 @@ interface BridgeInterface { /** * Collects data from the site + * + * @return void */ public function collectData(); diff --git a/lib/CacheFactory.php b/lib/CacheFactory.php index 78a0e83e..3f076d83 100644 --- a/lib/CacheFactory.php +++ b/lib/CacheFactory.php @@ -72,7 +72,29 @@ class CacheFactory 'enable_purge' => Configuration::getConfig('SQLiteCache', 'enable_purge'), ]); case MemcachedCache::class: - return new MemcachedCache(); + if (!extension_loaded('memcached')) { + throw new \Exception('"memcached" extension not loaded. Please check "php.ini"'); + } + $section = 'MemcachedCache'; + $host = Configuration::getConfig($section, 'host'); + $port = Configuration::getConfig($section, 'port'); + if (empty($host) && empty($port)) { + throw new \Exception('Configuration for ' . $section . ' missing.'); + } + if (empty($host)) { + throw new \Exception('"host" param is not set for ' . $section); + } + if (empty($port)) { + throw new \Exception('"port" param is not set for ' . $section); + } + if (!ctype_digit($port)) { + throw new \Exception('"port" param is invalid for ' . $section); + } + $port = intval($port); + if ($port < 1 || $port > 65535) { + throw new \Exception('"port" param is invalid for ' . $section); + } + return new MemcachedCache($host, $port); default: if (!file_exists(PATH_LIB_CACHES . $className . '.php')) { throw new \Exception('Unable to find the cache file'); diff --git a/lib/CacheInterface.php b/lib/CacheInterface.php index 85aa830f..0009a55c 100644 --- a/lib/CacheInterface.php +++ b/lib/CacheInterface.php @@ -2,15 +2,13 @@ interface CacheInterface { - public function setScope(string $scope): void; + public function get(string $key, $default = null); - public function setKey(array $key): void; + public function set(string $key, $value, int $ttl = null): void; - public function loadData(int $timeout = 86400); + public function delete(string $key): void; - public function saveData($data): void; + public function clear(): void; - public function getTime(): ?int; - - public function purgeCache(int $timeout = 86400): void; + public function prune(): void; } diff --git a/lib/Configuration.php b/lib/Configuration.php index f5615009..7ef97fa7 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -37,10 +37,6 @@ final class Configuration */ public static function verifyInstallation() { - if (version_compare(\PHP_VERSION, '7.4.0') === -1) { - throw new \Exception('RSS-Bridge requires at least PHP version 7.4.0!'); - } - $errors = []; // OpenSSL: https://www.php.net/manual/en/book.openssl.php @@ -211,6 +207,9 @@ final class Configuration if (!is_string(self::getConfig('error', 'output'))) { self::throwConfigError('error', 'output', 'Is not a valid String'); } + if (!in_array(self::getConfig('error', 'output'), ['feed', 'http', 'none'])) { + self::throwConfigError('error', 'output', 'Invalid output'); + } if ( !is_numeric(self::getConfig('error', 'report_limit')) diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index c91586d7..be467336 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -100,8 +100,8 @@ abstract class FeedExpander extends BridgeAbstract '*/*', ]; $httpHeaders = ['Accept: ' . implode(', ', $mimeTypes)]; - $content = getContents($url, $httpHeaders); - if ($content === '') { + $xml = getContents($url, $httpHeaders); + if ($xml === '') { throw new \Exception(sprintf('Unable to parse xml from `%s` because we got the empty string', $url), 10); } // Maybe move this call earlier up the stack frames @@ -109,7 +109,7 @@ abstract class FeedExpander extends BridgeAbstract libxml_use_internal_errors(true); // Consider replacing libxml with https://www.php.net/domdocument // Intentionally not using the silencing operator (@) because it has no effect here - $rssContent = simplexml_load_string(trim($content)); + $rssContent = simplexml_load_string(trim($xml)); if ($rssContent === false) { $xmlErrors = libxml_get_errors(); foreach ($xmlErrors as $xmlError) { diff --git a/lib/FormatInterface.php b/lib/FormatInterface.php index c0355804..49e36933 100644 --- a/lib/FormatInterface.php +++ b/lib/FormatInterface.php @@ -28,15 +28,7 @@ interface FormatInterface */ public function stringify(); - /** - * Set items - * - * @param array $bridges The items - * @return self The format object - * - * @todo Rename parameter `$bridges` to `$items` - */ - public function setItems(array $bridges); + public function setItems(array $items); /** * Return items diff --git a/lib/Logger.php b/lib/Logger.php index 5423f62c..073fedee 100644 --- a/lib/Logger.php +++ b/lib/Logger.php @@ -66,13 +66,24 @@ final class Logger } } } - // Intentionally not sanitizing $message + + if ($context) { + try { + $context = Json::encode($context); + } catch (\JsonException $e) { + $context['message'] = null; + $context = Json::encode($context); + } + } else { + $context = ''; + } $text = sprintf( "[%s] rssbridge.%s %s %s\n", now()->format('Y-m-d H:i:s'), $level, + // Intentionally not sanitizing $message $message, - $context ? Json::encode($context) : '' + $context ); // Log to stderr/stdout whatever that is @@ -81,6 +92,6 @@ final class Logger // Log to file // todo: extract to log handler - // file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); + //$bytes = file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); } } diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 8969dc54..1c6ce464 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -5,25 +5,7 @@ final class RssBridge private static HttpClient $httpClient; private static CacheInterface $cache; - public function main(array $argv = []) - { - if ($argv) { - parse_str(implode('&', array_slice($argv, 1)), $cliArgs); - $request = $cliArgs; - } else { - $request = array_merge($_GET, $_POST); - } - - try { - $this->run($request); - } catch (\Throwable $e) { - Logger::error(sprintf('Exception in RssBridge::main(): %s', create_sane_exception_message($e)), ['e' => $e]); - http_response_code(500); - print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); - } - } - - private function run($request): void + public function __construct() { Configuration::verifyInstallation(); @@ -33,6 +15,13 @@ final class RssBridge } Configuration::loadConfiguration($customConfig, getenv()); + set_exception_handler(function (\Throwable $e) { + Logger::error('Uncaught Exception', ['e' => $e]); + http_response_code(500); + print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); + exit(1); + }); + set_error_handler(function ($code, $message, $file, $line) { if ((error_reporting() & $code) === 0) { return false; @@ -45,7 +34,6 @@ final class RssBridge ); Logger::warning($text); if (Debug::isEnabled()) { - // todo: extract to log handler print sprintf("<pre>%s</pre>\n", e($text)); } }); @@ -72,38 +60,58 @@ final class RssBridge // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); date_default_timezone_set(Configuration::getConfig('system', 'timezone')); - $cacheFactory = new CacheFactory(); - self::$httpClient = new CurlHttpClient(); - self::$cache = $cacheFactory->create(); + + $cacheFactory = new CacheFactory(); + if (Debug::isEnabled()) { + self::$cache = $cacheFactory->create('array'); + } else { + self::$cache = $cacheFactory->create(); + } if (Configuration::getConfig('authentication', 'enable')) { $authenticationMiddleware = new AuthenticationMiddleware(); $authenticationMiddleware(); } + } - foreach ($request as $key => $value) { - if (!is_string($value)) { - throw new \Exception("Query parameter \"$key\" is not a string."); + public function main(array $argv = []): void + { + if ($argv) { + parse_str(implode('&', array_slice($argv, 1)), $cliArgs); + $request = $cliArgs; + } else { + $request = array_merge($_GET, $_POST); + } + + try { + foreach ($request as $key => $value) { + if (!is_string($value)) { + throw new \Exception("Query parameter \"$key\" is not a string."); + } } - } - $actionName = $request['action'] ?? 'Frontpage'; - $actionName = strtolower($actionName) . 'Action'; - $actionName = implode(array_map('ucfirst', explode('-', $actionName))); + $actionName = $request['action'] ?? 'Frontpage'; + $actionName = strtolower($actionName) . 'Action'; + $actionName = implode(array_map('ucfirst', explode('-', $actionName))); - $filePath = __DIR__ . '/../actions/' . $actionName . '.php'; - if (!file_exists($filePath)) { - throw new \Exception(sprintf('Invalid action: %s', $actionName)); - } - $className = '\\' . $actionName; - $action = new $className(); + $filePath = __DIR__ . '/../actions/' . $actionName . '.php'; + if (!file_exists($filePath)) { + throw new \Exception('Invalid action', 400); + } + $className = '\\' . $actionName; + $action = new $className(); - $response = $action->execute($request); - if (is_string($response)) { - print $response; - } elseif ($response instanceof Response) { - $response->send(); + $response = $action->execute($request); + if (is_string($response)) { + print $response; + } elseif ($response instanceof Response) { + $response->send(); + } + } catch (\Throwable $e) { + Logger::error('Exception in RssBridge::main()', ['e' => $e]); + http_response_code(500); + print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); } } @@ -114,6 +122,12 @@ final class RssBridge public static function getCache(): CacheInterface { - return self::$cache; + return self::$cache ?? new NullCache(); + } + + public function clearCache() + { + $cache = self::getCache(); + $cache->clear(); } } diff --git a/lib/TwitterClient.php b/lib/TwitterClient.php index 20f21482..f71e842c 100644 --- a/lib/TwitterClient.php +++ b/lib/TwitterClient.php @@ -12,11 +12,9 @@ class TwitterClient { $this->cache = $cache; - $cache->setScope('twitter'); - $cache->setKey(['cache']); - $cache->purgeCache(60 * 60 * 3); + $data = $this->cache->get('twitter') ?? []; + $this->data = $data; - $this->data = $this->cache->loadData() ?? []; $this->authorization = 'AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR'; $this->tw_consumer_key = '3nVuSoBZnx6U4vzUxf5w'; $this->tw_consumer_secret = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys'; @@ -273,9 +271,7 @@ class TwitterClient $guest_token = json_decode($response)->guest_token; $this->data['guest_token'] = $guest_token; - $this->cache->setScope('twitter'); - $this->cache->setKey(['cache']); - $this->cache->saveData($this->data); + $this->cache->set('twitter', $this->data); } private function fetchUserInfoByScreenName(string $screenName) @@ -299,9 +295,7 @@ class TwitterClient $userInfo = $response->data->user; $this->data[$screenName] = $userInfo; - $this->cache->setScope('twitter'); - $this->cache->setKey(['cache']); - $this->cache->saveData($this->data); + $this->cache->set('twitter', $this->data); return $userInfo; } @@ -434,9 +428,7 @@ class TwitterClient $listInfo = $response->data->user_by_screen_name->list; $this->data[$screenName . '-' . $listSlug] = $listInfo; - $this->cache->setScope('twitter'); - $this->cache->setKey(['cache']); - $this->cache->saveData($this->data); + $this->cache->set('twitter', $this->data); return $listInfo; } diff --git a/lib/bootstrap.php b/lib/bootstrap.php index e05dd94a..ca6cecdb 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -39,10 +39,10 @@ const MAX_FILE_SIZE = 10000000; // Files $files = [ __DIR__ . '/../lib/html.php', - __DIR__ . '/../lib/error.php', __DIR__ . '/../lib/contents.php', __DIR__ . '/../lib/php8backports.php', __DIR__ . '/../lib/utils.php', + __DIR__ . '/../lib/http.php', // Vendor __DIR__ . '/../vendor/parsedown/Parsedown.php', __DIR__ . '/../vendor/php-urljoin/src/urljoin.php', diff --git a/lib/contents.php b/lib/contents.php index c842ccbc..c1847758 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -1,101 +1,11 @@ <?php -final class Response -{ - public const STATUS_CODES = [ - '100' => 'Continue', - '101' => 'Switching Protocols', - '200' => 'OK', - '201' => 'Created', - '202' => 'Accepted', - '203' => 'Non-Authoritative Information', - '204' => 'No Content', - '205' => 'Reset Content', - '206' => 'Partial Content', - '300' => 'Multiple Choices', - '301' => 'Moved Permanently', - '302' => 'Found', - '303' => 'See Other', - '304' => 'Not Modified', - '305' => 'Use Proxy', - '400' => 'Bad Request', - '401' => 'Unauthorized', - '402' => 'Payment Required', - '403' => 'Forbidden', - '404' => 'Not Found', - '405' => 'Method Not Allowed', - '406' => 'Not Acceptable', - '407' => 'Proxy Authentication Required', - '408' => 'Request Timeout', - '409' => 'Conflict', - '410' => 'Gone', - '411' => 'Length Required', - '412' => 'Precondition Failed', - '413' => 'Request Entity Too Large', - '414' => 'Request-URI Too Long', - '415' => 'Unsupported Media Type', - '416' => 'Requested Range Not Satisfiable', - '417' => 'Expectation Failed', - '429' => 'Too Many Requests', - '500' => 'Internal Server Error', - '501' => 'Not Implemented', - '502' => 'Bad Gateway', - '503' => 'Service Unavailable', - '504' => 'Gateway Timeout', - '505' => 'HTTP Version Not Supported' - ]; - private string $body; - private int $code; - private array $headers; - - public function __construct( - string $body = '', - int $code = 200, - array $headers = [] - ) { - $this->body = $body; - $this->code = $code; - $this->headers = $headers; - } - - public function getBody() - { - return $this->body; - } - - public function getCode() - { - return $this->code; - } - - public function getHeaders() - { - return $this->headers; - } - - public function send(): void - { - http_response_code($this->code); - foreach ($this->headers as $name => $value) { - header(sprintf('%s: %s', $name, $value)); - } - print $this->body; - } -} - /** * Fetch data from an http url * * @param array $httpHeaders E.g. ['Content-type: text/plain'] * @param array $curlOptions Associative array e.g. [CURLOPT_MAXREDIRS => 3] - * @param bool $returnFull Whether to return an array: - * [ - * 'code' => int, - * 'header' => array, - * 'content' => string, - * 'status_lines' => array, - * ] - + * @param bool $returnFull Whether to return an array: ['code' => int, 'headers' => array, 'content' => string] * @return string|array */ function getContents( @@ -142,30 +52,35 @@ function getContents( } $cache = RssBridge::getCache(); - $cache->setScope('server'); - $cache->setKey([$url]); + $cacheKey = 'server_' . $url; - if (!Debug::isEnabled() && $cache->getTime() && $cache->loadData(86400 * 7)) { - $config['if_not_modified_since'] = $cache->getTime(); + /** @var Response $cachedResponse */ + $cachedResponse = $cache->get($cacheKey); + if ($cachedResponse) { + // considering popping + $cachedLastModified = $cachedResponse->getHeader('last-modified'); + if ($cachedLastModified) { + $cachedLastModified = new \DateTimeImmutable($cachedLastModified); + $config['if_not_modified_since'] = $cachedLastModified->getTimestamp(); + } } $response = $httpClient->request($url, $config); - switch ($response['code']) { + switch ($response->getCode()) { case 200: case 201: case 202: - if (isset($response['headers']['cache-control'])) { - $cachecontrol = $response['headers']['cache-control']; - $lastValue = array_pop($cachecontrol); - $directives = explode(',', $lastValue); + $cacheControl = $response->getHeader('cache-control'); + if ($cacheControl) { + $directives = explode(',', $cacheControl); $directives = array_map('trim', $directives); if (in_array('no-cache', $directives) || in_array('no-store', $directives)) { // Don't cache as instructed by the server break; } } - $cache->saveData($response['body']); + $cache->set($cacheKey, $response, 86400 * 10); break; case 301: case 302: @@ -174,16 +89,16 @@ function getContents( break; case 304: // Not Modified - $response['body'] = $cache->loadData(86400 * 7); + $response = $response->withBody($cachedResponse->getBody()); break; default: $exceptionMessage = sprintf( '%s resulted in %s %s %s', $url, - $response['code'], - Response::STATUS_CODES[$response['code']] ?? '', + $response->getCode(), + $response->getStatusLine(), // If debug, include a part of the response body in the exception message - Debug::isEnabled() ? mb_substr($response['body'], 0, 500) : '', + Debug::isEnabled() ? mb_substr($response->getBody(), 0, 500) : '', ); // The following code must be extracted if it grows too much @@ -194,141 +109,21 @@ function getContents( '<title>Security | Glassdoor', ]; foreach ($cloudflareTitles as $cloudflareTitle) { - if (str_contains($response['body'], $cloudflareTitle)) { - throw new CloudFlareException($exceptionMessage, $response['code']); + if (str_contains($response->getBody(), $cloudflareTitle)) { + throw new CloudFlareException($exceptionMessage, $response->getCode()); } } - throw new HttpException(trim($exceptionMessage), $response['code']); + throw new HttpException(trim($exceptionMessage), $response->getCode()); } if ($returnFull === true) { - // For legacy reasons, use content instead of body - $response['content'] = $response['body']; - unset($response['body']); - return $response; - } - return $response['body']; -} - -interface HttpClient -{ - public function request(string $url, array $config = []): array; -} - -final class CurlHttpClient implements HttpClient -{ - public function request(string $url, array $config = []): array - { - $defaults = [ - 'useragent' => null, - 'timeout' => 5, - 'headers' => [], - 'proxy' => null, - 'curl_options' => [], - 'if_not_modified_since' => null, - 'retries' => 3, - 'max_filesize' => null, - 'max_redirections' => 5, - ]; - $config = array_merge($defaults, $config); - - $ch = curl_init($url); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_MAXREDIRS, $config['max_redirections']); - curl_setopt($ch, CURLOPT_HEADER, false); - $httpHeaders = []; - foreach ($config['headers'] as $name => $value) { - $httpHeaders[] = sprintf('%s: %s', $name, $value); - } - curl_setopt($ch, CURLOPT_HTTPHEADER, $httpHeaders); - if ($config['useragent']) { - curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']); - } - curl_setopt($ch, CURLOPT_TIMEOUT, $config['timeout']); - curl_setopt($ch, CURLOPT_ENCODING, ''); - curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); - - if ($config['max_filesize']) { - // This option inspects the Content-Length header - curl_setopt($ch, CURLOPT_MAXFILESIZE, $config['max_filesize']); - curl_setopt($ch, CURLOPT_NOPROGRESS, false); - // This progress function will monitor responses who omit the Content-Length header - curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function ($ch, $downloadSize, $downloaded, $uploadSize, $uploaded) use ($config) { - if ($downloaded > $config['max_filesize']) { - // Return a non-zero value to abort the transfer - return -1; - } - return 0; - }); - } - - if ($config['proxy']) { - curl_setopt($ch, CURLOPT_PROXY, $config['proxy']); - } - if (curl_setopt_array($ch, $config['curl_options']) === false) { - throw new \Exception('Tried to set an illegal curl option'); - } - - if ($config['if_not_modified_since']) { - curl_setopt($ch, CURLOPT_TIMEVALUE, $config['if_not_modified_since']); - curl_setopt($ch, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - } - - $responseStatusLines = []; - $responseHeaders = []; - curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($ch, $rawHeader) use (&$responseHeaders, &$responseStatusLines) { - $len = strlen($rawHeader); - if ($rawHeader === "\r\n") { - return $len; - } - if (preg_match('#^HTTP/(2|1.1|1.0)#', $rawHeader)) { - $responseStatusLines[] = $rawHeader; - return $len; - } - $header = explode(':', $rawHeader); - if (count($header) === 1) { - return $len; - } - $name = mb_strtolower(trim($header[0])); - $value = trim(implode(':', array_slice($header, 1))); - if (!isset($responseHeaders[$name])) { - $responseHeaders[$name] = []; - } - $responseHeaders[$name][] = $value; - return $len; - }); - - $attempts = 0; - while (true) { - $attempts++; - $data = curl_exec($ch); - if ($data !== false) { - // The network call was successful, so break out of the loop - break; - } - if ($attempts > $config['retries']) { - // Finally give up - $curl_error = curl_error($ch); - $curl_errno = curl_errno($ch); - throw new HttpException(sprintf( - 'cURL error %s: %s (%s) for %s', - $curl_error, - $curl_errno, - 'https://curl.haxx.se/libcurl/c/libcurl-errors.html', - $url - )); - } - } - - $statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - curl_close($ch); return [ - 'code' => $statusCode, - 'status_lines' => $responseStatusLines, - 'headers' => $responseHeaders, - 'body' => $data, + 'code' => $response->getCode(), + 'headers' => $response->getHeaders(), + // For legacy reasons, use 'content' instead of 'body' + 'content' => $response->getBody(), ]; } + return $response->getBody(); } /** @@ -391,7 +186,7 @@ function getSimpleHTMLDOM( * _Notice_: Cached contents are forcefully removed after 24 hours (86400 seconds). * * @param string $url The URL. - * @param int $timeout Cache duration in seconds. + * @param int $ttl Cache duration in seconds. * @param array $header (optional) A list of cURL header. * For more information follow the links below. * * https://php.net/manual/en/function.curl-setopt.php @@ -416,7 +211,7 @@ function getSimpleHTMLDOM( */ function getSimpleHTMLDOMCached( $url, - $timeout = 86400, + $ttl = 86400, $header = [], $opts = [], $lowercase = true, @@ -427,14 +222,11 @@ function getSimpleHTMLDOMCached( $defaultSpanText = DEFAULT_SPAN_TEXT ) { $cache = RssBridge::getCache(); - $cache->setScope('pages'); - $cache->setKey([$url]); - $content = $cache->loadData($timeout); - if (!$content || Debug::isEnabled()) { + $cacheKey = 'pages_' . $url; + $content = $cache->get($cacheKey); + if (!$content) { $content = getContents($url, $header ?? [], $opts ?? []); - $cache->setScope('pages'); - $cache->setKey([$url]); - $cache->saveData($content); + $cache->set($cacheKey, $content, $ttl); } return str_get_html( $content, diff --git a/lib/error.php b/lib/error.php deleted file mode 100644 index 4439fb38..00000000 --- a/lib/error.php +++ /dev/null @@ -1,47 +0,0 @@ -<?php - -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * Throws an exception when called. - * - * @throws \Exception when called - * @param string $message The error message - * @param int $code The HTTP error code - * @link https://en.wikipedia.org/wiki/List_of_HTTP_status_codes List of HTTP - * status codes - */ -function returnError($message, $code) -{ - throw new \Exception($message, $code); -} - -/** - * Returns HTTP Error 400 (Bad Request) when called. - * - * @param string $message The error message - */ -function returnClientError($message) -{ - returnError($message, 400); -} - -/** - * Returns HTTP Error 500 (Internal Server Error) when called. - * - * @param string $message The error message - */ -function returnServerError($message) -{ - returnError($message, 500); -} diff --git a/lib/http.php b/lib/http.php new file mode 100644 index 00000000..c5e65e77 --- /dev/null +++ b/lib/http.php @@ -0,0 +1,252 @@ +<?php + +class HttpException extends \Exception +{ +} + +final class CloudFlareException extends HttpException +{ +} + +interface HttpClient +{ + public function request(string $url, array $config = []): Response; +} + +final class CurlHttpClient implements HttpClient +{ + public function request(string $url, array $config = []): Response + { + $defaults = [ + 'useragent' => null, + 'timeout' => 5, + 'headers' => [], + 'proxy' => null, + 'curl_options' => [], + 'if_not_modified_since' => null, + 'retries' => 3, + 'max_filesize' => null, + 'max_redirections' => 5, + ]; + $config = array_merge($defaults, $config); + + $ch = curl_init($url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_MAXREDIRS, $config['max_redirections']); + curl_setopt($ch, CURLOPT_HEADER, false); + $httpHeaders = []; + foreach ($config['headers'] as $name => $value) { + $httpHeaders[] = sprintf('%s: %s', $name, $value); + } + curl_setopt($ch, CURLOPT_HTTPHEADER, $httpHeaders); + if ($config['useragent']) { + curl_setopt($ch, CURLOPT_USERAGENT, $config['useragent']); + } + curl_setopt($ch, CURLOPT_TIMEOUT, $config['timeout']); + curl_setopt($ch, CURLOPT_ENCODING, ''); + curl_setopt($ch, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + + if ($config['max_filesize']) { + // This option inspects the Content-Length header + curl_setopt($ch, CURLOPT_MAXFILESIZE, $config['max_filesize']); + curl_setopt($ch, CURLOPT_NOPROGRESS, false); + // This progress function will monitor responses who omit the Content-Length header + curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function ($ch, $downloadSize, $downloaded, $uploadSize, $uploaded) use ($config) { + if ($downloaded > $config['max_filesize']) { + // Return a non-zero value to abort the transfer + return -1; + } + return 0; + }); + } + + if ($config['proxy']) { + curl_setopt($ch, CURLOPT_PROXY, $config['proxy']); + } + if (curl_setopt_array($ch, $config['curl_options']) === false) { + throw new \Exception('Tried to set an illegal curl option'); + } + + if ($config['if_not_modified_since']) { + curl_setopt($ch, CURLOPT_TIMEVALUE, $config['if_not_modified_since']); + curl_setopt($ch, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + } + + $responseStatusLines = []; + $responseHeaders = []; + curl_setopt($ch, CURLOPT_HEADERFUNCTION, function ($ch, $rawHeader) use (&$responseHeaders, &$responseStatusLines) { + $len = strlen($rawHeader); + if ($rawHeader === "\r\n") { + return $len; + } + if (preg_match('#^HTTP/(2|1.1|1.0)#', $rawHeader)) { + $responseStatusLines[] = trim($rawHeader); + return $len; + } + $header = explode(':', $rawHeader); + if (count($header) === 1) { + return $len; + } + $name = mb_strtolower(trim($header[0])); + $value = trim(implode(':', array_slice($header, 1))); + if (!isset($responseHeaders[$name])) { + $responseHeaders[$name] = []; + } + $responseHeaders[$name][] = $value; + return $len; + }); + + $attempts = 0; + while (true) { + $attempts++; + $data = curl_exec($ch); + if ($data !== false) { + // The network call was successful, so break out of the loop + break; + } + if ($attempts > $config['retries']) { + // Finally give up + $curl_error = curl_error($ch); + $curl_errno = curl_errno($ch); + throw new HttpException(sprintf( + 'cURL error %s: %s (%s) for %s', + $curl_error, + $curl_errno, + 'https://curl.haxx.se/libcurl/c/libcurl-errors.html', + $url + )); + } + } + + $statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + return new Response($data, $statusCode, $responseHeaders); + } +} + +final class Response +{ + public const STATUS_CODES = [ + '100' => 'Continue', + '101' => 'Switching Protocols', + '200' => 'OK', + '201' => 'Created', + '202' => 'Accepted', + '203' => 'Non-Authoritative Information', + '204' => 'No Content', + '205' => 'Reset Content', + '206' => 'Partial Content', + '300' => 'Multiple Choices', + '301' => 'Moved Permanently', + '302' => 'Found', + '303' => 'See Other', + '304' => 'Not Modified', + '305' => 'Use Proxy', + '400' => 'Bad Request', + '401' => 'Unauthorized', + '402' => 'Payment Required', + '403' => 'Forbidden', + '404' => 'Not Found', + '405' => 'Method Not Allowed', + '406' => 'Not Acceptable', + '407' => 'Proxy Authentication Required', + '408' => 'Request Timeout', + '409' => 'Conflict', + '410' => 'Gone', + '411' => 'Length Required', + '412' => 'Precondition Failed', + '413' => 'Request Entity Too Large', + '414' => 'Request-URI Too Long', + '415' => 'Unsupported Media Type', + '416' => 'Requested Range Not Satisfiable', + '417' => 'Expectation Failed', + '429' => 'Too Many Requests', + '500' => 'Internal Server Error', + '501' => 'Not Implemented', + '502' => 'Bad Gateway', + '503' => 'Service Unavailable', + '504' => 'Gateway Timeout', + '505' => 'HTTP Version Not Supported' + ]; + private string $body; + private int $code; + private array $headers; + + public function __construct( + string $body = '', + int $code = 200, + array $headers = [] + ) { + $this->body = $body; + $this->code = $code; + $this->headers = []; + + foreach ($headers as $name => $value) { + $name = mb_strtolower($name); + if (!isset($this->headers[$name])) { + $this->headers[$name] = []; + } + if (is_string($value)) { + $this->headers[$name][] = $value; + } + if (is_array($value)) { + $this->headers[$name] = $value; + } + } + } + + public function getBody() + { + return $this->body; + } + + public function getCode() + { + return $this->code; + } + + public function getStatusLine(): string + { + return self::STATUS_CODES[$this->code] ?? ''; + } + + public function getHeaders() + { + return $this->headers; + } + + /** + * @return string[]|string|null + */ + public function getHeader(string $name, bool $all = false) + { + $name = mb_strtolower($name); + $header = $this->headers[$name] ?? null; + if (!$header) { + return null; + } + if ($all) { + return $header; + } + return array_pop($header); + } + + public function withBody(string $body): Response + { + $clone = clone $this; + $clone->body = $body; + return $clone; + } + + public function send(): void + { + http_response_code($this->code); + foreach ($this->headers as $name => $values) { + foreach ($values as $value) { + header(sprintf('%s: %s', $name, $value)); + } + } + print $this->body; + } +} diff --git a/lib/utils.php b/lib/utils.php index 94f928cd..4c58d258 100644 --- a/lib/utils.php +++ b/lib/utils.php @@ -1,18 +1,17 @@ <?php -class HttpException extends \Exception -{ -} - -final class CloudFlareException extends HttpException -{ -} - +// https://github.com/nette/utils/blob/master/src/Utils/Json.php final class Json { - public static function encode($value): string + public static function encode($value, $pretty = true, bool $asciiSafe = false): string { - $flags = JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE; + $flags = JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES; + if (!$asciiSafe) { + $flags = $flags | JSON_UNESCAPED_UNICODE; + } + if ($pretty) { + $flags = $flags | JSON_PRETTY_PRINT; + } return \json_encode($value, $flags); } @@ -237,3 +236,13 @@ function create_random_string(int $bytes = 16): string { return bin2hex(openssl_random_pseudo_bytes($bytes)); } + +function returnClientError($message) +{ + throw new \Exception($message, 400); +} + +function returnServerError($message) +{ + throw new \Exception($message, 500); +} diff --git a/tests/Actions/ListActionTest.php b/tests/Actions/ListActionTest.php index e0625fb3..74a90254 100644 --- a/tests/Actions/ListActionTest.php +++ b/tests/Actions/ListActionTest.php @@ -17,7 +17,8 @@ class ListActionTest extends TestCase $action = new \ListAction(); $response = $action->execute([]); $headers = $response->getHeaders(); - $this->assertSame($headers['Content-Type'], 'application/json'); + $contentType = $response->getHeader('content-type'); + $this->assertSame($contentType, 'application/json'); } public function testOutput() diff --git a/tests/CacheTest.php b/tests/CacheTest.php index 9a8ada14..15d03ec1 100644 --- a/tests/CacheTest.php +++ b/tests/CacheTest.php @@ -27,17 +27,13 @@ class CacheTest extends TestCase 'path' => $temporaryFolder, 'enable_purge' => true, ]); - $sut->setScope('scope'); - $sut->purgeCache(-1); - $sut->setKey(['key']); + $sut->clear(); - $this->assertNull($sut->getTime()); - $this->assertNull($sut->loadData()); + $this->assertNull($sut->get('key')); - $sut->saveData('data'); - $this->assertSame('data', $sut->loadData()); - $this->assertIsNumeric($sut->getTime()); - $sut->purgeCache(-1); + $sut->set('key', 'data', 5); + $this->assertSame('data', $sut->get('key')); + $sut->clear(); // Intentionally not deleting the temp folder } From 3178deb5a8ce979baf186b69e5a427bb14adef4f Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 10 Sep 2023 23:35:40 +0200 Subject: [PATCH 101/716] fix: mastodon, cache tweaks, docs (#3661) * cache tweaks * docs * fix(mastodon): type bug --- bridges/ItakuBridge.php | 2 +- bridges/MastodonBridge.php | 10 +++++++++- bridges/PixivBridge.php | 8 +++++--- caches/SQLiteCache.php | 4 ++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index 6b0ebcb2..62a130ff 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -664,7 +664,7 @@ class ItakuBridge extends BridgeAbstract // Debug::log($url); if ($getJSON) { //get JSON object if ($cache) { - $data = $this->loadCacheValue($url, 86400); // 24 hours + $data = $this->loadCacheValue($url); if (is_null($data)) { $data = getContents($url, $httpHeaders, $curlOptions) or returnServerError("Could not load $url"); $this->saveCacheValue($url, $data); diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index 81401be9..54ac55bd 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -100,6 +100,10 @@ class MastodonBridge extends BridgeAbstract // We fetch the boosted content. try { $rtContent = $this->fetchAP($content['object']); + if (!$rtContent) { + // Sometimes fetchAP returns null. Someone should figure out why. json_decode failure? + break; + } $rtUser = $this->loadCacheValue($rtContent['attributedTo']); if (!isset($rtUser)) { // We fetch the author, since we cannot always assume the format of the URL. @@ -277,6 +281,10 @@ class MastodonBridge extends BridgeAbstract array_push($headers, $sig); } } - return json_decode(getContents($url, $headers), true); + try { + return Json::decode(getContents($url, $headers)); + } catch (\JsonException $e) { + return null; + } } } diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php index 5549c609..c4f5277f 100644 --- a/bridges/PixivBridge.php +++ b/bridges/PixivBridge.php @@ -353,10 +353,12 @@ class PixivBridge extends BridgeAbstract private function getCookie() { // checks if cookie is set, if not initialise it with the cookie from the config - $value = $this->loadCacheValue('cookie', 2678400 /* 30 days + 1 day to let cookie chance to renew */); + $value = $this->loadCacheValue('cookie'); if (!isset($value)) { $value = $this->getOption('cookie'); - $this->saveCacheValue('cookie', $this->getOption('cookie')); + + // 30 days + 1 day to let cookie chance to renew + $this->saveCacheValue('cookie', $this->getOption('cookie'), 2678400); } return $value; } @@ -370,7 +372,7 @@ class PixivBridge extends BridgeAbstract } if ($cache) { - $data = $this->loadCacheValue($url, 86400); // 24 hours + $data = $this->loadCacheValue($url); if (!$data) { $data = getContents($url, $httpHeaders, $curlOptions, true) or returnServerError("Could not load $url"); $this->saveCacheValue($url, $data); diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index beb33e88..09689566 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -2,6 +2,10 @@ declare(strict_types=1); +/** + * The storage table has a column `updated` which is incorrectly named. + * It should have been named `expiration` and the code treats it as an expiration date (in unix timestamp) + */ class SQLiteCache implements CacheInterface { private \SQLite3 $db; From a9cf1512e787d3dabb875fe033a0c08a4c274c65 Mon Sep 17 00:00:00 2001 From: ImportTaste <53661808+ImportTaste@users.noreply.github.com> Date: Sun, 10 Sep 2023 20:35:09 -0500 Subject: [PATCH 102/716] [SteamAppNewsBridge] Add tags filter (#3662) Undocumented tags filter discovered through /ISteamWebAPIUtil/GetSupportedAPIList/v1/ e.g. /ISteamNews/GetNewsForApp/v2/?appid=1091500&tags=patchnotes --- bridges/SteamAppNewsBridge.php | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bridges/SteamAppNewsBridge.php b/bridges/SteamAppNewsBridge.php index 3eedc4ab..085e6978 100644 --- a/bridges/SteamAppNewsBridge.php +++ b/bridges/SteamAppNewsBridge.php @@ -27,18 +27,26 @@ class SteamAppNewsBridge extends BridgeAbstract 'title' => '# of posts to retrieve (default 20)', 'type' => 'number', 'defaultValue' => 20 + ], + 'tags' => [ + 'name' => 'Tag Filter', + 'title' => 'Comma-separated list of tags to filter by', + 'type' => 'text', + 'exampleValue' => 'patchnotes' ] ]]; public function collectData() { - $api = 'https://api.steampowered.com/ISteamNews/GetNewsForApp/v2/'; + $apiTarget = 'https://api.steampowered.com/ISteamNews/GetNewsForApp/v2/'; // Example with params: https://api.steampowered.com/ISteamNews/GetNewsForApp/v2/?appid=730&maxlength=0&count=20 // More info at dev docs https://partner.steamgames.com/doc/webapi/ISteamNews - $url = $api . '?appid=' - . $this->getInput('appid') . '&maxlength=' - . $this->getInput('maxlength') . '&count=' - . $this->getInput('count'); + $url = + $apiTarget + . '?appid=' . $this->getInput('appid') + . '&maxlength=' . $this->getInput('maxlength') + . '&count=' . $this->getInput('count') + . '&tags=' . $this->getInput('tags'); // Get the JSON content $json = getContents($url); From 3e1e96e477a26cf81582c16e9ef1edf399702b31 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Mon, 11 Sep 2023 20:45:14 +0930 Subject: [PATCH 103/716] [PatreonBridge] resolve null coalescing issue (#3664) * extend post presentation * applied phpcbf note: phpcs does not like long null coalescing chains * resolved phpcs * resolved github comment https://github.com/RSS-Bridge/rss-bridge/pull/3617/#issuecomment-1699568400 * . * lint SteamAppNewsBridge --- bridges/PatreonBridge.php | 15 +++++++++------ bridges/SteamAppNewsBridge.php | 3 +-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/bridges/PatreonBridge.php b/bridges/PatreonBridge.php index 3d19cd81..d72bde20 100644 --- a/bridges/PatreonBridge.php +++ b/bridges/PatreonBridge.php @@ -132,8 +132,9 @@ class PatreonBridge extends BridgeAbstract $audio = $this->findInclude($posts, 'media', $id)->attributes ?? null; } } - $thumbnail = $post->attributes->thumbnail->large ?? $post->attributes->thumbnail->url; - $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url; + $thumbnail = $post->attributes->thumbnail->large ?? null; + $thumbnail = $thumbnail ?? $post->attributes->thumbnail->url ?? null; + $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url ?? null; $thumbnail = $thumbnail ?? $post->attributes->image->url; $audio_filename = $audio->file_name ?? $item['title']; $download_url = $audio->download_url ?? $item['uri']; @@ -146,15 +147,17 @@ class PatreonBridge extends BridgeAbstract break; case 'video_embed': - $thumbnail = $post->attributes->thumbnail->large ?? $post->attributes->thumbnail->url; - $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url; + $thumbnail = $post->attributes->thumbnail->large ?? null; + $thumbnail = $thumbnail ?? $post->attributes->thumbnail->url ?? null; + $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url ?? null; $thumbnail = $thumbnail ?? $post->attributes->image->url; $item['content'] .= "<p><a href=\"{$item['uri']}\">🎬 {$item['title']}<br><img src=\"{$thumbnail}\"></a></p>"; break; case 'video_external_file': - $thumbnail = $post->attributes->thumbnail->large ?? $post->attributes->thumbnail->url; - $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url; + $thumbnail = $post->attributes->thumbnail->large ?? null; + $thumbnail = $thumbnail ?? $post->attributes->thumbnail->url ?? null; + $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url ?? null; $thumbnail = $thumbnail ?? $post->attributes->image->url; $item['content'] .= "<p><a href=\"{$item['uri']}\">🎬 {$item['title']}<br><img src=\"{$thumbnail}\"></a></p>"; break; diff --git a/bridges/SteamAppNewsBridge.php b/bridges/SteamAppNewsBridge.php index 085e6978..81045800 100644 --- a/bridges/SteamAppNewsBridge.php +++ b/bridges/SteamAppNewsBridge.php @@ -41,8 +41,7 @@ class SteamAppNewsBridge extends BridgeAbstract $apiTarget = 'https://api.steampowered.com/ISteamNews/GetNewsForApp/v2/'; // Example with params: https://api.steampowered.com/ISteamNews/GetNewsForApp/v2/?appid=730&maxlength=0&count=20 // More info at dev docs https://partner.steamgames.com/doc/webapi/ISteamNews - $url = - $apiTarget + $url = $apiTarget . '?appid=' . $this->getInput('appid') . '&maxlength=' . $this->getInput('maxlength') . '&count=' . $this->getInput('count') From 4f5a492dde31377e2fa168430f26c09dae1425e4 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Mon, 11 Sep 2023 20:48:00 +0930 Subject: [PATCH 104/716] [BridgeAbstract] fix undefined index issue (#3665) * . * attempt to fix #2943 https://github.com/RSS-Bridge/rss-bridge/issues/2943 * Revert "." This reverts commit c0b6ccfea6ce873e9c9ce7c3600b3a96d9911468. * lint * Revert "attempt to fix #2943" This reverts commit 9f1a66e48d636a543e2171df212acf9731744bd0. * moved fix to BridgeAbstract * fix undefined index * lint --- lib/BridgeAbstract.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 36a77669..a69552fc 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -192,7 +192,7 @@ abstract class BridgeAbstract implements BridgeInterface if (isset($inputs[$name])) { $value = $inputs[$name]; } else { - if ($properties['type'] === 'checkbox') { + if ($properties['type'] ?? null === 'checkbox') { $value = false; } elseif (isset($properties['defaultValue'])) { $value = $properties['defaultValue']; From 4323a11667806e0f8f3316f426d3834d97ad545d Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:16:11 +0200 Subject: [PATCH 105/716] doc : clarification detectParameters function and test (#3666) * doc : clarification detectParameters function - clarification about the return value of the detectParameters function - add info about the constant TEST_DETECT_PARAMETERS to allow automatest test of the function detectParameters * doc : clarification detectParameters function Add reference to the findFeed action to encourage the implenentation of the detectParameters function * doc : clarification detectParameters function Add commas to improve reading * doc : clarification detectParameters function - fix link --- docs/05_Bridge_API/02_BridgeAbstract.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/05_Bridge_API/02_BridgeAbstract.md b/docs/05_Bridge_API/02_BridgeAbstract.md index 120361be..12d24bdc 100644 --- a/docs/05_Bridge_API/02_BridgeAbstract.md +++ b/docs/05_Bridge_API/02_BridgeAbstract.md @@ -437,9 +437,9 @@ If no icon is specified by the bridge, RSS-Bridge will use a default location: ` # detectParameters The `detectParameters` function takes a URL and attempts to extract a valid set of parameters for the current bridge. -If the passed URL is valid for this bridge the function should return an array of parameter -> value pairs that can be used by this bridge, or an empty array if the bridge requires no parameters. If the URL is not relevant for this bridge the function should return `null`. +If the passed URL is valid for this bridge, the function should return an array of parameter -> value pairs that can be used by this bridge, including context if available, or an empty array if the bridge requires no parameters. If the URL is not relevant for this bridge, the function should return `null`. -**Notice:** Implementing this function is optional. By default **RSS-Bridge** tries to match the supplied URL to the `URI` constant defined in the bridge which may be enough for bridges without any parameters defined. +**Notice:** Implementing this function is optional. By default, **RSS-Bridge** tries to match the supplied URL to the `URI` constant defined in the bridge, which may be enough for bridges without any parameters defined. ```PHP public function detectParameters($url){ @@ -455,6 +455,23 @@ public function detectParameters($url){ } ``` +**Notice:** This function is also used by the [findFeed](../04_For_Developers/04_Actions.md#findfeed) action. This action allows an user to get a list of all feeds corresponding to an URL. + +You can implement automated tests for the `detectParameters` function by adding the `TEST_DETECT_PARAMETERS` constant to your bridge class constant. + +`TEST_DETECT_PARAMETERS` is an array, with as key the URL passed to the `detectParameters`function and as value, the array of parameters returned by `detectParameters` + +```PHP + const TEST_DETECT_PARAMETERS = [ + 'https://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'https://instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'http://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + ]; +``` + +**Notice:** Adding this constant is optional. If the constant is not present, no automated test will be executed. + + *** # Helper Methods From 0175e13712eaecf87686193c951c87e41e4a058c Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite <alexandre@alapetite.fr> Date: Wed, 13 Sep 2023 18:08:22 +0200 Subject: [PATCH 106/716] Docker from Debian base image (#3500) * Docker from Debian base image * Fix expose https://github.com/RSS-Bridge/rss-bridge/discussions/3234 * Re-fix better logs https://github.com/RSS-Bridge/rss-bridge/pull/3333 * Update to Debian 12 Bookworm instead of Debian 10 Buster * Use Debian packaging instead of having to keep track of and manually install -dev libraries, and with LTS support * Update to PHP 8.2 instead of PHP 8.0 * Fix php.ini location * Minor order changes To optimise caching --- .devcontainer/nginx.conf | 4 ++-- .gitignore | 3 +++ Dockerfile | 43 +++++++++++++++++++++++++--------------- config/nginx.conf | 6 +++--- config/php-fpm.conf | 18 +++++++++++++++++ config/php.ini | 4 ++++ docker-entrypoint.sh | 4 ++-- 7 files changed, 59 insertions(+), 23 deletions(-) create mode 100644 config/php-fpm.conf create mode 100644 config/php.ini diff --git a/.devcontainer/nginx.conf b/.devcontainer/nginx.conf index 46502cb4..0e5db6dc 100644 --- a/.devcontainer/nginx.conf +++ b/.devcontainer/nginx.conf @@ -12,6 +12,6 @@ server { location ~ \.php$ { include snippets/fastcgi-php.conf; - fastcgi_pass 127.0.0.1:9000; + fastcgi_pass unix:/var/run/php/php8.2-fpm.sock; } -} \ No newline at end of file +} diff --git a/.gitignore b/.gitignore index f574992d..9725342d 100644 --- a/.gitignore +++ b/.gitignore @@ -230,6 +230,9 @@ pip-log.txt DEBUG config.ini.php config/* +!config/nginx.conf +!config/php-fpm.conf +!config/php.ini ###################### ## VisualStudioCode ## diff --git a/Dockerfile b/Dockerfile index 8157dc12..f504b51f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,47 @@ FROM lwthiker/curl-impersonate:0.5-ff-slim-buster AS curlimpersonate -FROM php:8.0.27-fpm-buster AS rssbridge +FROM debian:12-slim AS rssbridge LABEL description="RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one." LABEL repository="https://github.com/RSS-Bridge/rss-bridge" LABEL website="https://github.com/RSS-Bridge/rss-bridge" +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install --yes --no-install-recommends \ + ca-certificates \ nginx \ - zlib1g-dev \ - libzip-dev \ - libmemcached-dev \ nss-plugin-pem \ - libicu-dev && \ - docker-php-ext-install zip && \ - docker-php-ext-install intl && \ - pecl install memcached && \ - docker-php-ext-enable memcached && \ - docker-php-ext-enable opcache && \ - mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" + php-curl \ + php-fpm \ + php-intl \ + # php-json is enabled by default with PHP 8.2 in Debian 12 + php-mbstring \ + php-memcached \ + # php-opcache is enabled by default with PHP 8.2 in Debian 12 + # php-openssl is enabled by default with PHP 8.2 in Debian 12 + php-sqlite3 \ + php-xml \ + php-zip \ + # php-zlib is enabled by default with PHP 8.2 in Debian 12 + && \ + rm -rf /var/lib/apt/lists/* -COPY ./config/nginx.conf /etc/nginx/sites-enabled/default - -COPY --chown=www-data:www-data ./ /app/ +# logs should go to stdout / stderr +RUN ln -sfT /dev/stderr /var/log/nginx/error.log; \ + ln -sfT /dev/stdout /var/log/nginx/access.log; \ + chown -R --no-dereference www-data:adm /var/log/nginx/ COPY --from=curlimpersonate /usr/local/lib/libcurl-impersonate-ff.so /usr/local/lib/curl-impersonate/ - ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so - ENV CURL_IMPERSONATE ff91esr +COPY ./config/nginx.conf /etc/nginx/sites-available/default +COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf +COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.conf + +COPY --chown=www-data:www-data ./ /app/ + EXPOSE 80 ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/config/nginx.conf b/config/nginx.conf index bb7b1dcb..f0f189e7 100644 --- a/config/nginx.conf +++ b/config/nginx.conf @@ -2,8 +2,8 @@ server { listen 80 default_server; listen [::]:80 default_server; root /app; - access_log /dev/stdout; - error_log /dev/stderr; + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; index index.php; location ~ /(\.|vendor|tests) { @@ -13,6 +13,6 @@ server { location ~ \.php$ { include snippets/fastcgi-php.conf; - fastcgi_pass 127.0.0.1:9000; + fastcgi_pass unix:/var/run/php/php8.2-fpm.sock; } } diff --git a/config/php-fpm.conf b/config/php-fpm.conf new file mode 100644 index 00000000..a508a0f6 --- /dev/null +++ b/config/php-fpm.conf @@ -0,0 +1,18 @@ +; Inspired by https://github.com/docker-library/php/blob/master/8.2/bookworm/fpm/Dockerfile + +[global] +error_log = /proc/self/fd/2 + +; https://github.com/docker-library/php/pull/725#issuecomment-443540114 +log_limit = 8192 + +[www] +; php-fpm closes STDOUT on startup, so sending logs to /proc/self/fd/1 does not work. +; https://bugs.php.net/bug.php?id=73886 +access.log = /proc/self/fd/2 + +clear_env = no + +; Ensure worker stdout and stderr are sent to the main error log. +catch_workers_output = yes +decorate_workers_output = no diff --git a/config/php.ini b/config/php.ini new file mode 100644 index 00000000..115f1c89 --- /dev/null +++ b/config/php.ini @@ -0,0 +1,4 @@ +; Inspired by https://github.com/docker-library/php/blob/master/8.2/bookworm/fpm/Dockerfile + +; https://github.com/docker-library/php/issues/878#issuecomment-938595965' +fastcgi.logging = Off diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index aa95fa87..8dde842c 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -41,5 +41,5 @@ fi # nginx will daemonize nginx -# php-fpm will not -php-fpm +# php-fpm should not daemonize +php-fpm8.2 --nodaemonize From bb7f329e81deb7661ad88b40858a866da05c3e03 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 13 Sep 2023 22:48:37 +0200 Subject: [PATCH 107/716] fix(instructables): migrate from dom to json api (#3667) --- bridges/InstructablesBridge.php | 151 +++++++------------------------- 1 file changed, 30 insertions(+), 121 deletions(-) diff --git a/bridges/InstructablesBridge.php b/bridges/InstructablesBridge.php index e1d2ef0b..16e76e8a 100644 --- a/bridges/InstructablesBridge.php +++ b/bridges/InstructablesBridge.php @@ -237,132 +237,41 @@ class InstructablesBridge extends BridgeAbstract public function collectData() { - // Enable the following line to get the category list (dev mode) - // $this->listCategories(); + $category = $this->getInput('category'); + $filter = $this->getInput('filter'); - $html = getSimpleHTMLDOM($this->getURI()); - $html = defaultLinkTo($html, $this->getURI()); + $api = 'https://www.instructables.com/api_proxy/search/collections/projects/documents/search'; + //$sortBy = 'views:desc'; + $sortBy = 'publishDate:desc'; + //$filterBy = 'featureFlag:=true && category:=Circuits && channel: [Apple,Linux]'; + $filterBy = 'featureFlag:=true && category:=Circuits'; + //$filterBy = 'featureFlag:=true && teachers:=Teachers'; + //$filterBy = 'featureFlag:=true && category:=Craft'; + $params = [ + 'q' => '*', + 'query_by' => 'title,stepBody,screenName', + 'page' => '1', + 'sort_by' => $sortBy, + 'include_fields' => 'title,urlString,coverImageUrl,screenName,favorites,views,primaryClassification,featureFlag,prizeLevel,IMadeItCount', + 'filter_by' => $filterBy, + 'per_page' => '50', + ]; - $covers = $html->find(' - .category-projects-list > div, - .category-landing-projects-list > div, - '); + $url = $api . '?' . http_build_query($params); + /* phpcs:ignore */ + $key = 'TUIxY0xkNjdHV09KaFV1dEVxYVRHNGs1QW1sbzlNVVZBaVZKV2VrODc0VT02ZWFYeyJleGNsdWRlX2ZpZWxkcyI6WyJvdXRfb2YiLCJzZWFyY2hfdGltZV9tcyIsInN0ZXBCb2R5Il0sInBlcl9wYWdlIjo2MH0='; + $json = getContents($url, ["x-typesense-api-key: $key"]); + $data = Json::decode($json, false); - foreach ($covers as $cover) { + foreach ($data->hits as $hit) { + $document = $hit->document; $item = []; - - $item['uri'] = $cover->find('a.ible-title', 0)->href; - $item['title'] = $cover->find('a.ible-title', 0)->innertext; - $item['author'] = $this->getCategoryAuthor($cover); - $item['content'] = '<a href=' - . $item['uri'] - . '><img src=' - . $cover->find('img', 0)->getAttribute('data-src') - . '></a>'; - - $item['enclosures'][] = str_replace( - '.RECTANGLE1', - '.LARGE', - $cover->find('img', 0)->getAttribute('data-src') - ); - + $item['uri'] = 'https://www.instructables.com/' . $document->urlString; + $item['author'] = $document->screenName; + $item['title'] = $document->title; + $item['content'] = '<pre>' . Json::encode($document) . '</pre>'; + $item['enclosures'] = [$document->coverImageUrl]; $this->items[] = $item; } } - - public function getName() - { - switch ($this->queriedContext) { - case 'Category': - foreach (self::PARAMETERS[$this->queriedContext]['category']['values'] as $key => $value) { - $subcategory = array_search($this->getInput('category'), $value); - - if ($subcategory !== false) { - break; - } - } - - $filter = array_search( - $this->getInput('filter'), - self::PARAMETERS[$this->queriedContext]['filter']['values'] - ); - - return $subcategory . ' (' . $filter . ') - ' . static::NAME; - } - - return parent::getName(); - } - - public function getURI() - { - switch ($this->queriedContext) { - case 'Category': - return self::URI - . $this->getInput('category') - . $this->getInput('filter'); - } - - return parent::getURI(); - } - - /** - * Returns a list of categories for development purposes (used to build the - * parameters list) - */ - private function listCategories() - { - // Use home page to acquire main categories - $html = getSimpleHTMLDOM(self::URI); - $html = defaultLinkTo($html, self::URI); - - foreach ($html->find('.home-content-explore-link') as $category) { - // Use arbitrary category to receive full list - $html = getSimpleHTMLDOM($category->href); - - foreach ($html->find('.channel-thumbnail a') as $channel) { - $name = html_entity_decode(trim($channel->title)); - - // Remove unwanted entities - $name = str_replace("'", '', $name); - $name = str_replace(''', '', $name); - - $uri = $channel->href; - - $category_name = explode('/', $uri)[1]; - - if ( - !isset($categories) - || !array_key_exists($category_name, $categories) - || !in_array($uri, $categories[$category_name]) - ) { - $categories[$category_name][$name] = $uri; - } - } - } - - // Build PHP array manually - foreach ($categories as $key => $value) { - $name = ucfirst($key); - echo "'{$name}' => array(\n"; - echo "\t'All' => '/{$key}/',\n"; - foreach ($value as $name => $uri) { - echo "\t'{$name}' => '{$uri}',\n"; - } - echo "),\n"; - } - - die; - } - - /** - * Returns the author as anchor for a given cover. - */ - private function getCategoryAuthor($cover) - { - return '<a href=' - . $cover->find('.ible-author a', 0)->href - . '>' - . $cover->find('.ible-author a', 0)->innertext - . '</a>'; - } } From 409236e48e4cf8bc2e2ce6e6f73942feee6a8469 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 14 Sep 2023 03:26:01 +0200 Subject: [PATCH 108/716] fix: logic bug in 429 caching logic (#3669) --- bridges/RedditBridge.php | 2 +- bridges/YoutubeBridge.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 196f7d20..5f6070e3 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -90,8 +90,8 @@ class RedditBridge extends BridgeAbstract } catch (HttpException $e) { if ($e->getCode() === 429) { $this->cache->set($cacheKey, true, 60 * 16); - throw $e; } + throw $e; } } diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 8e3ac540..bff62a90 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -205,8 +205,8 @@ class YoutubeBridge extends BridgeAbstract } catch (HttpException $e) { if ($e->getCode() === 429) { $this->cache->set($cacheKey, true, 60 * 16); - throw $e; } + throw $e; } } From 3b91b1d260f8fc9fd725b9797a3a149e85d408d0 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Fri, 15 Sep 2023 01:58:06 +0200 Subject: [PATCH 109/716] [XPathBridge] add option to skip htmlspecialchars (#3672) --- bridges/XPathBridge.php | 24 ++++++++++++++++++-- docs/05_Bridge_API/04_XPathAbstract.md | 3 +++ lib/XPathAbstract.php | 31 +++++++++++++++++++++----- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/bridges/XPathBridge.php b/bridges/XPathBridge.php index 52346aac..35ec6ad1 100644 --- a/bridges/XPathBridge.php +++ b/bridges/XPathBridge.php @@ -59,6 +59,17 @@ EOL, 'type' => 'text', 'required' => false ], + 'raw_content' => [ + 'name' => 'Use raw item description', + 'title' => <<<"EOL" + Whether to use the raw item description or to replace certain characters with + special significance in HTML by HTML entities (using the PHP function htmlspecialchars). + EOL, + 'type' => 'checkbox', + 'defaultValue' => false, + 'required' => false + ], + 'uri' => [ 'name' => 'Item URL selector', 'title' => <<<"EOL" @@ -178,6 +189,15 @@ EOL, 'type' => 'checkbox', return urldecode($this->getInput('content')); } + /** + * Use raw item content + * @return bool + */ + protected function getSettingUseRawItemContent(): bool + { + return $this->getInput('raw_content'); + } + /** * XPath expression for extracting an item link from the item context * @return string @@ -226,9 +246,9 @@ EOL, 'type' => 'checkbox', /** * Fix encoding - * @return string + * @return bool */ - protected function getSettingFixEncoding() + protected function getSettingFixEncoding(): bool { return $this->getInput('fix_encoding'); } diff --git a/docs/05_Bridge_API/04_XPathAbstract.md b/docs/05_Bridge_API/04_XPathAbstract.md index cf091edc..fd697995 100644 --- a/docs/05_Bridge_API/04_XPathAbstract.md +++ b/docs/05_Bridge_API/04_XPathAbstract.md @@ -68,6 +68,9 @@ Should return the XPath expression for extracting an item title from the item co ### Method `getExpressionItemContent()` Should return the XPath expression for extracting an item's content from the item context. +### Method `getSettingUseRawItemContent()` +Should return the 'Use raw item content' setting value (bool true or false). + ### Method `getExpressionItemUri()` Should return the XPath expression for extracting an item link from the item context. diff --git a/lib/XPathAbstract.php b/lib/XPathAbstract.php index 05929322..bac3bfd7 100644 --- a/lib/XPathAbstract.php +++ b/lib/XPathAbstract.php @@ -76,6 +76,15 @@ abstract class XPathAbstract extends BridgeAbstract */ const XPATH_EXPRESSION_ITEM_CONTENT = ''; + /** + * Use raw item content + * Whether to use the raw item content or to replace certain characters with + * special significance in HTML by HTML entities (using the PHP function htmlspecialchars). + * + * Use {@see XPathAbstract::getSettingUseRawItemContent()} to read this parameter + */ + const SETTING_USE_RAW_ITEM_CONTENT = false; + /** * XPath expression for extracting an item link from the item context * This expression should match a node's attribute containing the article URL @@ -236,6 +245,15 @@ abstract class XPathAbstract extends BridgeAbstract return static::XPATH_EXPRESSION_ITEM_CONTENT; } + /** + * Use raw item content + * @return bool + */ + protected function getSettingUseRawItemContent(): bool + { + return static::SETTING_USE_RAW_ITEM_CONTENT; + } + /** * XPath expression for extracting an item link from the item context * @return string @@ -284,9 +302,9 @@ abstract class XPathAbstract extends BridgeAbstract /** * Fix encoding - * @return string + * @return bool */ - protected function getSettingFixEncoding() + protected function getSettingFixEncoding(): bool { return static::SETTING_FIX_ENCODING; } @@ -313,6 +331,8 @@ abstract class XPathAbstract extends BridgeAbstract return $this->getExpressionItemTitle(); case 'content': return $this->getExpressionItemContent(); + case 'raw_content': + return $this->getSettingUseRawItemContent(); case 'uri': return $this->getExpressionItemUri(); case 'author': @@ -417,7 +437,8 @@ abstract class XPathAbstract extends BridgeAbstract continue; } - $value = $this->getItemValueOrNodeValue($typedResult, $param === 'content'); + $isContent = $param === 'content'; + $value = $this->getItemValueOrNodeValue($typedResult, $isContent, $isContent && !$this->getSettingUseRawItemContent()); $item->__set($param, $this->formatParamValue($param, $value)); } @@ -573,7 +594,7 @@ abstract class XPathAbstract extends BridgeAbstract * @param $typedResult * @return string */ - protected function getItemValueOrNodeValue($typedResult, $returnXML = false) + protected function getItemValueOrNodeValue($typedResult, $returnXML = false, $escapeHtml = false) { if ($typedResult instanceof \DOMNodeList) { $item = $typedResult->item(0); @@ -596,7 +617,7 @@ abstract class XPathAbstract extends BridgeAbstract $text = trim($text); - if ($returnXML) { + if ($escapeHtml) { return htmlspecialchars($text); } return $text; From cf7e3eea5612f3466f297883395723e5b325fd1e Mon Sep 17 00:00:00 2001 From: Scott Colby <scolby33@users.noreply.github.com> Date: Fri, 15 Sep 2023 17:41:08 -0400 Subject: [PATCH 110/716] Add DeutscheWelle FeedExpander bridge. (#3673) * [DeutscheWelle] Add DeutscheWelle FeedExpander bridge. * [DeutscheWelle] Fix linting errors. --- bridges/DeutscheWelleBridge.php | 143 ++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 bridges/DeutscheWelleBridge.php diff --git a/bridges/DeutscheWelleBridge.php b/bridges/DeutscheWelleBridge.php new file mode 100644 index 00000000..2e10d670 --- /dev/null +++ b/bridges/DeutscheWelleBridge.php @@ -0,0 +1,143 @@ +<?php + +class DeutscheWelleBridge extends FeedExpander +{ + const MAINTAINER = 'No maintainer'; + const NAME = 'Deutsche Welle Bridge'; + const URI = 'https://www.dw.com'; + const DESCRIPTION = 'Returns the full articles instead of only the intro'; + const CACHE_TIMEOUT = 3600; + const PARAMETERS = [[ + 'feed' => [ + 'name' => 'feed', + 'type' => 'list', + 'values' => [ + 'All Top Stories and News Updates' + => 'http://rss.dw.com/atom/rss-en-all', + 'Top Stories' + => 'http://rss.dw.com/atom/rss-en-top', + 'Germany' + => 'http://rss.dw.com/atom/rss-en-ger', + 'World' + => 'http://rss.dw.com/atom/rss-en-world', + 'Europe' + => 'http://rss.dw.com/atom/rss-en-eu', + 'Business' + => 'http://rss.dw.com/atom/rss-en-bus', + 'Science' + => 'http://rss.dw.com/atom/rss_en_science', + 'Environment' + => 'http://rss.dw.com/atom/rss_en_environment', + 'Culture & Lifestyle' + => 'http://rss.dw.com/atom/rss-en-cul', + 'Sports' + => 'http://rss.dw.de/atom/rss-en-sports', + 'Visit Germany' + => 'http://rss.dw.com/atom/rss-en-visitgermany', + 'Asia' + => 'http://rss.dw.com/atom/rss-en-asia', + 'Deutsche Welle Gesamt' + => 'http://rss.dw.com/atom/rss-de-all', + 'Themen des Tages' + => 'http://rss.dw.com/atom/rss-de-top', + 'Nachrichten' + => 'http://rss.dw.com/atom/rss-de-news', + 'Wissenschaft' + => 'http://rss.dw.com/atom/rss-de-wissenschaft', + 'Sport' + => 'http://rss.dw.com/atom/rss-de-sport', + 'Deutschland entdecken' + => 'http://rss.dw.com/atom/rss-de-deutschlandentdecken', + 'Presse' + => 'http://rss.dw.com/atom/presse', + 'Politik' + => 'http://rss.dw.com/atom/rss_de_politik', + 'Wirtschaft' + => 'http://rss.dw.com/atom/rss-de-eco', + 'Kultur & Leben' + => 'http://rss.dw.com/atom/rss-de-cul', + 'Kultur & Leben: Buch' + => 'http://rss.dw.com/atom/rss-de-cul-buch', + 'Kultur & Leben: Film' + => 'http://rss.dw.com/atom/rss-de-cul-film', + 'Kultur & Leben: Musik' + => 'http://rss.dw.com/atom/rss-de-cul-musik', + ] + ] + ]]; + + public function collectData() + { + $this->collectExpandableDatas($this->getInput('feed')); + } + + protected function parseItem($item) + { + $item = parent::parseItem($item); + + $parsedUrl = parse_url($item['uri']); + unset($parsedUrl['query']); + $url = $this->unparseUrl($parsedUrl); + + $page = getSimpleHTMLDOM($url); + $page = defaultLinkTo($page, $url); + + $article = $page->find('article', 0); + + // author + $author = $article->find('.author-link > span', 0); + if ($author) { + $item['author'] = $author->text(); + } + + $teaser = $article->find('.teaser-text', 0); + if (!is_null($teaser)) { + $item['content'] = $teaser->outertext(); + } else { + $item['content'] = ''; + } + + // remove unneeded elements + foreach ( + $article->find( + 'header, .advertisement, [data-tracking-name="sharing-icons-inline"], a.external-link > svg, picture > source, .vjs-wrapper, .dw-widget, footer' + ) as $bad + ) { + $bad->remove(); + } + // reload html as remove() is buggy + $article = str_get_html($article->outertext()); + + // remove width and height values from img tags + foreach ($article->find('img') as $img) { + $img->width = null; + $img->height = null; + } + + // replace lazy-loaded images + foreach ($article->find('figure.placeholder-image') as $figure) { + $img = $figure->find('img', 0); + $img->src = str_replace('${formatId}', '906', $img->getAttribute('data-url')); + $img->style = null; + } + + $item['content'] .= $article->save(); + + return $item; + } + + // https://www.php.net/manual/en/function.parse-url.php#106731 + private function unparseUrl($parsed_url) + { + $scheme = isset($parsed_url['scheme']) ? $parsed_url['scheme'] . '://' : ''; + $host = isset($parsed_url['host']) ? $parsed_url['host'] : ''; + $port = isset($parsed_url['port']) ? ':' . $parsed_url['port'] : ''; + $user = isset($parsed_url['user']) ? $parsed_url['user'] : ''; + $pass = isset($parsed_url['pass']) ? $parsed_url['pass'] : ''; + $pass = ($user || $pass) ? "$pass@" : ''; + $path = isset($parsed_url['path']) ? $parsed_url['path'] : ''; + $query = isset($parsed_url['query']) ? '?' . $parsed_url['query'] : ''; + $fragment = isset($parsed_url['fragment']) ? '#' . $parsed_url['fragment'] : ''; + return "$scheme$user$pass$host$port$path$query$fragment"; + } +} From e6aef73a02f7220f2a2ce8acc9d4f6c627ceb88a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 20 Sep 2023 02:45:48 +0200 Subject: [PATCH 111/716] refactor (#3668) --- actions/DisplayAction.php | 8 +- docs/05_Bridge_API/02_BridgeAbstract.md | 49 ++--- docs/05_Bridge_API/03_FeedExpander.md | 5 +- docs/05_Bridge_API/index.md | 6 +- docs/07_Cache_API/index.md | 8 +- docs/09_Technical_recommendations/index.md | 2 +- lib/BridgeAbstract.php | 213 +++++---------------- lib/BridgeCard.php | 6 +- lib/BridgeFactory.php | 2 +- lib/BridgeInterface.php | 145 -------------- lib/FeedExpander.php | 2 +- lib/RssBridge.php | 21 +- lib/contents.php | 31 ++- lib/http.php | 17 +- tests/Bridges/BridgeImplementationTest.php | 3 +- 15 files changed, 134 insertions(+), 384 deletions(-) delete mode 100644 lib/BridgeInterface.php diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 7c59b3d5..efa4d3b5 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -84,7 +84,7 @@ class DisplayAction implements ActionInterface return $response; } - private function createResponse(array $request, BridgeInterface $bridge, FormatInterface $format) + private function createResponse(array $request, BridgeAbstract $bridge, FormatInterface $format) { $items = []; $infos = []; @@ -110,8 +110,6 @@ class DisplayAction implements ActionInterface 'icon' => $bridge->getIcon() ]; } catch (\Exception $e) { - $errorOutput = Configuration::getConfig('error', 'output'); - $reportLimit = Configuration::getConfig('error', 'report_limit'); if ($e instanceof HttpException) { // Reproduce (and log) these responses regardless of error output and report limit if ($e->getCode() === 429) { @@ -124,6 +122,8 @@ class DisplayAction implements ActionInterface } } Logger::error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]); + $errorOutput = Configuration::getConfig('error', 'output'); + $reportLimit = Configuration::getConfig('error', 'report_limit'); $errorCount = 1; if ($reportLimit > 1) { $errorCount = $this->logBridgeError($bridge->getName(), $e->getCode()); @@ -152,7 +152,7 @@ class DisplayAction implements ActionInterface return new Response($format->stringify(), 200, $headers); } - private function createFeedItemFromException($e, BridgeInterface $bridge): FeedItem + private function createFeedItemFromException($e, BridgeAbstract $bridge): FeedItem { $item = new FeedItem(); diff --git a/docs/05_Bridge_API/02_BridgeAbstract.md b/docs/05_Bridge_API/02_BridgeAbstract.md index 12d24bdc..a8e9db42 100644 --- a/docs/05_Bridge_API/02_BridgeAbstract.md +++ b/docs/05_Bridge_API/02_BridgeAbstract.md @@ -94,7 +94,7 @@ class MyBridge extends BridgeAbstract { const MAINTAINER = 'ghost'; public function collectData() { - $item = array(); // Create an empty item + $item = []; // Create an empty item $item['title'] = 'Hello World!'; @@ -121,11 +121,11 @@ class MyBridge extends BridgeAbstract { const URI = ''; const DESCRIPTION = 'No description provided'; const MAINTAINER = 'No maintainer'; - const PARAMETERS = array(); // Can be omitted! + const PARAMETERS = []; // Can be omitted! const CACHE_TIMEOUT = 3600; // Can be omitted! public function collectData() { - $item = array(); // Create an empty item + $item = []; // Create an empty item $item['title'] = 'Hello World!'; @@ -145,7 +145,7 @@ For information on how to read parameter values during execution, please refer t ## Adding parameters to a bridge -Parameters are specified as part of the bridge class. An empty list of parameters is defined as `const PARAMETERS = array();` +Parameters are specified as part of the bridge class. An empty list of parameters is defined as `const PARAMETERS = [];` <details><summary>Show example</summary><div> @@ -153,7 +153,7 @@ Parameters are specified as part of the bridge class. An empty list of parameter <?PHP class MyBridge extends BridgeAbstract { /* ... */ - const PARAMETERS = array(); // Empty list of parameters (can be omitted) + const PARAMETERS = []; // Empty list of parameters (can be omitted) /* ... */ } ``` @@ -172,10 +172,10 @@ A context is defined as a associative array of parameters. The name of a context <details><summary>Show example</summary><div> ```PHP -const PARAMETERS = array( - 'My Context 1' => array(), - 'My Context 2' => array() -); +const PARAMETERS = [ + 'My Context 1' => [], + 'My Context 2' => [], +]; ``` **Output** @@ -189,9 +189,9 @@ _Notice_: The name of a context can be left empty if only one context is needed! <details><summary>Show example</summary><div> ```PHP -const PARAMETERS = array( - array() -); +const PARAMETERS = [ + [] +]; ``` </div></details><br> @@ -201,25 +201,28 @@ You can also define a set of parameters that will be applied to every possible c <details><summary>Show example</summary><div> ```PHP -const PARAMETERS = array( - 'global' => array() // Applies to all contexts! -); +const PARAMETERS = [ + 'global' => [] // Applies to all contexts! +]; ``` </div></details> ## Level 2 - Parameter -Parameters are placed inside a context. They are defined as associative array of parameter specifications. Each parameter is defined by it's internal input name, a definition in the form `'n' => array();`, where `n` is the name with which the bridge can access the parameter during execution. +Parameters are placed inside a context. +They are defined as associative array of parameter specifications. +Each parameter is defined by it's internal input name, a definition in the form `'n' => [];`, +where `n` is the name with which the bridge can access the parameter during execution. <details><summary>Show example</summary><div> ```PHP -const PARAMETERS = array( - 'My Context' => array( - 'n' => array() - ) -); +const PARAMETERS = [ + 'My Context' => [ + 'n' => [] + ] +]; ``` </div></details><br> @@ -351,7 +354,7 @@ Elements collected by this function must be stored in `$this->items`. The `items ```PHP -$item = array(); // Create a new item +$item = []; // Create a new item $item['title'] = 'Hello World!'; @@ -448,7 +451,7 @@ public function detectParameters($url){ && preg_match($regex, $url, $urlMatches) > 0 && preg_match($regex, static::URI, $bridgeUriMatches) > 0 && $urlMatches[3] === $bridgeUriMatches[3]) { - return array(); + return []; } else { return null; } diff --git a/docs/05_Bridge_API/03_FeedExpander.md b/docs/05_Bridge_API/03_FeedExpander.md index 7e72670a..910d1abb 100644 --- a/docs/05_Bridge_API/03_FeedExpander.md +++ b/docs/05_Bridge_API/03_FeedExpander.md @@ -93,10 +93,11 @@ class MySiteBridge extends FeedExpander { const NAME = 'Unnamed bridge'; const URI = ''; const DESCRIPTION = 'No description provided'; - const PARAMETERS = array(); + const PARAMETERS = []; const CACHE_TIMEOUT = 3600; - public function collectData(){ + public function collectData() + { $this->collectExpandableDatas('your feed URI'); } } diff --git a/docs/05_Bridge_API/index.md b/docs/05_Bridge_API/index.md index 6115fa01..e49e47be 100644 --- a/docs/05_Bridge_API/index.md +++ b/docs/05_Bridge_API/index.md @@ -1,4 +1,8 @@ -A _Bridge_ is an class that allows **RSS-Bridge** to create an RSS-feed from a website. A _Bridge_ represents one element on the [Welcome screen](../01_General/04_Screenshots.md) and covers one or more sites to return feeds for. It is developed in a PHP file located in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)) and extends one of the base classes of **RSS-Bridge**: +A _Bridge_ is a class that allows **RSS-Bridge** to create an RSS-feed from a website. +A _Bridge_ represents one element on the [Welcome screen](../01_General/04_Screenshots.md) +and covers one or more sites to return feeds for. +It is developed in a PHP file located in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)) +and extends one of the base classes of **RSS-Bridge**: Base class | Description -----------|------------ diff --git a/docs/07_Cache_API/index.md b/docs/07_Cache_API/index.md index 57692847..17afbacc 100644 --- a/docs/07_Cache_API/index.md +++ b/docs/07_Cache_API/index.md @@ -1,4 +1,6 @@ -A _Cache_ is a class that allows **RSS-Bridge** to store fetched data in a local storage area on the server. Cache imlementations are placed in the `caches/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). A cache must implement the [`CacheInterface`](../07_Cache_API/02_CacheInterface.md) interface. - -For more information about how to create a new `Cache`, read [How to create a new cache?](../07_Cache_API/01_How_to_create_a_new_cache.md) +A _Cache_ is a class that allows **RSS-Bridge** to store fetched data in a local storage area on the server. +Cache imlementations are placed in the `caches/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). +A cache must implement the [`CacheInterface`](../07_Cache_API/02_CacheInterface.md) interface. +For more information about how to create a new `Cache`, read +[How to create a new cache?](../07_Cache_API/01_How_to_create_a_new_cache.md) diff --git a/docs/09_Technical_recommendations/index.md b/docs/09_Technical_recommendations/index.md index f2b15476..a57f0bbd 100644 --- a/docs/09_Technical_recommendations/index.md +++ b/docs/09_Technical_recommendations/index.md @@ -15,7 +15,7 @@ class TestBridge extends BridgeAbstract { const URI = ''; const DESCRIPTION = 'No description provided'; const MAINTAINER = 'No maintainer'; - const PARAMETERS = array(); + const PARAMETERS = []; const CACHE_TIMEOUT = 3600; public function collectData(){ diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index a69552fc..f51fe893 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -1,76 +1,15 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -abstract class BridgeAbstract implements BridgeInterface +abstract class BridgeAbstract { - /** - * Name of the bridge - * - * Use {@see BridgeAbstract::getName()} to read this parameter - */ const NAME = 'Unnamed bridge'; - - /** - * URI to the site the bridge is intended to be used for. - * - * Use {@see BridgeAbstract::getURI()} to read this parameter - */ const URI = ''; - - /** - * Donation URI to the site the bridge is intended to be used for. - * - * Use {@see BridgeAbstract::getDonationURI()} to read this parameter - */ const DONATION_URI = ''; - - /** - * A brief description of what the bridge can do - * - * Use {@see BridgeAbstract::getDescription()} to read this parameter - */ const DESCRIPTION = 'No description provided'; - - /** - * The name of the maintainer. Multiple maintainers can be separated by comma - * - * Use {@see BridgeAbstract::getMaintainer()} to read this parameter - */ const MAINTAINER = 'No maintainer'; - - /** - * The default cache timeout for the bridge - * - * Use {@see BridgeAbstract::getCacheTimeout()} to read this parameter - */ const CACHE_TIMEOUT = 3600; - - /** - * Configuration for the bridge - */ const CONFIGURATION = []; - - /** - * Parameters for the bridge - * - * Use {@see BridgeAbstract::getParameters()} to read this parameter - */ const PARAMETERS = []; - - /** - * Test cases for detectParameters for the bridge - */ const TEST_DETECT_PARAMETERS = []; /** @@ -83,49 +22,67 @@ abstract class BridgeAbstract implements BridgeInterface 'title' => 'Maximum number of items to return', ]; - /** - * Holds the list of items collected by the bridge - * - * Items must be collected by {@see BridgeInterface::collectData()} - * - * Use {@see BridgeAbstract::getItems()} to access items. - * - * @var array - */ protected array $items = []; - - /** - * Holds the list of input parameters used by the bridge - * - * Do not access this parameter directly! - * Use {@see BridgeAbstract::setInputs()} and {@see BridgeAbstract::getInput()} instead! - * - * @var array - */ protected array $inputs = []; - - /** - * Holds the name of the queried context - * - * @var string - */ - protected $queriedContext = ''; - - /** - * Holds the list of bridge-specific configurations from config.ini.php, used by the bridge. - */ + protected string $queriedContext = ''; private array $configuration = []; public function __construct() { } - /** {@inheritdoc} */ + abstract public function collectData(); + public function getItems() { return $this->items; } + public function getOption(string $name) + { + return $this->configuration[$name] ?? null; + } + + public function getDescription() + { + return static::DESCRIPTION; + } + + public function getMaintainer(): string + { + return static::MAINTAINER; + } + + public function getName() + { + return static::NAME; + } + + public function getIcon() + { + return static::URI . '/favicon.ico'; + } + + public function getParameters(): array + { + return static::PARAMETERS; + } + + public function getURI() + { + return static::URI; + } + + public function getDonationURI(): string + { + return static::DONATION_URI; + } + + public function getCacheTimeout() + { + return static::CACHE_TIMEOUT; + } + /** * Sets the input values for a given context. * @@ -299,10 +256,7 @@ abstract class BridgeAbstract implements BridgeInterface */ protected function getInput($input) { - if (!isset($this->inputs[$this->queriedContext][$input]['value'])) { - return null; - } - return $this->inputs[$this->queriedContext][$input]['value']; + return $this->inputs[$this->queriedContext][$input]['value'] ?? null; } /** @@ -340,63 +294,6 @@ abstract class BridgeAbstract implements BridgeInterface } } - /** - * Get bridge configuration value - */ - public function getOption($name) - { - return $this->configuration[$name] ?? null; - } - - /** {@inheritdoc} */ - public function getDescription() - { - return static::DESCRIPTION; - } - - /** {@inheritdoc} */ - public function getMaintainer() - { - return static::MAINTAINER; - } - - /** {@inheritdoc} */ - public function getName() - { - return static::NAME; - } - - /** {@inheritdoc} */ - public function getIcon() - { - return static::URI . '/favicon.ico'; - } - - /** {@inheritdoc} */ - public function getParameters() - { - return static::PARAMETERS; - } - - /** {@inheritdoc} */ - public function getURI() - { - return static::URI; - } - - /** {@inheritdoc} */ - public function getDonationURI() - { - return static::DONATION_URI; - } - - /** {@inheritdoc} */ - public function getCacheTimeout() - { - return static::CACHE_TIMEOUT; - } - - /** {@inheritdoc} */ public function detectParameters($url) { $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/'; @@ -411,11 +308,6 @@ abstract class BridgeAbstract implements BridgeInterface return null; } - /** - * Loads a cached value for the specified key - * - * @return mixed Cached value or null if the key doesn't exist or has expired - */ protected function loadCacheValue(string $key) { $cache = RssBridge::getCache(); @@ -423,11 +315,6 @@ abstract class BridgeAbstract implements BridgeInterface return $cache->get($cacheKey); } - /** - * Stores a value to cache with the specified key - * - * @param mixed $value Value to cache - */ protected function saveCacheValue(string $key, $value, $ttl = 86400) { $cache = RssBridge::getCache(); diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index 6eef3879..99c44fff 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -25,7 +25,7 @@ final class BridgeCard /** * Gets a single bridge card * - * @param class-string<BridgeInterface> $bridgeClassName The bridge name + * @param class-string<BridgeAbstract> $bridgeClassName The bridge name * @param array $formats A list of formats * @param bool $isActive Indicates if the bridge is active or not * @return string The bridge card @@ -116,7 +116,7 @@ CARD; /** * Get the form header for a bridge card * - * @param class-string<BridgeInterface> $bridgeClassName The bridge name + * @param class-string<BridgeAbstract> $bridgeClassName The bridge name * @param bool $isHttps If disabled, adds a warning to the form * @return string The form header */ @@ -143,7 +143,7 @@ This bridge is not fetching its content through a secure connection</div>'; /** * Get the form body for a bridge * - * @param class-string<BridgeInterface> $bridgeClassName The bridge name + * @param class-string<BridgeAbstract> $bridgeClassName The bridge name * @param array $formats A list of supported formats * @param bool $isActive Indicates if a bridge is enabled or not * @param bool $isHttps Indicates if a bridge uses HTTPS or not diff --git a/lib/BridgeFactory.php b/lib/BridgeFactory.php index f302a27a..12565d92 100644 --- a/lib/BridgeFactory.php +++ b/lib/BridgeFactory.php @@ -34,7 +34,7 @@ final class BridgeFactory } } - public function create(string $name): BridgeInterface + public function create(string $name): BridgeAbstract { return new $name(); } diff --git a/lib/BridgeInterface.php b/lib/BridgeInterface.php deleted file mode 100644 index 63bc7b70..00000000 --- a/lib/BridgeInterface.php +++ /dev/null @@ -1,145 +0,0 @@ -<?php - -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * The bridge interface - * - * A bridge is a class that is responsible for collecting and transforming data - * from one hosting provider into an internal representation of feed data, that - * can later be transformed into different feed formats (see {@see FormatInterface}). - * - * For this purpose, all bridges need to perform three common operations: - * - * 1. Collect data from a remote site. - * 2. Extract the required contents. - * 3. Add the contents to the internal data structure. - * - * Bridges can optionally specify parameters to customize bridge behavior based - * on user input. For example, a user could specify how many items to return in - * the feed and where to get them. - * - * In order to present a bridge on the home page, and for the purpose of bridge - * specific behaviour, additional information must be provided by the bridge: - * - * * **Name** - * The name of the bridge that can be displayed to users. - * - * * **Description** - * A brief description for the bridge that can be displayed to users. - * - * * **URI** - * A link to the hosting provider. - * - * * **Maintainer** - * The GitHub username of the bridge maintainer - * - * * **Parameters** - * A list of parameters for customization - * - * * **Icon** - * A link to the favicon of the hosting provider - * - * * **Cache timeout** - * The default cache timeout for the bridge. - */ -interface BridgeInterface -{ - /** - * Collects data from the site - * - * @return void - */ - public function collectData(); - - /** - * Returns the value for the selected configuration - * - * @param string $input The option name - * @return mixed|null The option value or null if the input is not defined - */ - public function getOption($name); - - /** - * Returns the description - * - * @return string Description - */ - public function getDescription(); - - /** - * Returns an array of collected items - * - * @return array Associative array of items - */ - public function getItems(); - - /** - * Returns the bridge maintainer - * - * @return string Bridge maintainer - */ - public function getMaintainer(); - - /** - * Returns the bridge name - * - * @return string Bridge name - */ - public function getName(); - - /** - * Returns the bridge icon - * - * @return string Bridge icon - */ - public function getIcon(); - - /** - * Returns the bridge parameters - * - * @return array Bridge parameters - */ - public function getParameters(); - - /** - * Returns the bridge URI - * - * @return string Bridge URI - */ - public function getURI(); - - /** - * Returns the bridge Donation URI - * - * @return string Bridge Donation URI - */ - public function getDonationURI(); - - /** - * Returns the cache timeout - * - * @return int Cache timeout - */ - public function getCacheTimeout(); - - /** - * Returns parameters from given URL or null if URL is not applicable - * - * @param string $url URL to extract parameters from - * @return array|null List of bridge parameters or null if detection failed. - */ - public function detectParameters($url); - - public function getShortName(): string; -} diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index be467336..af06cc16 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -74,7 +74,7 @@ abstract class FeedExpander extends BridgeAbstract /** * Collects data from an existing feed. * - * Children should call this function in {@see BridgeInterface::collectData()} + * Children should call this function in {@see BridgeAbstract::collectData()} * to extract a feed. * * @param string $url URL to the feed. diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 1c6ce464..32dad269 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -15,6 +15,9 @@ final class RssBridge } Configuration::loadConfiguration($customConfig, getenv()); + // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); + date_default_timezone_set(Configuration::getConfig('system', 'timezone')); + set_exception_handler(function (\Throwable $e) { Logger::error('Uncaught Exception', ['e' => $e]); http_response_code(500); @@ -57,9 +60,6 @@ final class RssBridge } }); - // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); - date_default_timezone_set(Configuration::getConfig('system', 'timezone')); - self::$httpClient = new CurlHttpClient(); $cacheFactory = new CacheFactory(); @@ -68,11 +68,6 @@ final class RssBridge } else { self::$cache = $cacheFactory->create(); } - - if (Configuration::getConfig('authentication', 'enable')) { - $authenticationMiddleware = new AuthenticationMiddleware(); - $authenticationMiddleware(); - } } public function main(array $argv = []): void @@ -81,6 +76,10 @@ final class RssBridge parse_str(implode('&', array_slice($argv, 1)), $cliArgs); $request = $cliArgs; } else { + if (Configuration::getConfig('authentication', 'enable')) { + $authenticationMiddleware = new AuthenticationMiddleware(); + $authenticationMiddleware(); + } $request = array_merge($_GET, $_POST); } @@ -124,10 +123,4 @@ final class RssBridge { return self::$cache ?? new NullCache(); } - - public function clearCache() - { - $cache = self::getCache(); - $cache->clear(); - } } diff --git a/lib/contents.php b/lib/contents.php index c1847758..e173b542 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -16,6 +16,13 @@ function getContents( ) { $httpClient = RssBridge::getHttpClient(); + $httpHeadersNormalized = []; + foreach ($httpHeaders as $httpHeader) { + $parts = explode(':', $httpHeader); + $headerName = trim($parts[0]); + $headerValue = trim(implode(':', array_slice($parts, 1))); + $httpHeadersNormalized[$headerName] = $headerValue; + } // Snagged from https://github.com/lwthiker/curl-impersonate/blob/main/firefox/curl_ff102 $defaultHttpHeaders = [ 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', @@ -27,13 +34,6 @@ function getContents( 'Sec-Fetch-User' => '?1', 'TE' => 'trailers', ]; - $httpHeadersNormalized = []; - foreach ($httpHeaders as $httpHeader) { - $parts = explode(':', $httpHeader); - $headerName = trim($parts[0]); - $headerValue = trim(implode(':', array_slice($parts, 1))); - $httpHeadersNormalized[$headerName] = $headerValue; - } $config = [ 'useragent' => Configuration::getConfig('http', 'useragent'), 'timeout' => Configuration::getConfig('http', 'timeout'), @@ -43,7 +43,7 @@ function getContents( $maxFileSize = Configuration::getConfig('http', 'max_filesize'); if ($maxFileSize) { - // Multiply with 2^20 (1M) to the value in bytes + // Convert from MB to B by multiplying with 2^20 (1M) $config['max_filesize'] = $maxFileSize * 2 ** 20; } @@ -57,7 +57,6 @@ function getContents( /** @var Response $cachedResponse */ $cachedResponse = $cache->get($cacheKey); if ($cachedResponse) { - // considering popping $cachedLastModified = $cachedResponse->getHeader('last-modified'); if ($cachedLastModified) { $cachedLastModified = new \DateTimeImmutable($cachedLastModified); @@ -101,21 +100,13 @@ function getContents( Debug::isEnabled() ? mb_substr($response->getBody(), 0, 500) : '', ); - // The following code must be extracted if it grows too much - $cloudflareTitles = [ - '<title>Just a moment...', - '<title>Please Wait...', - '<title>Attention Required!', - '<title>Security | Glassdoor', - ]; - foreach ($cloudflareTitles as $cloudflareTitle) { - if (str_contains($response->getBody(), $cloudflareTitle)) { - throw new CloudFlareException($exceptionMessage, $response->getCode()); - } + if (CloudFlareException::isCloudFlareResponse($response)) { + throw new CloudFlareException($exceptionMessage, $response->getCode()); } throw new HttpException(trim($exceptionMessage), $response->getCode()); } if ($returnFull === true) { + // todo: return the actual response object return [ 'code' => $response->getCode(), 'headers' => $response->getHeaders(), diff --git a/lib/http.php b/lib/http.php index c5e65e77..cc1d0e22 100644 --- a/lib/http.php +++ b/lib/http.php @@ -6,6 +6,21 @@ class HttpException extends \Exception final class CloudFlareException extends HttpException { + public static function isCloudFlareResponse(Response $response): bool + { + $cloudflareTitles = [ + '<title>Just a moment...', + '<title>Please Wait...', + '<title>Attention Required!', + '<title>Security | Glassdoor', + ]; + foreach ($cloudflareTitles as $cloudflareTitle) { + if (str_contains($response->getBody(), $cloudflareTitle)) { + return true; + } + } + return false; + } } interface HttpClient @@ -119,7 +134,7 @@ final class CurlHttpClient implements HttpClient } } - $statusCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + $statusCode = curl_getinfo($ch, CURLINFO_RESPONSE_CODE); curl_close($ch); return new Response($data, $statusCode, $responseHeaders); } diff --git a/tests/Bridges/BridgeImplementationTest.php b/tests/Bridges/BridgeImplementationTest.php index cf03f356..807649fc 100644 --- a/tests/Bridges/BridgeImplementationTest.php +++ b/tests/Bridges/BridgeImplementationTest.php @@ -3,7 +3,6 @@ namespace RssBridge\Tests\Bridges; use BridgeAbstract; -use BridgeInterface; use FeedExpander; use PHPUnit\Framework\TestCase; @@ -29,7 +28,7 @@ class BridgeImplementationTest extends TestCase public function testClassType($path) { $this->setBridge($path); - $this->assertInstanceOf(BridgeInterface::class, $this->obj); + $this->assertInstanceOf(BridgeAbstract::class, $this->obj); } /** From 0bf38e5c56a795301d540f4b0ce6e9bd9935b058 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 20 Sep 2023 03:15:15 +0200 Subject: [PATCH 112/716] fix: small notice errors (#3677) * fix notice * fix notice * tweak * tweaks --- bridges/CodebergBridge.php | 7 +++++- bridges/TelegramBridge.php | 7 ++++-- bridges/ThePirateBayBridge.php | 1 + bridges/TwitchBridge.php | 41 ++++++++++++++-------------------- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/bridges/CodebergBridge.php b/bridges/CodebergBridge.php index 83775885..c13ff004 100644 --- a/bridges/CodebergBridge.php +++ b/bridges/CodebergBridge.php @@ -181,7 +181,12 @@ class CodebergBridge extends BridgeAbstract $item['title'] = $message->find('span.message-wrapper', 0)->plaintext; $item['uri'] = $tr->find('td.sha', 0)->find('a', 0)->href; $item['author'] = $tr->find('td.author', 0)->plaintext; - $item['timestamp'] = $tr->find('td', 3)->find('span', 0)->title; + + $var = $tr->find('td', 3); + $var1 = $var->find('span', 0); + if ($var1) { + $item['timestamp'] = $var1->title; + } if ($message->find('pre.commit-body', 0)) { $message->find('pre.commit-body', 0)->style = ''; diff --git a/bridges/TelegramBridge.php b/bridges/TelegramBridge.php index 9d73e06e..a3c910e8 100644 --- a/bridges/TelegramBridge.php +++ b/bridges/TelegramBridge.php @@ -55,8 +55,11 @@ class TelegramBridge extends BridgeAbstract $item['title'] = $this->itemTitle; $item['timestamp'] = $messageDiv->find('span.tgme_widget_message_meta', 0)->find('time', 0)->datetime; $item['enclosures'] = $this->enclosures; - $author = trim($messageDiv->find('a.tgme_widget_message_owner_name', 0)->plaintext); - $item['author'] = html_entity_decode($author, ENT_QUOTES); + + $messageOwner = $messageDiv->find('a.tgme_widget_message_owner_name', 0); + if ($messageOwner) { + $item['author'] = html_entity_decode(trim($messageOwner->plaintext), ENT_QUOTES); + } $this->items[] = $item; } diff --git a/bridges/ThePirateBayBridge.php b/bridges/ThePirateBayBridge.php index 5b305c82..27732db5 100644 --- a/bridges/ThePirateBayBridge.php +++ b/bridges/ThePirateBayBridge.php @@ -65,6 +65,7 @@ class ThePirateBayBridge extends BridgeAbstract '207' => 'HD Movies', '208' => 'HD TV-Shows', '209' => '3D', + '211' => 'UHD/4k Movies', '212' => 'UHD/4k TV-Shows', '299' => 'Other', '301' => 'Windows', diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index 146fed3d..e4abaa60 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -90,7 +90,8 @@ EOD; 'channel' => $channel, 'types' => self::BROADCAST_TYPES[$type] ]; - $data = $this->apiRequest($query, $variables); + $response = $this->apiRequest($query, $variables); + $data = $response->data; if ($data->user === null) { throw new \Exception(sprintf('Unable to find channel `%s`', $channel)); } @@ -205,37 +206,29 @@ EOD; ); } - // GraphQL: https://graphql.org/ - // Tool for developing/testing queries: https://github.com/skevy/graphiql-app + /** + * GraphQL: https://graphql.org/ + * Tool for developing/testing queries: https://github.com/skevy/graphiql-app + * + * Official instructions for obtaining your own client ID can be found here: + * https://dev.twitch.tv/docs/v5/#getting-a-client-id + */ private function apiRequest($query, $variables) { $request = [ - 'query' => $query, - 'variables' => $variables + 'query' => $query, + 'variables' => $variables, ]; - /** - * Official instructions for obtaining your own client ID can be found here: - * https://dev.twitch.tv/docs/v5/#getting-a-client-id - */ - $header = [ - 'Client-ID: kimne78kx3ncx6brgo4mv6wki5h1ko' + $headers = [ + 'Client-ID: kimne78kx3ncx6brgo4mv6wki5h1ko', ]; $opts = [ CURLOPT_CUSTOMREQUEST => 'POST', - CURLOPT_POSTFIELDS => json_encode($request) + CURLOPT_POSTFIELDS => json_encode($request), ]; - - Logger::debug("Sending GraphQL query:\n" . $query); - Logger::debug("Sending GraphQL variables:\n" . json_encode($variables, JSON_PRETTY_PRINT)); - $response = json_decode(getContents('https://gql.twitch.tv/gql', $header, $opts)); - Logger::debug("Got GraphQL response:\n" . json_encode($response, JSON_PRETTY_PRINT)); - - if (isset($response->errors)) { - $messages = array_column($response->errors, 'message'); - throw new \Exception(sprintf('twitch api: `%s`', implode("\n", $messages))); - } - - return $response->data; + $json = getContents('https://gql.twitch.tv/gql', $headers, $opts); + $result = Json::decode($json, false); + return $result; } public function getName() From 360f953be82b7340bd153991bdc87f699db598a4 Mon Sep 17 00:00:00 2001 From: Julien Papasian <papjul@users.noreply.github.com> Date: Thu, 21 Sep 2023 18:55:53 +0200 Subject: [PATCH 113/716] Fix #3674 - [Nautiljon] Remove requirement of cURL NSS (#3679) Looks like it works with OpenSSL now. --- bridges/NautiljonBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/NautiljonBridge.php b/bridges/NautiljonBridge.php index 611ffc2f..a689e5f5 100644 --- a/bridges/NautiljonBridge.php +++ b/bridges/NautiljonBridge.php @@ -4,7 +4,7 @@ class NautiljonBridge extends BridgeAbstract { const NAME = 'Nautiljon Bridge'; const URI = 'https://www.nautiljon.com'; - const DESCRIPTION = 'Actualités et Brèves de Nautiljon. Nécessite une version NSS de cURL pour fonctionner.'; + const DESCRIPTION = 'Actualités et Brèves de Nautiljon.'; const MAINTAINER = 'papjul'; const PARAMETERS = [ From 7329b83cc0fe1a5f707f864b1f3d62efd4be2172 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 21 Sep 2023 22:05:55 +0200 Subject: [PATCH 114/716] refactor: logger (#3678) --- actions/DisplayAction.php | 14 +- actions/SetBridgeCacheAction.php | 12 +- bridges/EZTVBridge.php | 1 - bridges/ElloBridge.php | 5 +- bridges/FeedMergeBridge.php | 4 +- bridges/ImgsedBridge.php | 2 +- bridges/InstagramBridge.php | 5 +- bridges/RedditBridge.php | 6 - bridges/SoundcloudBridge.php | 3 - bridges/SpotifyBridge.php | 5 +- bridges/TwitterBridge.php | 3 +- bridges/YoutubeBridge.php | 8 +- bridges/ZDNetBridge.php | 4 +- caches/FileCache.php | 10 +- caches/MemcachedCache.php | 11 +- caches/SQLiteCache.php | 13 +- lib/BridgeAbstract.php | 17 +- lib/BridgeFactory.php | 9 +- lib/CacheFactory.php | 14 +- lib/Debug.php | 4 +- lib/FeedExpander.php | 2 +- lib/Logger.php | 97 ------------ lib/RssBridge.php | 36 +++-- lib/bootstrap.php | 1 + lib/logger.php | 172 +++++++++++++++++++++ tests/Actions/ActionImplementationTest.php | 69 --------- tests/Actions/ListActionTest.php | 76 --------- tests/BridgeFactoryTest.php | 19 +-- tests/Bridges/BridgeImplementationTest.php | 5 +- tests/CacheTest.php | 8 +- 30 files changed, 297 insertions(+), 338 deletions(-) delete mode 100644 lib/Logger.php create mode 100644 lib/logger.php delete mode 100644 tests/Actions/ActionImplementationTest.php delete mode 100644 tests/Actions/ListActionTest.php diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index efa4d3b5..87b040c2 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -3,13 +3,19 @@ class DisplayAction implements ActionInterface { private CacheInterface $cache; + private Logger $logger; + + public function __construct() + { + $this->cache = RssBridge::getCache(); + $this->logger = RssBridge::getLogger(); + } public function execute(array $request) { if (Configuration::getConfig('system', 'enable_maintenance_mode')) { return new Response('503 Service Unavailable', 503); } - $this->cache = RssBridge::getCache(); $cacheKey = 'http_' . json_encode($request); /** @var Response $cachedResponse */ $cachedResponse = $this->cache->get($cacheKey); @@ -113,15 +119,15 @@ class DisplayAction implements ActionInterface if ($e instanceof HttpException) { // Reproduce (and log) these responses regardless of error output and report limit if ($e->getCode() === 429) { - Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); + $this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); return new Response('429 Too Many Requests', 429); } if ($e->getCode() === 503) { - Logger::info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); + $this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); return new Response('503 Service Unavailable', 503); } } - Logger::error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]); + $this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]); $errorOutput = Configuration::getConfig('error', 'output'); $reportLimit = Configuration::getConfig('error', 'report_limit'); $errorCount = 1; diff --git a/actions/SetBridgeCacheAction.php b/actions/SetBridgeCacheAction.php index a8e712d4..c9264a27 100644 --- a/actions/SetBridgeCacheAction.php +++ b/actions/SetBridgeCacheAction.php @@ -14,6 +14,13 @@ class SetBridgeCacheAction implements ActionInterface { + private CacheInterface $cache; + + public function __construct() + { + $this->cache = RssBridge::getCache(); + } + public function execute(array $request) { $authenticationMiddleware = new ApiAuthenticationMiddleware(); @@ -35,18 +42,15 @@ class SetBridgeCacheAction implements ActionInterface // whitelist control if (!$bridgeFactory->isEnabled($bridgeClassName)) { throw new \Exception('This bridge is not whitelisted', 401); - die; } $bridge = $bridgeFactory->create($bridgeClassName); $bridge->loadConfiguration(); $value = $request['value']; - $cache = RssBridge::getCache(); - $cacheKey = get_class($bridge) . '_' . $key; $ttl = 86400 * 3; - $cache->set($cacheKey, $value, $ttl); + $this->cache->set($cacheKey, $value, $ttl); header('Content-Type: text/plain'); echo 'done'; diff --git a/bridges/EZTVBridge.php b/bridges/EZTVBridge.php index a2db3ead..73318f0c 100644 --- a/bridges/EZTVBridge.php +++ b/bridges/EZTVBridge.php @@ -48,7 +48,6 @@ class EZTVBridge extends BridgeAbstract public function collectData() { $eztv_uri = $this->getEztvUri(); - Logger::debug($eztv_uri); $ids = explode(',', trim($this->getInput('ids'))); foreach ($ids as $id) { $data = json_decode(getContents(sprintf('%s/api/get-torrents?imdb_id=%s', $eztv_uri, $id))); diff --git a/bridges/ElloBridge.php b/bridges/ElloBridge.php index 9017bc11..42c88a06 100644 --- a/bridges/ElloBridge.php +++ b/bridges/ElloBridge.php @@ -113,15 +113,14 @@ class ElloBridge extends BridgeAbstract private function getAPIKey() { - $cache = RssBridge::getCache(); $cacheKey = 'ElloBridge_key'; - $apiKey = $cache->get($cacheKey); + $apiKey = $this->cache->get($cacheKey); if (!$apiKey) { $keyInfo = getContents(self::URI . 'api/webapp-token') or returnServerError('Unable to get token.'); $apiKey = json_decode($keyInfo)->token->access_token; $ttl = 60 * 60 * 20; - $cache->set($cacheKey, $apiKey, $ttl); + $this->cache->set($cacheKey, $apiKey, $ttl); } return $apiKey; diff --git a/bridges/FeedMergeBridge.php b/bridges/FeedMergeBridge.php index cf1b10a2..f2c1d9d5 100644 --- a/bridges/FeedMergeBridge.php +++ b/bridges/FeedMergeBridge.php @@ -63,7 +63,7 @@ TEXT; try { $this->collectExpandableDatas($feed); } catch (HttpException $e) { - Logger::warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e))); + $this->logger->warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e))); $this->items[] = [ 'title' => 'RSS-Bridge: ' . $e->getMessage(), // Give current time so it sorts to the top @@ -73,7 +73,7 @@ TEXT; } catch (\Exception $e) { if (str_starts_with($e->getMessage(), 'Unable to parse xml')) { // Allow this particular exception from FeedExpander - Logger::warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e))); + $this->logger->warning(sprintf('Exception in FeedMergeBridge: %s', create_sane_exception_message($e))); continue; } throw $e; diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index cf17acb4..70b79866 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -217,7 +217,7 @@ HTML, if ($relativeDate) { date_sub($date, $relativeDate); } else { - Logger::info(sprintf('Unable to parse date string: %s', $dateString)); + $this->logger->info(sprintf('Unable to parse date string: %s', $dateString)); } return date_format($date, 'r'); } diff --git a/bridges/InstagramBridge.php b/bridges/InstagramBridge.php index 9a846fb1..1714a691 100644 --- a/bridges/InstagramBridge.php +++ b/bridges/InstagramBridge.php @@ -98,9 +98,8 @@ class InstagramBridge extends BridgeAbstract return $username; } - $cache = RssBridge::getCache(); $cacheKey = 'InstagramBridge_' . $username; - $pk = $cache->get($cacheKey); + $pk = $this->cache->get($cacheKey); if (!$pk) { $data = $this->getContents(self::URI . 'web/search/topsearch/?query=' . $username); @@ -112,7 +111,7 @@ class InstagramBridge extends BridgeAbstract if (!$pk) { returnServerError('Unable to find username in search result.'); } - $cache->set($cacheKey, $pk); + $this->cache->set($cacheKey, $pk); } return $pk; } diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 5f6070e3..8d46f7bd 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -72,12 +72,6 @@ class RedditBridge extends BridgeAbstract ] ] ]; - private CacheInterface $cache; - - public function __construct() - { - $this->cache = RssBridge::getCache(); - } public function collectData() { diff --git a/bridges/SoundcloudBridge.php b/bridges/SoundcloudBridge.php index 5664761b..e389c965 100644 --- a/bridges/SoundcloudBridge.php +++ b/bridges/SoundcloudBridge.php @@ -36,15 +36,12 @@ class SoundCloudBridge extends BridgeAbstract private $feedTitle = null; private $feedIcon = null; - private CacheInterface $cache; private $clientIdRegex = '/client_id.*?"(.+?)"/'; private $widgetRegex = '/widget-.+?\.js/'; public function collectData() { - $this->cache = RssBridge::getCache(); - $res = $this->getUser($this->getInput('u')); $this->feedTitle = $res->username; diff --git a/bridges/SpotifyBridge.php b/bridges/SpotifyBridge.php index eb847f3d..c02acd25 100644 --- a/bridges/SpotifyBridge.php +++ b/bridges/SpotifyBridge.php @@ -278,10 +278,9 @@ class SpotifyBridge extends BridgeAbstract private function fetchAccessToken() { - $cache = RssBridge::getCache(); $cacheKey = sprintf('SpotifyBridge:%s:%s', $this->getInput('clientid'), $this->getInput('clientsecret')); - $token = $cache->get($cacheKey); + $token = $this->cache->get($cacheKey); if ($token) { $this->token = $token; } else { @@ -294,7 +293,7 @@ class SpotifyBridge extends BridgeAbstract $data = Json::decode($json); $this->token = $data['access_token']; - $cache->set($cacheKey, $this->token, 3600); + $this->cache->set($cacheKey, $this->token, 3600); } } diff --git a/bridges/TwitterBridge.php b/bridges/TwitterBridge.php index b9586150..93301038 100644 --- a/bridges/TwitterBridge.php +++ b/bridges/TwitterBridge.php @@ -234,8 +234,7 @@ EOD $tweets = []; // Get authentication information - $cache = RssBridge::getCache(); - $api = new TwitterClient($cache); + $api = new TwitterClient($this->cache); // Try to get all tweets switch ($this->queriedContext) { case 'By username': diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index bff62a90..b544f762 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -77,12 +77,6 @@ class YoutubeBridge extends BridgeAbstract private $channel_name = ''; // This took from repo BetterVideoRss of VerifiedJoseph. const URI_REGEX = '/(https?:\/\/(?:www\.)?(?:[a-zA-Z0-9-.]{2,256}\.[a-z]{2,20})(\:[0-9]{2 ,4})?(?:\/[a-zA-Z0-9@:%_\+.,~#"\'!?&\/\/=\-*]+|\/)?)/ims'; //phpcs:ignore - private CacheInterface $cache; - - public function __construct() - { - $this->cache = RssBridge::getCache(); - } private function collectDataInternal() { @@ -368,7 +362,7 @@ class YoutubeBridge extends BridgeAbstract $scriptRegex = '/var ytInitialData = (.*?);<\/script>/'; $result = preg_match($scriptRegex, $html, $matches); if (! $result) { - Logger::debug('Could not find ytInitialData'); + $this->logger->debug('Could not find ytInitialData'); return null; } return json_decode($matches[1]); diff --git a/bridges/ZDNetBridge.php b/bridges/ZDNetBridge.php index 693f542c..00b272ce 100644 --- a/bridges/ZDNetBridge.php +++ b/bridges/ZDNetBridge.php @@ -180,13 +180,13 @@ class ZDNetBridge extends FeedExpander $article = getSimpleHTMLDOMCached($item['uri']); if (!$article) { - Logger::info('Unable to parse the dom from ' . $item['uri']); + $this->logger->info('Unable to parse the dom from ' . $item['uri']); return $item; } $articleTag = $article->find('article', 0) ?? $article->find('.c-articleContent', 0); if (!$articleTag) { - Logger::info('Unable to parse <article> tag in ' . $item['uri']); + $this->logger->info('Unable to parse <article> tag in ' . $item['uri']); return $item; } $contents = $articleTag->innertext; diff --git a/caches/FileCache.php b/caches/FileCache.php index 1495971a..703fb6db 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -4,10 +4,14 @@ declare(strict_types=1); class FileCache implements CacheInterface { + private Logger $logger; private array $config; - public function __construct(array $config = []) - { + public function __construct( + Logger $logger, + array $config = [] + ) { + $this->logger = $logger; $default = [ 'path' => null, 'enable_purge' => true, @@ -28,7 +32,7 @@ class FileCache implements CacheInterface } $item = unserialize(file_get_contents($cacheFile)); if ($item === false) { - Logger::warning(sprintf('Failed to unserialize: %s', $cacheFile)); + $this->logger->warning(sprintf('Failed to unserialize: %s', $cacheFile)); $this->delete($key); return $default; } diff --git a/caches/MemcachedCache.php b/caches/MemcachedCache.php index 78035435..f994c1ae 100644 --- a/caches/MemcachedCache.php +++ b/caches/MemcachedCache.php @@ -4,10 +4,15 @@ declare(strict_types=1); class MemcachedCache implements CacheInterface { + private Logger $logger; private \Memcached $conn; - public function __construct(string $host, int $port) - { + public function __construct( + Logger $logger, + string $host, + int $port + ) { + $this->logger = $logger; $this->conn = new \Memcached(); // This call does not actually connect to server yet if (!$this->conn->addServer($host, $port)) { @@ -29,7 +34,7 @@ class MemcachedCache implements CacheInterface $expiration = $ttl === null ? 0 : time() + $ttl; $result = $this->conn->set($key, $value, $expiration); if ($result === false) { - Logger::warning('Failed to store an item in memcached', [ + $this->logger->warning('Failed to store an item in memcached', [ 'key' => $key, 'code' => $this->conn->getLastErrorCode(), 'message' => $this->conn->getLastErrorMessage(), diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index 09689566..94f6e289 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -8,11 +8,15 @@ declare(strict_types=1); */ class SQLiteCache implements CacheInterface { - private \SQLite3 $db; + private Logger $logger; private array $config; + private \SQLite3 $db; - public function __construct(array $config) - { + public function __construct( + Logger $logger, + array $config + ) { + $this->logger = $logger; $default = [ 'file' => null, 'timeout' => 5000, @@ -59,7 +63,7 @@ class SQLiteCache implements CacheInterface $blob = $row['value']; $value = unserialize($blob); if ($value === false) { - Logger::error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100))); + $this->logger->error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100))); // delete? return $default; } @@ -68,6 +72,7 @@ class SQLiteCache implements CacheInterface // delete? return $default; } + public function set(string $key, $value, int $ttl = null): void { $cacheKey = $this->createCacheKey($key); diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index f51fe893..a3d84188 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -27,8 +27,15 @@ abstract class BridgeAbstract protected string $queriedContext = ''; private array $configuration = []; - public function __construct() - { + protected CacheInterface $cache; + protected Logger $logger; + + public function __construct( + CacheInterface $cache, + Logger $logger + ) { + $this->cache = $cache; + $this->logger = $logger; } abstract public function collectData(); @@ -310,16 +317,14 @@ abstract class BridgeAbstract protected function loadCacheValue(string $key) { - $cache = RssBridge::getCache(); $cacheKey = $this->getShortName() . '_' . $key; - return $cache->get($cacheKey); + return $this->cache->get($cacheKey); } protected function saveCacheValue(string $key, $value, $ttl = 86400) { - $cache = RssBridge::getCache(); $cacheKey = $this->getShortName() . '_' . $key; - $cache->set($cacheKey, $value, $ttl); + $this->cache->set($cacheKey, $value, $ttl); } public function getShortName(): string diff --git a/lib/BridgeFactory.php b/lib/BridgeFactory.php index 12565d92..c3da4bfe 100644 --- a/lib/BridgeFactory.php +++ b/lib/BridgeFactory.php @@ -2,12 +2,17 @@ final class BridgeFactory { + private CacheInterface $cache; + private Logger $logger; private $bridgeClassNames = []; private $enabledBridges = []; private $missingEnabledBridges = []; public function __construct() { + $this->cache = RssBridge::getCache(); + $this->logger = RssBridge::getLogger(); + // Create all possible bridge class names from fs foreach (scandir(__DIR__ . '/../bridges/') as $file) { if (preg_match('/^([^.]+Bridge)\.php$/U', $file, $m)) { @@ -29,14 +34,14 @@ final class BridgeFactory $this->enabledBridges[] = $bridgeClassName; } else { $this->missingEnabledBridges[] = $enabledBridge; - Logger::info(sprintf('Bridge not found: %s', $enabledBridge)); + $this->logger->info(sprintf('Bridge not found: %s', $enabledBridge)); } } } public function create(string $name): BridgeAbstract { - return new $name(); + return new $name($this->cache, $this->logger); } public function isEnabled(string $bridgeName): bool diff --git a/lib/CacheFactory.php b/lib/CacheFactory.php index 3f076d83..df78d9cb 100644 --- a/lib/CacheFactory.php +++ b/lib/CacheFactory.php @@ -4,6 +4,14 @@ declare(strict_types=1); class CacheFactory { + private Logger $logger; + + public function __construct( + Logger $logger + ) { + $this->logger = $logger; + } + public function create(string $name = null): CacheInterface { $name ??= Configuration::getConfig('cache', 'type'); @@ -49,7 +57,7 @@ class CacheFactory if (!is_writable($fileCacheConfig['path'])) { throw new \Exception(sprintf('The FileCache path is not writable: %s', $fileCacheConfig['path'])); } - return new FileCache($fileCacheConfig); + return new FileCache($this->logger, $fileCacheConfig); case SQLiteCache::class: if (!extension_loaded('sqlite3')) { throw new \Exception('"sqlite3" extension not loaded. Please check "php.ini"'); @@ -66,7 +74,7 @@ class CacheFactory } elseif (!is_dir(dirname($file))) { throw new \Exception(sprintf('Invalid configuration for %s', 'SQLiteCache')); } - return new SQLiteCache([ + return new SQLiteCache($this->logger, [ 'file' => $file, 'timeout' => Configuration::getConfig('SQLiteCache', 'timeout'), 'enable_purge' => Configuration::getConfig('SQLiteCache', 'enable_purge'), @@ -94,7 +102,7 @@ class CacheFactory if ($port < 1 || $port > 65535) { throw new \Exception('"port" param is invalid for ' . $section); } - return new MemcachedCache($host, $port); + return new MemcachedCache($this->logger, $host, $port); default: if (!file_exists(PATH_LIB_CACHES . $className . '.php')) { throw new \Exception('Unable to find the cache file'); diff --git a/lib/Debug.php b/lib/Debug.php index 48dbb31a..4333b3a5 100644 --- a/lib/Debug.php +++ b/lib/Debug.php @@ -24,6 +24,8 @@ class Debug array_pop($trace); $lastFrame = $trace[array_key_last($trace)]; $text = sprintf('%s(%s): %s', $lastFrame['file'], $lastFrame['line'], $message); - Logger::debug($text); + + $logger = RssBridge::getLogger(); + $logger->debug($text); } } diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index af06cc16..14c931e6 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -113,7 +113,7 @@ abstract class FeedExpander extends BridgeAbstract if ($rssContent === false) { $xmlErrors = libxml_get_errors(); foreach ($xmlErrors as $xmlError) { - Logger::debug(trim($xmlError->message)); + Debug::log(trim($xmlError->message)); } if ($xmlErrors) { // Render only the first error into exception message diff --git a/lib/Logger.php b/lib/Logger.php deleted file mode 100644 index 073fedee..00000000 --- a/lib/Logger.php +++ /dev/null @@ -1,97 +0,0 @@ -<?php - -declare(strict_types=1); - -final class Logger -{ - public static function debug(string $message, array $context = []) - { - self::log('DEBUG', $message, $context); - } - - public static function info(string $message, array $context = []): void - { - self::log('INFO', $message, $context); - } - - public static function warning(string $message, array $context = []): void - { - self::log('WARNING', $message, $context); - } - - public static function error(string $message, array $context = []): void - { - self::log('ERROR', $message, $context); - } - - private static function log(string $level, string $message, array $context = []): void - { - if (!Debug::isEnabled() && $level === 'DEBUG') { - // Don't log this debug log record because debug mode is disabled - return; - } - - if (isset($context['e'])) { - /** @var \Throwable $e */ - $e = $context['e']; - unset($context['e']); - $context['type'] = get_class($e); - $context['code'] = $e->getCode(); - $context['message'] = sanitize_root($e->getMessage()); - $context['file'] = sanitize_root($e->getFile()); - $context['line'] = $e->getLine(); - $context['url'] = get_current_url(); - $context['trace'] = trace_to_call_points(trace_from_exception($e)); - // Don't log these exceptions - // todo: this logic belongs in log handler - $ignoredExceptions = [ - 'You must specify a format', - 'Format name invalid', - 'Unknown format given', - 'Bridge name invalid', - 'Invalid action', - 'twitter: No results for this query', - // telegram - 'Unable to find channel. The channel is non-existing or non-public', - // fb - 'This group is not public! RSS-Bridge only supports public groups!', - 'You must be logged in to view this page', - 'Unable to get the page id. You should consider getting the ID by hand', - // tiktok 404 - 'https://www.tiktok.com/@', - ]; - foreach ($ignoredExceptions as $ignoredException) { - if (str_starts_with($e->getMessage(), $ignoredException)) { - return; - } - } - } - - if ($context) { - try { - $context = Json::encode($context); - } catch (\JsonException $e) { - $context['message'] = null; - $context = Json::encode($context); - } - } else { - $context = ''; - } - $text = sprintf( - "[%s] rssbridge.%s %s %s\n", - now()->format('Y-m-d H:i:s'), - $level, - // Intentionally not sanitizing $message - $message, - $context - ); - - // Log to stderr/stdout whatever that is - // todo: extract to log handler - error_log($text); - - // Log to file - // todo: extract to log handler - //$bytes = file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); - } -} diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 32dad269..0ec7174d 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -2,8 +2,9 @@ final class RssBridge { - private static HttpClient $httpClient; private static CacheInterface $cache; + private static Logger $logger; + private static HttpClient $httpClient; public function __construct() { @@ -19,7 +20,7 @@ final class RssBridge date_default_timezone_set(Configuration::getConfig('system', 'timezone')); set_exception_handler(function (\Throwable $e) { - Logger::error('Uncaught Exception', ['e' => $e]); + self::$logger->error('Uncaught Exception', ['e' => $e]); http_response_code(500); print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); exit(1); @@ -35,7 +36,7 @@ final class RssBridge sanitize_root($file), $line ); - Logger::warning($text); + self::$logger->warning($text); if (Debug::isEnabled()) { print sprintf("<pre>%s</pre>\n", e($text)); } @@ -52,17 +53,23 @@ final class RssBridge sanitize_root($error['file']), $error['line'] ); - Logger::error($message); + self::$logger->error($message); if (Debug::isEnabled()) { - // todo: extract to log handler print sprintf("<pre>%s</pre>\n", e($message)); } } }); + self::$logger = new SimpleLogger('rssbridge'); + if (Debug::isEnabled()) { + self::$logger->addHandler(new StreamHandler(Logger::DEBUG)); + } else { + self::$logger->addHandler(new StreamHandler(Logger::INFO)); + } + self::$httpClient = new CurlHttpClient(); - $cacheFactory = new CacheFactory(); + $cacheFactory = new CacheFactory(self::$logger); if (Debug::isEnabled()) { self::$cache = $cacheFactory->create('array'); } else { @@ -108,19 +115,24 @@ final class RssBridge $response->send(); } } catch (\Throwable $e) { - Logger::error('Exception in RssBridge::main()', ['e' => $e]); + self::$logger->error('Exception in RssBridge::main()', ['e' => $e]); http_response_code(500); print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); } } + public static function getCache(): CacheInterface + { + return self::$cache; + } + + public static function getLogger(): Logger + { + return self::$logger; + } + public static function getHttpClient(): HttpClient { return self::$httpClient; } - - public static function getCache(): CacheInterface - { - return self::$cache ?? new NullCache(); - } } diff --git a/lib/bootstrap.php b/lib/bootstrap.php index ca6cecdb..c8cf4e99 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -43,6 +43,7 @@ $files = [ __DIR__ . '/../lib/php8backports.php', __DIR__ . '/../lib/utils.php', __DIR__ . '/../lib/http.php', + __DIR__ . '/../lib/logger.php', // Vendor __DIR__ . '/../vendor/parsedown/Parsedown.php', __DIR__ . '/../vendor/php-urljoin/src/urljoin.php', diff --git a/lib/logger.php b/lib/logger.php new file mode 100644 index 00000000..ed1f1179 --- /dev/null +++ b/lib/logger.php @@ -0,0 +1,172 @@ +<?php + +declare(strict_types=1); + +interface Logger +{ + public const DEBUG = 10; + public const INFO = 20; + public const WARNING = 30; + public const ERROR = 40; + + public const LEVEL_NAMES = [ + self::DEBUG => 'DEBUG', + self::INFO => 'INFO', + self::WARNING => 'WARNING', + self::ERROR => 'ERROR', + ]; + + public function debug(string $message, array $context = []); + + public function info(string $message, array $context = []): void; + + public function warning(string $message, array $context = []): void; + + public function error(string $message, array $context = []): void; +} + +final class SimpleLogger implements Logger +{ + private string $name; + private array $handlers; + + /** + * @param callable[] $handlers + */ + public function __construct( + string $name, + array $handlers = [] + ) { + $this->name = $name; + $this->handlers = $handlers; + } + + public function addHandler(callable $fn) + { + $this->handlers[] = $fn; + } + + public function debug(string $message, array $context = []) + { + $this->log(self::DEBUG, $message, $context); + } + + public function info(string $message, array $context = []): void + { + $this->log(self::INFO, $message, $context); + } + + public function warning(string $message, array $context = []): void + { + $this->log(self::WARNING, $message, $context); + } + + public function error(string $message, array $context = []): void + { + $this->log(self::ERROR, $message, $context); + } + + private function log(int $level, string $message, array $context = []): void + { + foreach ($this->handlers as $handler) { + $handler([ + 'name' => $this->name, + 'created_at' => now(), + 'level' => $level, + 'level_name' => self::LEVEL_NAMES[$level], + 'message' => $message, + 'context' => $context, + ]); + } + } +} + +final class StreamHandler +{ + private int $level; + + public function __construct(int $level = Logger::DEBUG) + { + $this->level = $level; + } + + public function __invoke(array $record) + { + if ($record['level'] < $this->level) { + return; + } + if (isset($record['context']['e'])) { + /** @var \Throwable $e */ + $e = $record['context']['e']; + unset($record['context']['e']); + $record['context']['type'] = get_class($e); + $record['context']['code'] = $e->getCode(); + $record['context']['message'] = sanitize_root($e->getMessage()); + $record['context']['file'] = sanitize_root($e->getFile()); + $record['context']['line'] = $e->getLine(); + $record['context']['url'] = get_current_url(); + $record['context']['trace'] = trace_to_call_points(trace_from_exception($e)); + + $ignoredExceptions = [ + 'You must specify a format', + 'Format name invalid', + 'Unknown format given', + 'Bridge name invalid', + 'Invalid action', + 'twitter: No results for this query', + // telegram + 'Unable to find channel. The channel is non-existing or non-public', + // fb + 'This group is not public! RSS-Bridge only supports public groups!', + 'You must be logged in to view this page', + 'Unable to get the page id. You should consider getting the ID by hand', + // tiktok 404 + 'https://www.tiktok.com/@', + ]; + foreach ($ignoredExceptions as $ignoredException) { + if (str_starts_with($e->getMessage(), $ignoredException)) { + return; + } + } + } + $context = ''; + if ($record['context']) { + try { + $context = Json::encode($record['context']); + } catch (\JsonException $e) { + $record['context']['message'] = null; + $context = Json::encode($record['context']); + } + } + $text = sprintf( + "[%s] %s.%s %s %s\n", + $record['created_at']->format('Y-m-d H:i:s'), + $record['name'], + $record['level_name'], + // Should probably sanitize message for output context + $record['message'], + $context + ); + error_log($text); + //$bytes = file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); + } +} + +final class NullLogger implements Logger +{ + public function debug(string $message, array $context = []) + { + } + + public function info(string $message, array $context = []): void + { + } + + public function warning(string $message, array $context = []): void + { + } + + public function error(string $message, array $context = []): void + { + } +} diff --git a/tests/Actions/ActionImplementationTest.php b/tests/Actions/ActionImplementationTest.php deleted file mode 100644 index e70dd7e2..00000000 --- a/tests/Actions/ActionImplementationTest.php +++ /dev/null @@ -1,69 +0,0 @@ -<?php - -namespace RssBridge\Tests\Actions; - -use ActionInterface; -use PHPUnit\Framework\TestCase; - -class ActionImplementationTest extends TestCase -{ - private $class; - private $obj; - - public function setUp(): void - { - \Configuration::loadConfiguration(); - } - - /** - * @dataProvider dataActionsProvider - */ - public function testClassName($path) - { - $this->setAction($path); - $this->assertTrue($this->class === ucfirst($this->class), 'class name must start with uppercase character'); - $this->assertEquals(0, substr_count($this->class, ' '), 'class name must not contain spaces'); - $this->assertStringEndsWith('Action', $this->class, 'class name must end with "Action"'); - } - - /** - * @dataProvider dataActionsProvider - */ - public function testClassType($path) - { - $this->setAction($path); - $this->assertInstanceOf(ActionInterface::class, $this->obj); - } - - /** - * @dataProvider dataActionsProvider - */ - public function testVisibleMethods($path) - { - $allowedMethods = get_class_methods(ActionInterface::class); - sort($allowedMethods); - - $this->setAction($path); - - $methods = array_diff(get_class_methods($this->obj), ['__construct']); - sort($methods); - - $this->assertEquals($allowedMethods, $methods); - } - - public function dataActionsProvider() - { - $actions = []; - foreach (glob(PATH_LIB_ACTIONS . '*.php') as $path) { - $actions[basename($path, '.php')] = [$path]; - } - return $actions; - } - - private function setAction($path) - { - $this->class = '\\' . basename($path, '.php'); - $this->assertTrue(class_exists($this->class), 'class ' . $this->class . ' doesn\'t exist'); - $this->obj = new $this->class(); - } -} diff --git a/tests/Actions/ListActionTest.php b/tests/Actions/ListActionTest.php deleted file mode 100644 index 74a90254..00000000 --- a/tests/Actions/ListActionTest.php +++ /dev/null @@ -1,76 +0,0 @@ -<?php - -namespace RssBridge\Tests\Actions; - -use BridgeFactory; -use PHPUnit\Framework\TestCase; - -class ListActionTest extends TestCase -{ - public function setUp(): void - { - \Configuration::loadConfiguration(); - } - - public function testHeaders() - { - $action = new \ListAction(); - $response = $action->execute([]); - $headers = $response->getHeaders(); - $contentType = $response->getHeader('content-type'); - $this->assertSame($contentType, 'application/json'); - } - - public function testOutput() - { - $action = new \ListAction(); - $response = $action->execute([]); - $data = $response->getBody(); - - $items = json_decode($data, true); - - $this->assertNotNull($items, 'invalid JSON output: ' . json_last_error_msg()); - - $this->assertArrayHasKey('total', $items, 'Missing "total" parameter'); - $this->assertIsInt($items['total'], 'Invalid type'); - - $this->assertArrayHasKey('bridges', $items, 'Missing "bridges" array'); - - $this->assertEquals( - $items['total'], - count($items['bridges']), - 'Item count doesn\'t match' - ); - - $bridgeFactory = new BridgeFactory(); - - $this->assertEquals( - count($bridgeFactory->getBridgeClassNames()), - count($items['bridges']), - 'Number of bridges doesn\'t match' - ); - - $expectedKeys = [ - 'status', - 'uri', - 'name', - 'icon', - 'parameters', - 'maintainer', - 'description' - ]; - - $allowedStatus = [ - 'active', - 'inactive' - ]; - - foreach ($items['bridges'] as $bridge) { - foreach ($expectedKeys as $key) { - $this->assertArrayHasKey($key, $bridge, 'Missing key "' . $key . '"'); - } - - $this->assertContains($bridge['status'], $allowedStatus, 'Invalid status value'); - } - } -} diff --git a/tests/BridgeFactoryTest.php b/tests/BridgeFactoryTest.php index a97711ef..a12faf48 100644 --- a/tests/BridgeFactoryTest.php +++ b/tests/BridgeFactoryTest.php @@ -6,25 +6,14 @@ use PHPUnit\Framework\TestCase; class BridgeFactoryTest extends TestCase { - public function setUp(): void - { - \Configuration::loadConfiguration(); - } - public function testNormalizeBridgeName() { $this->assertSame('TwitterBridge', \BridgeFactory::normalizeBridgeName('TwitterBridge')); $this->assertSame('TwitterBridge', \BridgeFactory::normalizeBridgeName('TwitterBridge.php')); $this->assertSame('TwitterBridge', \BridgeFactory::normalizeBridgeName('Twitter')); - } - - public function testSanitizeBridgeName() - { - $sut = new \BridgeFactory(); - - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitterbridge')); - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitter')); - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('tWitTer')); - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('TWITTERBRIDGE')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitterbridge')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitter')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('tWitTer')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('TWITTERBRIDGE')); } } diff --git a/tests/Bridges/BridgeImplementationTest.php b/tests/Bridges/BridgeImplementationTest.php index 807649fc..af9d7db1 100644 --- a/tests/Bridges/BridgeImplementationTest.php +++ b/tests/Bridges/BridgeImplementationTest.php @@ -231,7 +231,10 @@ class BridgeImplementationTest extends TestCase { $this->class = '\\' . basename($path, '.php'); $this->assertTrue(class_exists($this->class), 'class ' . $this->class . ' doesn\'t exist'); - $this->obj = new $this->class(); + $this->obj = new $this->class( + new \NullCache(), + new \NullLogger() + ); } private function checkUrl($url) diff --git a/tests/CacheTest.php b/tests/CacheTest.php index 15d03ec1..491db75a 100644 --- a/tests/CacheTest.php +++ b/tests/CacheTest.php @@ -8,13 +8,13 @@ class CacheTest extends TestCase { public function testConfig() { - $sut = new \FileCache(['path' => '/tmp/']); + $sut = new \FileCache(new \NullLogger(), ['path' => '/tmp/']); $this->assertSame(['path' => '/tmp/', 'enable_purge' => true], $sut->getConfig()); - $sut = new \FileCache(['path' => '/', 'enable_purge' => false]); + $sut = new \FileCache(new \NullLogger(), ['path' => '/', 'enable_purge' => false]); $this->assertSame(['path' => '/', 'enable_purge' => false], $sut->getConfig()); - $sut = new \FileCache(['path' => '/tmp', 'enable_purge' => true]); + $sut = new \FileCache(new \NullLogger(), ['path' => '/tmp', 'enable_purge' => true]); $this->assertSame(['path' => '/tmp/', 'enable_purge' => true], $sut->getConfig()); } @@ -23,7 +23,7 @@ class CacheTest extends TestCase $temporaryFolder = sprintf('%s/rss_bridge_%s/', sys_get_temp_dir(), create_random_string()); mkdir($temporaryFolder); - $sut = new \FileCache([ + $sut = new \FileCache(new \NullLogger(), [ 'path' => $temporaryFolder, 'enable_purge' => true, ]); From 7a9bfa1087106d8f759f3fbcd8b1856caa8a6f74 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Fri, 22 Sep 2023 05:40:13 +0200 Subject: [PATCH 115/716] [YoutubeBridge] handle new youtube description system / fix missing description (#3682) * [YoutubeBridge] handle new youtube description system * [YoutubeBridge] fix unrelated warnings * [YoutubeBridge] discard everything when one link can not be matched & add more boundary chars * [YoutubeBridge] rebase on master & minor fixes --- bridges/YoutubeBridge.php | 224 ++++++++++++++++++++++++++++---------- 1 file changed, 167 insertions(+), 57 deletions(-) diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index b544f762..66b7614f 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -10,7 +10,7 @@ class YoutubeBridge extends BridgeAbstract { const NAME = 'YouTube Bridge'; - const URI = 'https://www.youtube.com/'; + const URI = 'https://www.youtube.com'; const CACHE_TIMEOUT = 10800; // 3h const DESCRIPTION = 'Returns the 10 newest videos by username/channel/playlist or search'; @@ -74,7 +74,7 @@ class YoutubeBridge extends BridgeAbstract private $feedName = ''; private $feeduri = ''; - private $channel_name = ''; + private $feedIconUrl = ''; // This took from repo BetterVideoRss of VerifiedJoseph. const URI_REGEX = '/(https?:\/\/(?:www\.)?(?:[a-zA-Z0-9-.]{2,256}\.[a-z]{2,20})(\:[0-9]{2 ,4})?(?:\/[a-zA-Z0-9@:%_\+.,~#"\'!?&\/\/=\-*]+|\/)?)/ims'; //phpcs:ignore @@ -87,16 +87,16 @@ class YoutubeBridge extends BridgeAbstract if ($this->getInput('u')) { /* User and Channel modes */ - $this->request = $this->getInput('u'); - $url_feed = self::URI . 'feeds/videos.xml?user=' . urlencode($this->request); - $url_listing = self::URI . 'user/' . urlencode($this->request) . '/videos'; + $request = $this->getInput('u'); + $url_feed = self::URI . '/feeds/videos.xml?user=' . urlencode($request); + $url_listing = self::URI . '/user/' . urlencode($request) . '/videos'; } elseif ($this->getInput('c')) { - $this->request = $this->getInput('c'); - $url_feed = self::URI . 'feeds/videos.xml?channel_id=' . urlencode($this->request); - $url_listing = self::URI . 'channel/' . urlencode($this->request) . '/videos'; + $request = $this->getInput('c'); + $url_feed = self::URI . '/feeds/videos.xml?channel_id=' . urlencode($request); + $url_listing = self::URI . '/channel/' . urlencode($request) . '/videos'; } elseif ($this->getInput('custom')) { - $this->request = $this->getInput('custom'); - $url_listing = self::URI . urlencode($this->request) . '/videos'; + $request = $this->getInput('custom'); + $url_listing = self::URI . '/' . urlencode($request) . '/videos'; } if (!empty($url_feed) || !empty($url_listing)) { @@ -105,7 +105,7 @@ class YoutubeBridge extends BridgeAbstract $html = $this->ytGetSimpleHTMLDOM($url_listing); $jsonData = $this->getJSONData($html); $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl; - $this->iconURL = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; + $this->feedIconUrl = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; } if (!$this->skipFeeds()) { $html = $this->ytGetSimpleHTMLDOM($url_feed); @@ -123,7 +123,7 @@ class YoutubeBridge extends BridgeAbstract // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; $this->parseJSONListing($jsonData); } else { - returnServerError('Unable to get data from YouTube. Username/Channel: ' . $this->request); + returnServerError('Unable to get data from YouTube. Username/Channel: ' . $request); } } $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); @@ -133,9 +133,9 @@ class YoutubeBridge extends BridgeAbstract // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration" // This cache will be used to find out, which videos to fetch // to make feed of 15 items or more, if there a lot of videos published on that date. - $this->request = $this->getInput('p'); - $url_feed = self::URI . 'feeds/videos.xml?playlist_id=' . urlencode($this->request); - $url_listing = self::URI . 'playlist?list=' . urlencode($this->request); + $request = $this->getInput('p'); + $url_feed = self::URI . '/feeds/videos.xml?playlist_id=' . urlencode($request); + $url_listing = self::URI . '/playlist?list=' . urlencode($request); $html = $this->ytGetSimpleHTMLDOM($url_listing); $jsonData = $this->getJSONData($html); // TODO: this method returns only first 100 video items @@ -160,10 +160,10 @@ class YoutubeBridge extends BridgeAbstract }); } elseif ($this->getInput('s')) { /* search mode */ - $this->request = $this->getInput('s'); + $request = $this->getInput('s'); $url_listing = self::URI - . 'results?search_query=' - . urlencode($this->request) + . '/results?search_query=' + . urlencode($request) . '&sp=CAI%253D'; $html = $this->ytGetSimpleHTMLDOM($url_listing); @@ -180,7 +180,7 @@ class YoutubeBridge extends BridgeAbstract } $this->parseJSONListing($jsonData); $this->feeduri = $url_listing; - $this->feedName = 'Search: ' . $this->request; + $this->feedName = 'Search: ' . $request; } else { /* no valid mode */ returnClientError("You must either specify either:\n - YouTube @@ -206,7 +206,7 @@ class YoutubeBridge extends BridgeAbstract private function ytBridgeQueryVideoInfo($vid, &$author, &$desc, &$time) { - $html = $this->ytGetSimpleHTMLDOM(self::URI . "watch?v=$vid", true); + $html = $this->ytGetSimpleHTMLDOM(self::URI . "/watch?v=$vid", true); // Skip unavailable videos if (strpos($html->innertext, 'IS_UNAVAILABLE_PAGE') !== false) { @@ -224,7 +224,7 @@ class YoutubeBridge extends BridgeAbstract } $jsonData = $this->getJSONData($html); - if (! isset($jsonData->contents)) { + if (!isset($jsonData->contents)) { return; } @@ -240,34 +240,149 @@ class YoutubeBridge extends BridgeAbstract returnServerError('Could not find videoSecondaryInfoRenderer. Error at: ' . $vid); } - if (isset($videoSecondaryInfo->description)) { - foreach ($videoSecondaryInfo->description->runs as $description) { - if (isset($description->navigationEndpoint)) { - $metadata = $description->navigationEndpoint->commandMetadata->webCommandMetadata; - $web_type = $metadata->webPageType; - $url = $metadata->url; - $text = ''; - switch ($web_type) { - case 'WEB_PAGE_TYPE_UNKNOWN': - $url_components = parse_url($url); - if (isset($url_components['query']) && strpos($url_components['query'], '&q=') !== false) { - parse_str($url_components['query'], $params); - $url = urldecode($params['q']); - } - $text = $url; - break; - case 'WEB_PAGE_TYPE_WATCH': - case 'WEB_PAGE_TYPE_BROWSE': - $url = 'https://www.youtube.com' . $url; - $text = $description->text; - break; - } - $desc .= "<a href=\"$url\" target=\"_blank\">$text</a>"; - } else { - $desc .= nl2br($description->text); - } + $desc = $videoSecondaryInfo->attributedDescription->content ?? ''; + + // Default whitespace chars used by trim + non-breaking spaces (https://en.wikipedia.org/wiki/Non-breaking_space) + $whitespaceChars = " \t\n\r\0\x0B\u{A0}\u{2060}\u{202F}\u{2007}"; + $descEnhancements = $this->ytBridgeGetVideoDescriptionEnhancements($videoSecondaryInfo, $desc, self::URI, $whitespaceChars); + foreach ($descEnhancements as $descEnhancement) { + if (isset($descEnhancement['url'])) { + $descBefore = mb_substr($desc, 0, $descEnhancement['pos']); + $descValue = mb_substr($desc, $descEnhancement['pos'], $descEnhancement['len']); + $descAfter = mb_substr($desc, $descEnhancement['pos'] + $descEnhancement['len'], null); + + // Extended trim for the display value of internal links, e.g.: + // FAVICON • Video Name + // FAVICON / @ChannelName + $descValue = trim($descValue, $whitespaceChars . '•/'); + + $desc = sprintf('%s<a href="%s" target="_blank">%s</a>%s', $descBefore, $descEnhancement['url'], $descValue, $descAfter); } } + + $desc = nl2br($desc); + } + + private function ytBridgeGetVideoDescriptionEnhancements( + object $videoSecondaryInfo, + string $descriptionContent, + string $baseUrl, + string $whitespaceChars + ): array { + $commandRuns = $videoSecondaryInfo->attributedDescription->commandRuns ?? []; + if (count($commandRuns) <= 0) { + return []; + } + + $enhancements = []; + + $boundaryWhitespaceChars = mb_str_split($whitespaceChars); + $boundaryStartChars = array_merge($boundaryWhitespaceChars, [':', '-', '(']); + $boundaryEndChars = array_merge($boundaryWhitespaceChars, [',', '.', "'", ')']); + $hashtagBoundaryEndChars = array_merge($boundaryEndChars, ['#', '-']); + + $descriptionContentLength = mb_strlen($descriptionContent); + + $minPositionOffset = 0; + + $prevStartPosition = 0; + $totalLength = 0; + $maxPositionByStartIndex = []; + foreach (array_reverse($commandRuns) as $commandRun) { + $endPosition = $commandRun->startIndex + $commandRun->length; + if ($endPosition < $prevStartPosition) { + $totalLength += 1; + } + $totalLength += $commandRun->length; + $maxPositionByStartIndex[$commandRun->startIndex] = $totalLength; + $prevStartPosition = $commandRun->startIndex; + } + + foreach ($commandRuns as $commandRun) { + $commandMetadata = $commandRun->onTap->innertubeCommand->commandMetadata->webCommandMetadata ?? null; + if (!isset($commandMetadata)) { + continue; + } + + $enhancement = null; + + /* + $commandRun->startIndex can be offset by few positions in the positive direction + when some multibyte characters (e.g. emojis, but maybe also others) are used in the plain text video description. + (probably some difference between php and javascript in handling multibyte characters) + This loop should correct the position in most cases. It searches for the next word (determined by a set of boundary chars) with the expected length. + Several safeguards ensure that the correct word is chosen. When a link can not be matched, + everything will be discarded to prevent corrupting the description. + Hashtags require a different set of boundary chars. + */ + $isHashtag = $commandMetadata->webPageType === 'WEB_PAGE_TYPE_BROWSE'; + $prevEnhancement = end($enhancements); + $minPosition = $prevEnhancement === false ? 0 : $prevEnhancement['pos'] + $prevEnhancement['len']; + $maxPosition = $descriptionContentLength - $maxPositionByStartIndex[$commandRun->startIndex]; + $position = min($commandRun->startIndex - $minPositionOffset, $maxPosition); + while ($position >= $minPosition) { + // The link display value can only ever include a new line at the end (which will be removed further below), never in between. + $newLinePosition = mb_strpos($descriptionContent, "\n", $position); + if ($newLinePosition !== false && $newLinePosition < $position + ($commandRun->length - 1)) { + $position = $newLinePosition - ($commandRun->length - 1); + continue; + } + + $firstChar = mb_substr($descriptionContent, $position, 1); + $boundaryStart = mb_substr($descriptionContent, $position - 1, 1); + $boundaryEndIndex = $position + $commandRun->length; + $boundaryEnd = mb_substr($descriptionContent, $boundaryEndIndex, 1); + + $boundaryStartIsValid = $position === 0 || + in_array($boundaryStart, $boundaryStartChars) || + ($isHashtag && $firstChar === '#'); + $boundaryEndIsValid = $boundaryEndIndex === $descriptionContentLength || + in_array($boundaryEnd, $isHashtag ? $hashtagBoundaryEndChars : $boundaryEndChars); + + if ($boundaryStartIsValid && $boundaryEndIsValid) { + $minPositionOffset = $commandRun->startIndex - $position; + $enhancement = [ + 'pos' => $position, + 'len' => $commandRun->length, + ]; + break; + } + + $position--; + } + + if (!isset($enhancement)) { + $this->logger->debug(sprintf('Position %d cannot be corrected in "%s"', $commandRun->startIndex, substr($descriptionContent, 0, 50) . '...')); + // Skip to prevent the description from becoming corrupted + continue; + } + + // $commandRun->length sometimes incorrectly includes the newline as last char + $lastChar = mb_substr($descriptionContent, $enhancement['pos'] + $enhancement['len'] - 1, 1); + if ($lastChar === "\n") { + $enhancement['len'] -= 1; + } + + $commandUrl = parse_url($commandMetadata->url); + if ($commandUrl['path'] === '/redirect') { + parse_str($commandUrl['query'], $commandUrlQuery); + $enhancement['url'] = urldecode($commandUrlQuery['q']); + } else if (isset($commandUrl['host'])) { + $enhancement['url'] = $commandMetadata->url; + } else { + $enhancement['url'] = $baseUrl . $commandMetadata->url; + } + + $enhancements[] = $enhancement; + } + + if (count($enhancements) !== count($commandRuns)) { + // At least one link can not be matched. Discard everything to prevent corrupting the description. + return []; + } + + // Sort by position in descending order to be able to safely replace values + return array_reverse($enhancements); } private function ytBridgeAddItem($vid, $title, $author, $desc, $time, $thumbnail = '') @@ -277,12 +392,12 @@ class YoutubeBridge extends BridgeAbstract $item['title'] = $title; $item['author'] = $author; $item['timestamp'] = $time; - $item['uri'] = self::URI . 'watch?v=' . $vid; + $item['uri'] = self::URI . '/watch?v=' . $vid; if (!$thumbnail) { // Fallback to default thumbnail if there aren't any provided. $thumbnail = '0'; } - $thumbnailUri = str_replace('/www.', '/img.', self::URI) . 'vi/' . $vid . '/' . $thumbnail . '.jpg'; + $thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $vid . '/' . $thumbnail . '.jpg'; $item['content'] = '<a href="' . $item['uri'] . '"><img src="' . $thumbnailUri . '" /></a><br />' . $desc; $this->items[] = $item; } @@ -398,11 +513,6 @@ class YoutubeBridge extends BridgeAbstract $vid = $wrapper->videoId; $title = $wrapper->title->runs[0]->text; - if (isset($wrapper->ownerText)) { - $this->channel_name = $wrapper->ownerText->runs[0]->text; - } elseif (isset($wrapper->shortBylineText)) { - $this->channel_name = $wrapper->shortBylineText->runs[0]->text; - } $author = ''; $desc = ''; @@ -450,7 +560,7 @@ class YoutubeBridge extends BridgeAbstract public function getURI() { if (!is_null($this->getInput('p'))) { - return static::URI . 'playlist?list=' . $this->getInput('p'); + return static::URI . '/playlist?list=' . $this->getInput('p'); } elseif ($this->feeduri) { return $this->feeduri; } @@ -474,10 +584,10 @@ class YoutubeBridge extends BridgeAbstract public function getIcon() { - if (empty($this->iconURL)) { + if (empty($this->feedIconUrl)) { return parent::getIcon(); } else { - return $this->iconURL; + return $this->feedIconUrl; } } } From a3c29f3a52faddca776706759ef7ee33b68c32d5 Mon Sep 17 00:00:00 2001 From: mruac <ant8672@gmail.com> Date: Fri, 22 Sep 2023 17:08:05 +0930 Subject: [PATCH 116/716] resolve comment (#3683) https://github.com/RSS-Bridge/rss-bridge/pull/3617#issuecomment-1730244049 --- bridges/PatreonBridge.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bridges/PatreonBridge.php b/bridges/PatreonBridge.php index d72bde20..a2162425 100644 --- a/bridges/PatreonBridge.php +++ b/bridges/PatreonBridge.php @@ -135,7 +135,7 @@ class PatreonBridge extends BridgeAbstract $thumbnail = $post->attributes->thumbnail->large ?? null; $thumbnail = $thumbnail ?? $post->attributes->thumbnail->url ?? null; $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url ?? null; - $thumbnail = $thumbnail ?? $post->attributes->image->url; + $thumbnail = $thumbnail ?? $post->attributes->image->url ?? null; $audio_filename = $audio->file_name ?? $item['title']; $download_url = $audio->download_url ?? $item['uri']; $item['content'] .= "<p><a href\"{$download_url}\"><img src=\"{$thumbnail}\"><br/>🎧 {$audio_filename}</a><br/>"; @@ -150,7 +150,7 @@ class PatreonBridge extends BridgeAbstract $thumbnail = $post->attributes->thumbnail->large ?? null; $thumbnail = $thumbnail ?? $post->attributes->thumbnail->url ?? null; $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url ?? null; - $thumbnail = $thumbnail ?? $post->attributes->image->url; + $thumbnail = $thumbnail ?? $post->attributes->image->url ?? null; $item['content'] .= "<p><a href=\"{$item['uri']}\">🎬 {$item['title']}<br><img src=\"{$thumbnail}\"></a></p>"; break; @@ -158,7 +158,7 @@ class PatreonBridge extends BridgeAbstract $thumbnail = $post->attributes->thumbnail->large ?? null; $thumbnail = $thumbnail ?? $post->attributes->thumbnail->url ?? null; $thumbnail = $thumbnail ?? $post->attributes->image->thumb_url ?? null; - $thumbnail = $thumbnail ?? $post->attributes->image->url; + $thumbnail = $thumbnail ?? $post->attributes->image->url ?? null; $item['content'] .= "<p><a href=\"{$item['uri']}\">🎬 {$item['title']}<br><img src=\"{$thumbnail}\"></a></p>"; break; @@ -166,9 +166,9 @@ class PatreonBridge extends BridgeAbstract $item['content'] .= '<p>'; foreach ($post->relationships->images->data as $key => $image) { $image = $this->findInclude($posts, 'media', $image->id)->attributes; - $image_fullres = $image->download_url ?? $image->image_urls->url ?? $image->image_urls->original; + $image_fullres = $image->download_url ?? $image->image_urls->url ?? $image->image_urls->original ?? null; $filename = $image->file_name ?? ''; - $image_url = $image->image_urls->url ?? $image->image_urls->original; + $image_url = $image->image_urls->url ?? $image->image_urls->original ?? null; $item['enclosures'][] = $image_fullres; $item['content'] .= "<a href=\"{$image_fullres}\">{$filename}<br/><img src=\"{$image_url}\"></a><br/><br/>"; } From 39d671079863e3ba18534094146b32eaebd617a8 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Fri, 22 Sep 2023 20:41:39 +0200 Subject: [PATCH 117/716] fix(twitch) (#3685) --- bridges/TwitchBridge.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index e4abaa60..4cc0375d 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -96,6 +96,9 @@ EOD; throw new \Exception(sprintf('Unable to find channel `%s`', $channel)); } $user = $data->user; + if ($user->videos === null) { + throw new HttpException('Service Unavailable', 503); + } foreach ($user->videos->edges as $edge) { $video = $edge->node; From a6a1d553d9a5e17cd45f11f1e1c8d597e7184c77 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Fri, 22 Sep 2023 20:59:45 +0200 Subject: [PATCH 118/716] tweaks (#3686) --- bridges/BundesbankBridge.php | 4 +++- bridges/ReutersBridge.php | 4 ++++ bridges/TwitchBridge.php | 8 +++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bridges/BundesbankBridge.php b/bridges/BundesbankBridge.php index 4335cb69..0c6b943f 100644 --- a/bridges/BundesbankBridge.php +++ b/bridges/BundesbankBridge.php @@ -71,7 +71,9 @@ class BundesbankBridge extends BridgeAbstract $item['content'] .= '<strong>' . $study->find('.teasable__subtitle', 0)->plaintext . '</strong>'; } - $item['content'] .= '<p>' . $study->find('.teasable__text', 0)->plaintext . '</p>'; + $teasable = $study->find('.teasable__text', 0); + $teasableText = $teasable->plaintext ?? ''; + $item['content'] .= '<p>' . $teasableText . '</p>'; $item['timestamp'] = strtotime($study->find('.teasable__date', 0)->plaintext); diff --git a/bridges/ReutersBridge.php b/bridges/ReutersBridge.php index 2acadfc3..fdf4e2a9 100644 --- a/bridges/ReutersBridge.php +++ b/bridges/ReutersBridge.php @@ -466,6 +466,10 @@ EOD; } $rows = $content['rows']; foreach ($rows as $row) { + if (!is_array($row)) { + // some rows are null + continue; + } $tr = '<tr>'; foreach ($row as $data) { $tr .= '<td>' . $data . '</td>'; diff --git a/bridges/TwitchBridge.php b/bridges/TwitchBridge.php index 4cc0375d..f408f885 100644 --- a/bridges/TwitchBridge.php +++ b/bridges/TwitchBridge.php @@ -157,9 +157,11 @@ EOD; // Add played games list to content $item['content'] .= '<p><b>Played games:</b><ul>'; - if (count($video->moments->edges) > 0) { - foreach ($video->moments->edges as $edge) { - $moment = $edge->node; + + $momentEdges = $video->moments->edges ?? []; + if (count($momentEdges) > 0) { + foreach ($momentEdges as $momentEdge) { + $moment = $momentEdge->node; $item['categories'][] = $moment->description; $item['content'] .= '<li><a href="' From 07f49225d98b43f34132ebd616716ea447f7c3f4 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 23 Sep 2023 16:52:39 +0200 Subject: [PATCH 119/716] fix: bug in refactor (#3688) --- lib/BridgeAbstract.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index a3d84188..e074ce74 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -24,7 +24,7 @@ abstract class BridgeAbstract protected array $items = []; protected array $inputs = []; - protected string $queriedContext = ''; + protected ?string $queriedContext = ''; private array $configuration = []; protected CacheInterface $cache; From cb6c931b1f62b80634b73af63ace9693bc3cbe76 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 23 Sep 2023 17:50:41 +0200 Subject: [PATCH 120/716] fix(duckduckgo): order by date (#3689) --- bridges/AwwwardsBridge.php | 38 ++++++++++++++++++------------------ bridges/DuckDuckGoBridge.php | 15 +++++++++++--- bridges/EngadgetBridge.php | 11 ++++++++--- bridges/JustWatchBridge.php | 13 ++++++------ bridges/YandexZenBridge.php | 10 ++++++---- bridges/YoutubeBridge.php | 5 ++++- 6 files changed, 55 insertions(+), 37 deletions(-) diff --git a/bridges/AwwwardsBridge.php b/bridges/AwwwardsBridge.php index a1adfede..849d04d5 100644 --- a/bridges/AwwwardsBridge.php +++ b/bridges/AwwwardsBridge.php @@ -14,29 +14,10 @@ class AwwwardsBridge extends BridgeAbstract private $sites = []; - public function getIcon() - { - return 'https://www.awwwards.com/favicon.ico'; - } - - private function fetchSites() - { - Debug::log('Fetching all sites'); - $sites = getSimpleHTMLDOM(self::SITESURI); - - Debug::log('Parsing all JSON data'); - foreach ($sites->find('.grid-sites li') as $site) { - $decode = html_entity_decode($site->attr['data-collectable-model-value'], ENT_QUOTES, 'utf-8'); - $decode = json_decode($decode, true); - $this->sites[] = $decode; - } - } - public function collectData() { $this->fetchSites(); - Debug::log('Building RSS feed'); foreach ($this->sites as $site) { $item = []; $item['title'] = $site['title']; @@ -56,4 +37,23 @@ class AwwwardsBridge extends BridgeAbstract } } } + + public function getIcon() + { + return 'https://www.awwwards.com/favicon.ico'; + } + + private function fetchSites() + { + $sites = getSimpleHTMLDOM(self::SITESURI); + foreach ($sites->find('.grid-sites li') as $li) { + $encodedJson = $li->attr['data-collectable-model-value'] ?? null; + if (!$encodedJson) { + continue; + } + $json = html_entity_decode($encodedJson, ENT_QUOTES, 'utf-8'); + $site = Json::decode($json); + $this->sites[] = $site; + } + } } diff --git a/bridges/DuckDuckGoBridge.php b/bridges/DuckDuckGoBridge.php index 5edf248b..6f9069fa 100644 --- a/bridges/DuckDuckGoBridge.php +++ b/bridges/DuckDuckGoBridge.php @@ -8,7 +8,7 @@ class DuckDuckGoBridge extends BridgeAbstract const CACHE_TIMEOUT = 21600; // 6h const DESCRIPTION = 'Returns results from DuckDuckGo.'; - const SORT_DATE = '+sort:date'; + const SORT_DATE = ' sort:date'; const SORT_RELEVANCE = ''; const PARAMETERS = [ [ @@ -31,13 +31,22 @@ class DuckDuckGoBridge extends BridgeAbstract public function collectData() { - $html = getSimpleHTMLDOM(self::URI . 'html/?kd=-1&q=' . $this->getInput('u') . $this->getInput('sort')); + $query = [ + 'kd' => '-1', + 'q' => $this->getInput('u') . $this->getInput('sort'), + ]; + $url = 'https://duckduckgo.com/html/?' . http_build_query($query); + $html = getSimpleHTMLDOM($url); foreach ($html->find('div.result') as $element) { $item = []; $item['uri'] = $element->find('a.result__a', 0)->href; $item['title'] = $element->find('h2.result__title', 0)->plaintext; - $item['content'] = $element->find('a.result__snippet', 0)->plaintext; + + $snippet = $element->find('a.result__snippet', 0); + if ($snippet) { + $item['content'] = $snippet->plaintext; + } $this->items[] = $item; } } diff --git a/bridges/EngadgetBridge.php b/bridges/EngadgetBridge.php index 2bed4a4a..c219c0ff 100644 --- a/bridges/EngadgetBridge.php +++ b/bridges/EngadgetBridge.php @@ -10,14 +10,19 @@ class EngadgetBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas(static::URI . 'rss.xml', 15); + $max = 10; + $this->collectExpandableDatas(static::URI . 'rss.xml', $max); } protected function parseItem($newsItem) { $item = parent::parseItem($newsItem); - // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); + $url = (string) $newsItem->link; + if (!$url) { + return $item; + } + // todo: remove querystring tracking + $articlePage = getSimpleHTMLDOM($url); // figure contain's the main article image $article = $articlePage->find('figure', 0); // .article-text has the actual article diff --git a/bridges/JustWatchBridge.php b/bridges/JustWatchBridge.php index 66b61aff..98dcd338 100644 --- a/bridges/JustWatchBridge.php +++ b/bridges/JustWatchBridge.php @@ -170,22 +170,21 @@ class JustWatchBridge extends BridgeAbstract $item = []; $item['uri'] = $title->find('a', 0)->href; + $posterImage = $title->find('.title-poster__image > img', 0); $itemTitle = sprintf( '%s - %s', $provider->find('picture > img', 0)->alt ?? '', - $title->find('.title-poster__image > img', 0)->alt ?? '' + $posterImage->alt ?? '' ); $item['title'] = $itemTitle; - $imageUrl = $title->find('.title-poster__image > img', 0)->attr['src'] ?? ''; + $imageUrl = $posterImage->attr['src'] ?? ''; if (str_starts_with($imageUrl, 'data')) { - $imageUrl = $title->find('.title-poster__image > img', 0)->attr['data-src']; + $imageUrl = $posterImage->attr['data-src']; } - $content = '<b>Provider:</b> ' - . $provider->find('picture > img', 0)->alt . '<br>'; - $content .= '<b>Media:</b> ' - . $title->find('.title-poster__image > img', 0)->alt . '<br>'; + $content = '<b>Provider:</b> ' . $provider->find('picture > img', 0)->alt . '<br>'; + $content .= '<b>Media:</b> ' . ($posterImage->alt ?? '') . '<br>'; if (isset($title->find('.title-poster__badge', 0)->plaintext)) { $content .= '<b>Type:</b> Series<br>'; diff --git a/bridges/YandexZenBridge.php b/bridges/YandexZenBridge.php index ee1ff506..8a3db48b 100644 --- a/bridges/YandexZenBridge.php +++ b/bridges/YandexZenBridge.php @@ -45,10 +45,12 @@ class YandexZenBridge extends BridgeAbstract $item['timestamp'] = date(DateTimeInterface::ATOM, $publicationDateUnixTimestamp); } - $item['content'] = $post->text . "<br /><img src='$post->image' />"; - $item['enclosures'] = [ - $post->image, - ]; + $postImage = $post->image ?? null; + $item['content'] = $post->text; + if ($postImage) { + $item['content'] .= "<br /><img src='$postImage' />"; + $item['enclosures'] = [$postImage]; + } $this->items[] = $item; } diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 66b7614f..418e715e 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -228,7 +228,10 @@ class YoutubeBridge extends BridgeAbstract return; } - $jsonData = $jsonData->contents->twoColumnWatchNextResults->results->results->contents; + $jsonData = $jsonData->contents->twoColumnWatchNextResults->results->results->contents ?? null; + if (!$jsonData) { + throw new \Exception('Unable to find json data'); + } $videoSecondaryInfo = null; foreach ($jsonData as $item) { if (isset($item->videoSecondaryInfoRenderer)) { From b3b0736761127faf9d009a450c5a2193dfc72519 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 23 Sep 2023 18:54:14 +0200 Subject: [PATCH 121/716] feat: improve error/exception ui (#3690) --- actions/ConnectivityAction.php | 2 +- actions/DisplayAction.php | 21 +++++++++++-------- lib/AuthenticationMiddleware.php | 4 +++- lib/RssBridge.php | 4 ++-- lib/html.php | 5 +++-- lib/http.php | 6 +++--- templates/access-denied.html.php | 4 ---- templates/error.php | 15 +++++++++++++ .../{error.html.php => exception.html.php} | 17 +++++++++++++-- 9 files changed, 54 insertions(+), 24 deletions(-) delete mode 100644 templates/access-denied.html.php create mode 100644 templates/error.php rename templates/{error.html.php => exception.html.php} (86%) diff --git a/actions/ConnectivityAction.php b/actions/ConnectivityAction.php index 3bc82a9d..cfffd195 100644 --- a/actions/ConnectivityAction.php +++ b/actions/ConnectivityAction.php @@ -34,7 +34,7 @@ class ConnectivityAction implements ActionInterface public function execute(array $request) { if (!Debug::isEnabled()) { - return new Response('This action is only available in debug mode!'); + return new Response('This action is only available in debug mode!', 403); } $bridgeName = $request['bridge'] ?? null; diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 87b040c2..138319fe 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -14,7 +14,10 @@ class DisplayAction implements ActionInterface public function execute(array $request) { if (Configuration::getConfig('system', 'enable_maintenance_mode')) { - return new Response('503 Service Unavailable', 503); + return new Response(render(__DIR__ . '/../templates/error.html.php', [ + 'title' => '503 Service Unavailable', + 'message' => 'RSS-Bridge is down for maintenance.', + ]), 503); } $cacheKey = 'http_' . json_encode($request); /** @var Response $cachedResponse */ @@ -36,19 +39,19 @@ class DisplayAction implements ActionInterface $bridgeName = $request['bridge'] ?? null; if (!$bridgeName) { - return new Response('Missing bridge param', 400); + return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge parameter']), 400); } $bridgeFactory = new BridgeFactory(); $bridgeClassName = $bridgeFactory->createBridgeClassName($bridgeName); if (!$bridgeClassName) { - return new Response('Bridge not found', 404); + return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Bridge not found']), 404); } $format = $request['format'] ?? null; if (!$format) { - return new Response('You must specify a format!', 400); + return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']), 400); } if (!$bridgeFactory->isEnabled($bridgeClassName)) { - return new Response('This bridge is not whitelisted', 400); + return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400); } $noproxy = $request['_noproxy'] ?? null; @@ -120,11 +123,11 @@ class DisplayAction implements ActionInterface // Reproduce (and log) these responses regardless of error output and report limit if ($e->getCode() === 429) { $this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); - return new Response('429 Too Many Requests', 429); + return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429); } if ($e->getCode() === 503) { $this->logger->info(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e))); - return new Response('503 Service Unavailable', 503); + return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 503); } } $this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]); @@ -140,7 +143,7 @@ class DisplayAction implements ActionInterface // Render the exception as a feed item $items[] = $this->createFeedItemFromException($e, $bridge); } elseif ($errorOutput === 'http') { - return new Response(render(__DIR__ . '/../templates/error.html.php', ['e' => $e]), 500); + return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 500); } elseif ($errorOutput === 'none') { // Do nothing (produces an empty feed) } @@ -173,7 +176,7 @@ class DisplayAction implements ActionInterface $item->setUid($bridge->getName() . '_' . $uniqueIdentifier); $content = render_template(__DIR__ . '/../templates/bridge-error.html.php', [ - 'error' => render_template(__DIR__ . '/../templates/error.html.php', ['e' => $e]), + 'error' => render_template(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 'searchUrl' => self::createGithubSearchUrl($bridge), 'issueUrl' => self::createGithubIssueUrl($bridge, $e, create_sane_exception_message($e)), 'maintainer' => $bridge->getMaintainer(), diff --git a/lib/AuthenticationMiddleware.php b/lib/AuthenticationMiddleware.php index c77e1b91..8c2f6b29 100644 --- a/lib/AuthenticationMiddleware.php +++ b/lib/AuthenticationMiddleware.php @@ -44,6 +44,8 @@ final class AuthenticationMiddleware { http_response_code(401); header('WWW-Authenticate: Basic realm="RSS-Bridge"'); - return render('access-denied.html.php'); + return render(__DIR__ . '/../templates/error.html.php', [ + 'message' => 'Please authenticate in order to access this instance!', + ]); } } diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 0ec7174d..da093cab 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -22,7 +22,7 @@ final class RssBridge set_exception_handler(function (\Throwable $e) { self::$logger->error('Uncaught Exception', ['e' => $e]); http_response_code(500); - print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); + print render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]); exit(1); }); @@ -117,7 +117,7 @@ final class RssBridge } catch (\Throwable $e) { self::$logger->error('Exception in RssBridge::main()', ['e' => $e]); http_response_code(500); - print render(__DIR__ . '/../templates/error.html.php', ['e' => $e]); + print render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]); } } diff --git a/lib/html.php b/lib/html.php index 489fb5b4..505221fc 100644 --- a/lib/html.php +++ b/lib/html.php @@ -34,8 +34,9 @@ function render(string $template, array $context = []): string } /** - * Render template as absolute path or relative to templates folder. - * Do not pass user input in $template + * Render php template with context + * + * DO NOT PASS USER INPUT IN $template or $context */ function render_template(string $template, array $context = []): string { diff --git a/lib/http.php b/lib/http.php index cc1d0e22..10ce86c9 100644 --- a/lib/http.php +++ b/lib/http.php @@ -115,8 +115,8 @@ final class CurlHttpClient implements HttpClient $attempts = 0; while (true) { $attempts++; - $data = curl_exec($ch); - if ($data !== false) { + $body = curl_exec($ch); + if ($body !== false) { // The network call was successful, so break out of the loop break; } @@ -136,7 +136,7 @@ final class CurlHttpClient implements HttpClient $statusCode = curl_getinfo($ch, CURLINFO_RESPONSE_CODE); curl_close($ch); - return new Response($data, $statusCode, $responseHeaders); + return new Response($body, $statusCode, $responseHeaders); } } diff --git a/templates/access-denied.html.php b/templates/access-denied.html.php deleted file mode 100644 index 64968680..00000000 --- a/templates/access-denied.html.php +++ /dev/null @@ -1,4 +0,0 @@ - -<h1> - Please authenticate in order to access this instance -</h1> diff --git a/templates/error.php b/templates/error.php new file mode 100644 index 00000000..16ab20b0 --- /dev/null +++ b/templates/error.php @@ -0,0 +1,15 @@ +<?php +/** + * This template is for rendering error messages (not exceptions) + */ +?> + +<?php if (isset($title)): ?> + <h1> + <?= e($title) ?> + </h1> +<?php endif; ?> + +<p> + <?= e($message) ?> +</p> diff --git a/templates/error.html.php b/templates/exception.html.php similarity index 86% rename from templates/error.html.php rename to templates/exception.html.php index 5220ba7c..3fe523f8 100644 --- a/templates/error.html.php +++ b/templates/exception.html.php @@ -1,3 +1,8 @@ +<?php +/** + * This template is used for rendering exceptions + */ +?> <div class="error"> <?php if ($e instanceof HttpException): ?> @@ -12,7 +17,7 @@ <?php endif; ?> <?php if ($e->getCode() === 404): ?> - <h2>The website was not found</h2> + <h2>404 Page Not Found</h2> <p> RSS-Bridge tried to fetch a page on a website. But it doesn't exists. @@ -20,13 +25,21 @@ <?php endif; ?> <?php if ($e->getCode() === 429): ?> - <h2>Try again later</h2> + <h2>429 Try again later</h2> <p> RSS-Bridge tried to fetch a website. They told us to try again later. </p> <?php endif; ?> + <?php if ($e->getCode() === 503): ?> + <h2>503 Service Unavailable</h2> + <p> + Common causes are a server that is down for maintenance + or that is overloaded. + </p> + <?php endif; ?> + <?php else: ?> <?php if ($e->getCode() === 10): ?> <h2>The rss feed is completely empty</h2> From f943f8d0020c347a053b6e123afff8951c80f78a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 23 Sep 2023 19:28:52 +0200 Subject: [PATCH 122/716] fix: typo in prior commit (#3693) --- templates/{error.php => error.html.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename templates/{error.php => error.html.php} (100%) diff --git a/templates/error.php b/templates/error.html.php similarity index 100% rename from templates/error.php rename to templates/error.html.php From bab02bf19020775091c003b1aa040ccf2adb9a1a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 23 Sep 2023 19:29:04 +0200 Subject: [PATCH 123/716] fix(flickr) (#3692) --- bridges/FlickrBridge.php | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/bridges/FlickrBridge.php b/bridges/FlickrBridge.php index 48df9757..522c7284 100644 --- a/bridges/FlickrBridge.php +++ b/bridges/FlickrBridge.php @@ -145,7 +145,6 @@ class FlickrBridge extends BridgeAbstract . '</p>'; $item['enclosures'] = $this->extractEnclosures($model); - $this->items[] = $item; } } @@ -255,17 +254,22 @@ class FlickrBridge extends BridgeAbstract { $areas = []; $limit = 320 * 240; - - foreach ($model['sizes']['data'] as $size) { - $size = $size['data']; - $image_area = $size['width'] * $size['height']; - - if ($image_area >= $limit) { - $areas[$image_area] = $size['url']; + $sizes = $model['sizes']['data']; + foreach ($sizes as $sizeData) { + $sizeData = $sizeData['data']; + $area = $sizeData['width'] * $sizeData['height']; + if ($area >= $limit) { + $areas[$area] = $sizeData['url']; } } - - return $this->fixURL(min($areas)); + if ($areas) { + $minKey = min(array_keys($areas)); + $url = $areas[$minKey]; + } else { + $array_key_first = array_key_first($sizes); + $url = $sizes[$array_key_first]['data']['url']; + } + return $this->fixURL($url); } private function fixURL($url) From 0c69148cff1995839dd37eae228d4f2a6434f562 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 23 Sep 2023 20:39:02 +0200 Subject: [PATCH 124/716] fix(vice): news rss changed (#3694) * fix: typo in prior commit * fix(vice): news rss changed --- bridges/ViceBridge.php | 6 +++++- lib/XPathAbstract.php | 3 ++- templates/exception.html.php | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bridges/ViceBridge.php b/bridges/ViceBridge.php index 1c44d246..35414020 100644 --- a/bridges/ViceBridge.php +++ b/bridges/ViceBridge.php @@ -5,7 +5,7 @@ class ViceBridge extends FeedExpander const MAINTAINER = 'IceWreck'; const NAME = 'Vice Bridge'; const URI = 'https://www.vice.com/'; - const CACHE_TIMEOUT = 3600; // This is a news site, so don't cache for more than 10 mins + const CACHE_TIMEOUT = 3600; const DESCRIPTION = 'RSS feed for vice publications like Vice News, Munchies, Motherboard, etc.'; const PARAMETERS = [ [ 'feed' => [ @@ -24,6 +24,10 @@ class ViceBridge extends FeedExpander public function collectData() { $feed = $this->getInput('feed'); + if ($feed === 'rss') { + // They changed url in Sep 2023 + $feed = 'en/rss'; + } $feedURL = 'https://www.vice.com/' . $feed; $this->collectExpandableDatas($feedURL, 10); } diff --git a/lib/XPathAbstract.php b/lib/XPathAbstract.php index bac3bfd7..e30bb5eb 100644 --- a/lib/XPathAbstract.php +++ b/lib/XPathAbstract.php @@ -390,7 +390,7 @@ abstract class XPathAbstract extends BridgeAbstract * Should provide the feed's items. * * @param \DOMXPath $xpath - * @return \DOMNodeList + * @return \DOMNodeList|false */ protected function provideFeedItems(\DOMXPath $xpath) { @@ -417,6 +417,7 @@ abstract class XPathAbstract extends BridgeAbstract $entries = $this->provideFeedItems($xpath); if ($entries === false) { + // malformed return; } diff --git a/templates/exception.html.php b/templates/exception.html.php index 3fe523f8..6ea747f2 100644 --- a/templates/exception.html.php +++ b/templates/exception.html.php @@ -25,7 +25,7 @@ <?php endif; ?> <?php if ($e->getCode() === 429): ?> - <h2>429 Try again later</h2> + <h2>429 Too Many Requests</h2> <p> RSS-Bridge tried to fetch a website. They told us to try again later. From d33808ea9eddfb5fd7371e2ad0ba5c8d232a9b1f Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sat, 23 Sep 2023 23:49:01 +0200 Subject: [PATCH 125/716] fix: image (#3698) --- bridges/PornhubBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/PornhubBridge.php b/bridges/PornhubBridge.php index 104463a8..b8f1dec8 100644 --- a/bridges/PornhubBridge.php +++ b/bridges/PornhubBridge.php @@ -88,7 +88,8 @@ class PornhubBridge extends BridgeAbstract $item['uri'] = 'https://www.pornhub.com' . $url; // Content - $image = $element->find('img', 0)->getAttribute('data-src'); + $videoImage = $element->find('img', 0); + $image = $videoImage->getAttribute('data-src') ?: $videoImage->getAttribute('src'); if ($show_images === true) { $item['content'] = '<a href="' . $item['uri'] . '"><img src="' . $image . '"></a>'; } From 0dc6c66840a0dfd13b4ccb7408196c810b02ce46 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 24 Sep 2023 00:03:21 +0200 Subject: [PATCH 126/716] fix: add duration (#3699) --- bridges/PornhubBridge.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bridges/PornhubBridge.php b/bridges/PornhubBridge.php index b8f1dec8..788ef58a 100644 --- a/bridges/PornhubBridge.php +++ b/bridges/PornhubBridge.php @@ -87,11 +87,15 @@ class PornhubBridge extends BridgeAbstract $url = $element->find('a', 0)->href; $item['uri'] = 'https://www.pornhub.com' . $url; + // Duration + $marker = $element->find('div.marker-overlays var', 0); + $duration = $marker->innertext ?? ''; + // Content $videoImage = $element->find('img', 0); $image = $videoImage->getAttribute('data-src') ?: $videoImage->getAttribute('src'); if ($show_images === true) { - $item['content'] = '<a href="' . $item['uri'] . '"><img src="' . $image . '"></a>'; + $item['content'] = sprintf('<a href="%s"><img src="%s"></a><br>%s', $item['uri'], $image, $duration); } $uploaded = explode('/', $image); From ce353c1e4f1ea7f2b1170fa3564dcdca85350c8c Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Sun, 24 Sep 2023 16:12:30 +0200 Subject: [PATCH 127/716] [CssSelectorBridge] Fix URL filtering (#3676) (#3701) Co-authored-by: tougaj <tougaj@users.noreply.github.com> --- bridges/CssSelectorBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index ce158758..8408633b 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -114,7 +114,7 @@ class CssSelectorBridge extends BridgeAbstract { if (!empty($url_pattern)) { $url_pattern = '/' . str_replace('/', '\/', $url_pattern) . '/'; - $links = array_filter($links, function ($url) { + $links = array_filter($links, function ($url) use ($url_pattern) { return preg_match($url_pattern, $url) === 1; }); } From 437afd67e0bc339bf328488f4c7411d71a872647 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 24 Sep 2023 18:15:14 +0200 Subject: [PATCH 128/716] fix: various fixes (#3702) * fix: symfonycasts * various fixes --- bridges/AtmoNouvelleAquitaineBridge.php | 3 + bridges/BrutBridge.php | 77 +++++-------------------- bridges/SitemapBridge.php | 2 +- bridges/SymfonyCastsBridge.php | 21 ++++--- composer.json | 4 +- 5 files changed, 32 insertions(+), 75 deletions(-) diff --git a/bridges/AtmoNouvelleAquitaineBridge.php b/bridges/AtmoNouvelleAquitaineBridge.php index d4244fa9..d2621b9a 100644 --- a/bridges/AtmoNouvelleAquitaineBridge.php +++ b/bridges/AtmoNouvelleAquitaineBridge.php @@ -30,6 +30,9 @@ class AtmoNouvelleAquitaineBridge extends BridgeAbstract public function collectData() { + // this bridge is broken and unmaintained + return; + $uri = self::URI . '/monair/commune/' . $this->getInput('cities'); $html = getSimpleHTMLDOM($uri); diff --git a/bridges/BrutBridge.php b/bridges/BrutBridge.php index aa2a1b4d..0db0851a 100644 --- a/bridges/BrutBridge.php +++ b/bridges/BrutBridge.php @@ -38,50 +38,20 @@ class BrutBridge extends BridgeAbstract ] ]; - const CACHE_TIMEOUT = 1800; // 30 mins - - private $jsonRegex = '/window\.__PRELOADED_STATE__ = ((?:.*)});/'; - public function collectData() { - $html = getSimpleHTMLDOM($this->getURI()); - - $results = $html->find('div.results', 0); - - foreach ($results->find('li.col-6.col-sm-4.col-md-3.col-lg-2.px-2.pb-4') as $li) { - $item = []; - - $videoPath = self::URI . $li->children(0)->href; - $videoPageHtml = getSimpleHTMLDOMCached($videoPath, 3600); - - $json = $this->extractJson($videoPageHtml); - $id = array_keys((array) $json->media->index)[0]; - - $item['uri'] = $videoPath; - $item['title'] = $json->media->index->$id->title; - $item['timestamp'] = $json->media->index->$id->published_at; - $item['enclosures'][] = $json->media->index->$id->media->thumbnail; - - $description = $json->media->index->$id->description; - $article = ''; - - if (is_null($json->media->index->$id->media->seo_article) === false) { - $article = markdownToHtml($json->media->index->$id->media->seo_article); - } - - $item['content'] = <<<EOD - <video controls poster="{$json->media->index->$id->media->thumbnail}" preload="none"> - <source src="{$json->media->index->$id->media->mp4_url}" type="video/mp4"> - </video> - <p>{$description}</p> - {$article} -EOD; - - $this->items[] = $item; - - if (count($this->items) >= 10) { - break; - } + $url = $this->getURI(); + $html = getSimpleHTMLDOM($url); + $regex = '/window.__PRELOADED_STATE__ = (.*);/'; + preg_match($regex, $html, $parts); + $data = Json::decode($parts[1], false); + foreach ($data->medias->index as $uid => $media) { + $this->items[] = [ + 'uid' => $uid, + 'title' => $media->metadata->slug, + 'uri' => $media->share_url, + 'timestamp' => $media->published_at, + ]; } } @@ -90,35 +60,14 @@ EOD; if (!is_null($this->getInput('edition')) && !is_null($this->getInput('category'))) { return self::URI . '/' . $this->getInput('edition') . '/' . $this->getInput('category'); } - return parent::getURI(); } public function getName() { if (!is_null($this->getInput('edition')) && !is_null($this->getInput('category'))) { - return $this->getKey('category') . ' - ' . - $this->getKey('edition') . ' - Brut.'; + return $this->getKey('category') . ' - ' . $this->getKey('edition') . ' - Brut.'; } - return parent::getName(); } - - /** - * Extract JSON from page - */ - private function extractJson($html) - { - if (!preg_match($this->jsonRegex, $html, $parts)) { - returnServerError('Failed to extract data from page'); - } - - $data = json_decode($parts[1]); - - if ($data === false) { - returnServerError('Failed to decode extracted data'); - } - - return $data; - } } diff --git a/bridges/SitemapBridge.php b/bridges/SitemapBridge.php index eec9d658..660504ba 100644 --- a/bridges/SitemapBridge.php +++ b/bridges/SitemapBridge.php @@ -72,7 +72,7 @@ class SitemapBridge extends CssSelectorBridge $sitemap_xml = $this->getSitemapXml($sitemap_url, !empty($site_map)); $links = $this->sitemapXmlToList($sitemap_xml, $url_pattern, empty($limit) ? 10 : $limit); - if (empty($links) && empty(sitemapXmlToList($sitemap_xml))) { + if (empty($links) && empty($this->sitemapXmlToList($sitemap_xml))) { returnClientError('Could not retrieve URLs with Timestamps from Sitemap: ' . $sitemap_url); } diff --git a/bridges/SymfonyCastsBridge.php b/bridges/SymfonyCastsBridge.php index 29ba87cd..e3261d98 100644 --- a/bridges/SymfonyCastsBridge.php +++ b/bridges/SymfonyCastsBridge.php @@ -10,22 +10,27 @@ class SymfonyCastsBridge extends BridgeAbstract public function collectData() { - $html = getSimpleHTMLDOM('https://symfonycasts.com/updates/find'); - $dives = $html->find('div'); + $url = 'https://symfonycasts.com/updates/find'; + $html = getSimpleHTMLDOM($url); + + /** @var simple_html_dom_node[] $dives */ + $dives = $html->find('div.user-notification-not-viewed'); - /* @var simple_html_dom $div */ foreach ($dives as $div) { - $id = $div->getAttribute('data-mark-update-id-value'); $type = $div->find('h5', 0); - $title = $div->find('span', 0); + $title = $div->find('a', 0); $dateString = $div->find('h5.font-gray', 0); $href = $div->find('a', 0); - $url = 'https://symfonycasts.com' . $href->getAttribute('href'); + $hrefAttribute = $href->getAttribute('href'); + $url = 'https://symfonycasts.com' . $hrefAttribute; - $item = []; // Create an empty item - $item['uid'] = $id; + $item = []; + $item['uid'] = $div->getAttribute('data-mark-update-update-url-value'); $item['title'] = $title->innertext; + + // this natural language date string does not work $item['timestamp'] = $dateString->innertext; + $item['content'] = $type->plaintext . '<a href="' . $url . '">' . $title . '</a>'; $item['uri'] = $url; $this->items[] = $item; // Add item to the list diff --git a/composer.json b/composer.json index a08c9666..31e31d74 100644 --- a/composer.json +++ b/composer.json @@ -28,8 +28,7 @@ "ext-openssl": "*", "ext-libxml": "*", "ext-simplexml": "*", - "ext-json": "*", - "ext-intl": "*" + "ext-json": "*" }, "require-dev": { "phpunit/phpunit": "^9", @@ -39,6 +38,7 @@ "ext-memcached": "Allows to use memcached as cache type", "ext-sqlite3": "Allows to use an SQLite database for caching", "ext-zip": "Required for FDroidRepoBridge", + "ext-intl": "Required for OLXBridge", "ext-dom": "Allows to use some bridges based on XPath expressions" }, "autoload-dev": { From f321f000c170c45aadd750bddd25d5074b4e281f Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 24 Sep 2023 18:34:09 +0200 Subject: [PATCH 129/716] feat: add url component (#3684) * feat: add url library * fix --- bridges/RedditBridge.php | 25 ++++--- lib/bootstrap.php | 1 + lib/url.php | 145 +++++++++++++++++++++++++++++++++++++++ tests/UrlTest.php | 47 +++++++++++++ 4 files changed, 208 insertions(+), 10 deletions(-) create mode 100644 lib/url.php create mode 100644 tests/UrlTest.php diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 8d46f7bd..f761afaa 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -305,25 +305,30 @@ class RedditBridge extends BridgeAbstract public function detectParameters($url) { - $parsed_url = parse_url($url); - - $host = $parsed_url['host'] ?? null; - - if ($host != 'www.reddit.com' && $host != 'old.reddit.com') { + try { + $urlObject = Url::fromString($url); + } catch (UrlException $e) { return null; } - $path = explode('/', $parsed_url['path']); + $host = $urlObject->getHost(); + $path = $urlObject->getPath(); - if ($path[1] == 'r') { + $pathSegments = explode('/', $path); + + if ($host !== 'www.reddit.com' && $host !== 'old.reddit.com') { + return null; + } + + if ($pathSegments[1] == 'r') { return [ 'context' => 'single', - 'r' => $path[2] + 'r' => $pathSegments[2], ]; - } elseif ($path[1] == 'user') { + } elseif ($pathSegments[1] == 'user') { return [ 'context' => 'user', - 'u' => $path[2] + 'u' => $pathSegments[2], ]; } else { return null; diff --git a/lib/bootstrap.php b/lib/bootstrap.php index c8cf4e99..dc1c0f04 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -44,6 +44,7 @@ $files = [ __DIR__ . '/../lib/utils.php', __DIR__ . '/../lib/http.php', __DIR__ . '/../lib/logger.php', + __DIR__ . '/../lib/url.php', // Vendor __DIR__ . '/../vendor/parsedown/Parsedown.php', __DIR__ . '/../vendor/php-urljoin/src/urljoin.php', diff --git a/lib/url.php b/lib/url.php new file mode 100644 index 00000000..2dcbbba5 --- /dev/null +++ b/lib/url.php @@ -0,0 +1,145 @@ +<?php + +declare(strict_types=1); + +final class UrlException extends \Exception +{ +} + +/** + * Intentionally restrictive url parser + */ +final class Url +{ + private string $scheme; + private string $host; + private int $port; + private string $path; + private ?string $queryString; + + private function __construct() + { + } + + public static function fromString(string $url): self + { + if (!self::validate($url)) { + throw new UrlException(sprintf('Illegal url: "%s"', $url)); + } + + $parts = parse_url($url); + if ($parts === false) { + throw new UrlException(sprintf('Invalid url %s', $url)); + } + + return (new self()) + ->withScheme($parts['scheme'] ?? '') + ->withHost($parts['host']) + ->withPort($parts['port'] ?? 80) + ->withPath($parts['path'] ?? '/') + ->withQueryString($parts['query'] ?? null); + } + + public static function validate(string $url): bool + { + if (strlen($url) > 1500) { + return false; + } + $pattern = '#^https?://' // scheme + . '([a-z0-9-]+\.?)+' // one or more domain names + . '(\.[a-z]{1,24})?' // optional global tld + . '(:\d+)?' // optional port + . '($|/|\?)#i'; // end of string or slash or question mark + + return preg_match($pattern, $url) === 1; + } + + public function getScheme(): string + { + return $this->scheme; + } + + public function getHost(): string + { + return $this->host; + } + + public function getPort(): int + { + return $this->port; + } + + public function getPath(): string + { + return $this->path; + } + + public function getQueryString(): string + { + return $this->queryString; + } + + public function withScheme(string $scheme): self + { + if (!in_array($scheme, ['http', 'https'])) { + throw new UrlException(sprintf('Invalid scheme %s', $scheme)); + } + $clone = clone $this; + $clone->scheme = $scheme; + return $clone; + } + + public function withHost(string $host): self + { + $clone = clone $this; + $clone->host = $host; + return $clone; + } + + public function withPort(int $port) + { + $clone = clone $this; + $clone->port = $port; + return $clone; + } + + public function withPath(string $path): self + { + if (!str_starts_with($path, '/')) { + throw new UrlException(sprintf('Path must start with forward slash: %s', $path)); + } + $clone = clone $this; + $clone->path = $path; + return $clone; + } + + public function withQueryString(?string $queryString): self + { + $clone = clone $this; + $clone->queryString = $queryString; + return $clone; + } + + public function __toString() + { + if ($this->port === 80) { + $port = ''; + } else { + $port = ':' . $this->port; + } + if ($this->queryString) { + $queryString = '?' . $this->queryString; + } else { + $queryString = ''; + } + + return sprintf( + '%s://%s%s%s%s', + $this->scheme, + $this->host, + $port, + $this->path, + $queryString + ); + } +} diff --git a/tests/UrlTest.php b/tests/UrlTest.php new file mode 100644 index 00000000..d45f319b --- /dev/null +++ b/tests/UrlTest.php @@ -0,0 +1,47 @@ +<?php + +declare(strict_types=1); + +namespace RssBridge\Tests; + +use PHPUnit\Framework\TestCase; +use Url; + +class UrlTest extends TestCase +{ + public function testBasicUsages() + { + $urls = [ + 'http://example.com/', + 'http://example.com:9000/', + 'https://example.com/', + 'https://example.com/?foo', + 'https://example.com/?foo=bar', + ]; + foreach ($urls as $url) { + $this->assertSame($url, Url::fromString($url)->__toString()); + } + } + + public function testNormalization() + { + $urls = [ + 'http://example.com' => 'http://example.com/', + 'https://example.com/?' => 'https://example.com/', + 'https://example.com/foo?' => 'https://example.com/foo', + 'http://example.com:80/' => 'http://example.com/', + ]; + foreach ($urls as $from => $to) { + $this->assertSame($to, Url::fromString($from)->__toString()); + } + } + + public function testMutation() + { + $this->assertSame('http://example.com/foo', (Url::fromString('http://example.com/'))->withPath('/foo')->__toString()); + $this->assertSame('http://example.com/foo?a=b', (Url::fromString('http://example.com/?a=b'))->withPath('/foo')->__toString()); + $this->assertSame('http://example.com/', (Url::fromString('http://example.com/'))->withPath('/')->__toString()); + $this->assertSame('http://example.com/qqq?foo=bar', (Url::fromString('http://example.com/qqq'))->withQueryString('foo=bar')->__toString()); + $this->assertSame('http://example.net/qqq?foo=bar', (Url::fromString('http://example.com/qqq?foo=bar'))->withHost('example.net')->__toString()); + } +} From 857e908929ef26da41646c04804cbd4aa106ec52 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 24 Sep 2023 20:53:07 +0200 Subject: [PATCH 130/716] chore: prepare 2023-09-24 release (#3703) --- caches/FileCache.php | 4 +- caches/SQLiteCache.php | 4 + .../prepare_release/fetch_contributors.php | 53 ------ .../prepare_release/rssbridge-log-helper.el | 151 ------------------ contrib/prepare_release/template.md | 23 --- lib/Configuration.php | 2 +- lib/RssBridge.php | 2 + templates/frontpage.html.php | 12 +- 8 files changed, 19 insertions(+), 232 deletions(-) delete mode 100644 contrib/prepare_release/fetch_contributors.php delete mode 100644 contrib/prepare_release/rssbridge-log-helper.el delete mode 100644 contrib/prepare_release/template.md diff --git a/caches/FileCache.php b/caches/FileCache.php index 703fb6db..1ae88704 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -36,7 +36,7 @@ class FileCache implements CacheInterface $this->delete($key); return $default; } - $expiration = $item['expiration']; + $expiration = $item['expiration'] ?? time(); if ($expiration === 0 || $expiration > time()) { return $item['value']; } @@ -92,7 +92,7 @@ class FileCache implements CacheInterface unlink($cacheFile); continue; } - $expiration = $item['expiration']; + $expiration = $item['expiration'] ?? time(); if ($expiration === 0 || $expiration > time()) { continue; } diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index 94f6e289..becedde4 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -37,10 +37,14 @@ class SQLiteCache implements CacheInterface $this->db = new \SQLite3($config['file']); $this->db->enableExceptions(true); $this->db->exec("CREATE TABLE storage ('key' BLOB PRIMARY KEY, 'value' BLOB, 'updated' INTEGER)"); + // Consider uncommenting this to add an index on expiration + //$this->db->exec('CREATE INDEX idx_storage_updated ON storage (updated)'); } $this->db->busyTimeout($config['timeout']); + // https://www.sqlite.org/pragma.html#pragma_journal_mode $this->db->exec('PRAGMA journal_mode = wal'); + // https://www.sqlite.org/pragma.html#pragma_synchronous $this->db->exec('PRAGMA synchronous = NORMAL'); } diff --git a/contrib/prepare_release/fetch_contributors.php b/contrib/prepare_release/fetch_contributors.php deleted file mode 100644 index dd99229f..00000000 --- a/contrib/prepare_release/fetch_contributors.php +++ /dev/null @@ -1,53 +0,0 @@ -<?php - -/* Generate the "Contributors" list for README.md automatically utilizing the GitHub API */ - -require __DIR__ . '/../../lib/bootstrap.php'; - -$url = 'https://api.github.com/repos/rss-bridge/rss-bridge/contributors'; -$contributors = []; -$next = true; - -while ($next) { /* Collect all contributors */ - $headers = [ - 'Accept' => 'application/json', - 'Content-Type' => 'application/json', - 'User-Agent' => 'RSS-Bridge', - ]; - $httpClient = new CurlHttpClient(); - $response = $httpClient->request($url, ['headers' => $headers]); - - $json = $response->getBody(); - $json_decode = Json::decode($json, false); - foreach ($json_decode as $contributor) { - $contributors[] = $contributor; - } - - // Extract links to "next", "last", etc... - $link1 = $response->getHeader('link'); - $links = explode(',', $link1); - $next = false; - - // Check if there is a link with 'rel="next"' - foreach ($links as $link) { - [$url, $type] = explode(';', $link, 2); - - if (trim($type) === 'rel="next"') { - $url = trim(preg_replace('/([<>])/', '', $url)); - $next = true; - break; - } - } -} - -/* Example JSON data: https://api.github.com/repos/rss-bridge/rss-bridge/contributors */ - -// We want contributors sorted by name -usort($contributors, function ($a, $b) { - return strcasecmp($a->login, $b->login); -}); - -// Export as Markdown list -foreach ($contributors as $contributor) { - echo " * [{$contributor->login}]({$contributor->html_url})\n"; -} diff --git a/contrib/prepare_release/rssbridge-log-helper.el b/contrib/prepare_release/rssbridge-log-helper.el deleted file mode 100644 index a4b28226..00000000 --- a/contrib/prepare_release/rssbridge-log-helper.el +++ /dev/null @@ -1,151 +0,0 @@ -;;; rssbridge-log-helper.el --- A helper for preparing RSS-Bridge releases -*- lexical-binding:t; coding:utf-8 -*- - -;;; Commentary: - -;; Keyboard abbreviations used below: -;; C-x == Ctrl + x -;; M-x == Alt + x - -;; How to use this helper? -;; 1. Run "git log --reverse 2021-04-25..master > tmp.md" (2021-04-25 is an example tag of a previous version) -;; 2. Copy the contents of template.md to the start of tmp.md -;; 3. Open Emacs. Type M-x load-file <ENTER> -;; 4. Enter in the path to rssbridge-log-helper.el then <ENTER> -;; 5. Type M-x find-file <ENTER> -;; 6. Enter the path to tmp.md then <ENTER> -;; 7. Type M-x rssbridge-log-transient-state <ENTER> -;; 8. You can now use the following shortcuts to organize the commits: -;; x: Delete commit -;; g: Copy as general change -;; n: Copy as new bridge -;; m: Copy as modified bridge -;; r: Copy as removed bridge -;; <any key>: Quit -;; 9. Once you are done with all the commits, type C-x then C-s -;; 10. Exit Emacs with C-x then C-c - -;;; Code: - -(defun rssbridge-log--get-commit-block () - "Select a commit block that begins before the cursor." - (re-search-backward "^commit ") ;; (move-beginning-of-line nil) - (set-mark-command nil) - (right-char) - (re-search-forward "^commit ") - (move-end-of-line 1)) - -(defun rssbridge-log--goto-first-commit () - "Find the first commit in the file." - (goto-char (point-min)) - (re-search-forward "^commit ")) - -(defun rssbridge-log--remove-until-prev-commit-block () - "Remove from start of current line to previous commit block." - (move-beginning-of-line nil) - (set-mark-command nil) - (re-search-backward "^commit ") - (delete-region (region-beginning) (region-end))) - -(defun rssbridge-log--remove-until-next-commit-block () - "Remove from start of current line to next commit block." - (move-beginning-of-line nil) - (set-mark-command nil) - (re-search-forward "^commit ") - (move-beginning-of-line nil) - (delete-region (region-beginning) (region-end))) - -(defun rssbridge-log--cut-paste (arg) - "Copy current line to header that matches ARG." - (kill-whole-line 0) - (rssbridge-log--remove-until-next-commit-block) - (goto-char (point-min)) - (re-search-forward arg) - (move-end-of-line 1) - (newline) - (yank) - (set-mark-command 1) - (re-search-forward "^commit ") - (recenter)) - -(defun rssbridge-log-remove () - "Remove the current commit block. - -You can bind this function or use `rssbridge-log-transient-state' -to access the function." - (interactive) - (rssbridge-log--get-commit-block) - (rssbridge-log--remove-until-prev-commit-block) - (set-mark-command 1) - (re-search-forward "^commit ")) - -(defun rssbridge-log-copy-as-new () - "Copy the current commit block as a new bridge. - -You can bind this function or use `rssbridge-log-transient-state' -to access the function." - (interactive) - (rssbridge-log--get-commit-block) - (re-search-backward "^.*\\[\\(.*\\)\\].*\\((.*)\\)" (region-beginning)) - (replace-match "* \\1 () \\2") - (rssbridge-log--remove-until-prev-commit-block) - (rssbridge-log--cut-paste "## New bridges")) - -(defun rssbridge-log-copy-as-mod () - "Copy the current commit block as a modified bridge. - -You can bind this function or use `rssbridge-log-transient-state' -to access the function." - (interactive) - (rssbridge-log--get-commit-block) - (re-search-backward "^.*\\[\\(.*\\)\\]" (region-beginning)) - (replace-match "* \\1:") - (rssbridge-log--remove-until-prev-commit-block) - (rssbridge-log--cut-paste "## Modified bridges")) - -(defun rssbridge-log-copy-as-gen () - "Copy the current commit block as a general change. - -You can bind this function or use `rssbridge-log-transient-state' -to access the function." - (interactive) - (rssbridge-log--get-commit-block) - (re-search-backward "^.*\\[\\(.*\\)\\]" (region-beginning)) - (replace-match "* \\1:") - (rssbridge-log--remove-until-prev-commit-block) - (rssbridge-log--cut-paste "## General changes")) - -(defun rssbridge-log-copy-as-rem () - "Copy the current commit block as a removed bridge. - -You can bind this function or use `rssbridge-log-transient-state' -to access the function." - (interactive) - (rssbridge-log--get-commit-block) - (re-search-backward "^.*\\[\\(.*\\)\\]" (region-beginning)) - (replace-match "* \\1:") - (rssbridge-log--remove-until-prev-commit-block) - (rssbridge-log--cut-paste "## Removed bridges")) - - -(defun rssbridge-log-transient-state () - "Create a transient map for convienience. -x: Delete commit -g: Copy as general change -n: Copy as new bridge -m: Copy as modified bridge -r: Copy as removed bridge -<any key>: Quit" - (interactive) - (rssbridge-log--goto-first-commit) - (set-transient-map - (let ((map (make-sparse-keymap))) - (define-key map "x" 'rssbridge-log-remove) - (define-key map "g" 'rssbridge-log-copy-as-gen) - (define-key map "n" 'rssbridge-log-copy-as-new) - (define-key map "m" 'rssbridge-log-copy-as-mod) - (define-key map "r" 'rssbridge-log-copy-as-rem) - map) - t)) - -(provide 'rssbridge-log-helper) -;;; rssbridge-log-helper.el ends here diff --git a/contrib/prepare_release/template.md b/contrib/prepare_release/template.md deleted file mode 100644 index 7314113d..00000000 --- a/contrib/prepare_release/template.md +++ /dev/null @@ -1,23 +0,0 @@ -<!-- Checklist (hidden in release) - -- [ ] List all changes -- [ ] Update list of contributors (see README.md) -- [ ] Update release date in Configuration.php -- [ ] Set tag version to current date (YYYY-mm-dd) -- [ ] Change release title to current date (RSS-Bridge YYYY-mm-dd) - ---> - -## General changes - - -## New bridges - - -## Modified bridges - - -## Removed bridges - - -<!-- Template: No bridges were removed in this release! --> diff --git a/lib/Configuration.php b/lib/Configuration.php index 7ef97fa7..c38d7cc9 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -19,7 +19,7 @@ */ final class Configuration { - private const VERSION = 'dev.2023-07-11'; + private const VERSION = '2023-09-24'; private static $config = []; diff --git a/lib/RssBridge.php b/lib/RssBridge.php index da093cab..6ba952eb 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -30,6 +30,8 @@ final class RssBridge if ((error_reporting() & $code) === 0) { return false; } + // In the future, uncomment this: + //throw new \ErrorException($message, 0, $code, $file, $line); $text = sprintf( '%s at %s line %s', sanitize_root($message), diff --git a/templates/frontpage.html.php b/templates/frontpage.html.php index 99e2ffd9..a0d274da 100644 --- a/templates/frontpage.html.php +++ b/templates/frontpage.html.php @@ -29,8 +29,16 @@ <?= raw($bridges) ?> <section class="footer"> - <a href="https://github.com/rss-bridge/rss-bridge">RSS-Bridge ~ Public Domain</a><br> - <p class="version"><?= e(Configuration::getVersion()) ?></p> + <a href="https://github.com/RSS-Bridge/rss-bridge"> + https://github.com/RSS-Bridge/rss-bridge + </a> + + <br> + <br> + + <p class="version"> + <?= e(Configuration::getVersion()) ?> + </p> <?= $active_bridges ?>/<?= $total_bridges ?> active bridges.<br> From 09f3c1532adc20c5850e3f0751c24d2b402386d0 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Sun, 24 Sep 2023 21:13:01 +0200 Subject: [PATCH 131/716] [core] improve pull request artifacts comment (#3705) --- .github/prtester.py | 240 ++++++++++++++++++++++++++------------------ 1 file changed, 143 insertions(+), 97 deletions(-) diff --git a/.github/prtester.py b/.github/prtester.py index df6cc1ff..a2a7ab84 100644 --- a/.github/prtester.py +++ b/.github/prtester.py @@ -1,113 +1,159 @@ +import argparse import requests -import itertools from bs4 import BeautifulSoup from datetime import datetime +from typing import Iterable import os.path # This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge # # This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of # RSS-Bridge, generate a feed for each of the bridges and save the output as html files. -# It also replaces the default static CSS link with a hardcoded link to @em92's public instance, so viewing +# It also add a <base> tag with the url of em's public instance, so viewing # the HTML file locally will actually work as designed. -def testBridges(bridges,status): - for bridge in bridges: - if bridge.get('data-ref'): # Some div entries are empty, this ignores those - bridgeid = bridge.get('id') - bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata - print(bridgeid + "\n") - bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html' - forms = bridge.find_all("form") - formid = 1 - for form in forms: - # a bridge can have multiple contexts, named 'forms' in html - # this code will produce a fully working formstring that should create a working feed when called - # this will create an example feed for every single context, to test them all - formstring = '' - errormessages = [] - parameters = form.find_all("input") - lists = form.find_all("select") - # this for/if mess cycles through all available input parameters, checks if it required, then pulls - # the default or examplevalue and then combines it all together into the formstring - # if an example or default value is missing for a required attribute, it will throw an error - # any non-required fields are not tested!!! - for parameter in parameters: - if parameter.get('type') == 'hidden' and parameter.get('name') == 'context': - cleanvalue = parameter.get('value').replace(" ","+") - formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue - if parameter.get('type') == 'number' or parameter.get('type') == 'text': - if parameter.has_attr('required'): - if parameter.get('placeholder') == '': - if parameter.get('value') == '': - errormessages.append(parameter.get('name')) - else: - formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value') +class Instance: + name = '' + url = '' + +def main(instances: Iterable[Instance], with_upload: bool, comment_title: str): + start_date = datetime.now() + table_rows = [] + for instance in instances: + page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page + soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup + bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page + table_rows += testBridges(instance, bridge_cards, with_upload) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version + with open(file=os.getcwd() + '/comment.txt', mode='w+', encoding='utf-8') as file: + table_rows_value = '\n'.join(sorted(table_rows)) + file.write(f''' +## {comment_title} +| Bridge | Context | Status | +| - | - | - | +{table_rows_value} + +*last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}* + '''.strip()) + +def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) -> Iterable: + instance_suffix = '' + if instance.name: + instance_suffix = f' ({instance.name})' + table_rows = [] + for bridge_card in bridge_cards: + bridgeid = bridge_card.get('id') + bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata + print(f'{bridgeid}{instance_suffix}\n') + bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html' + bridge_name = bridgeid.replace('Bridge', '') + context_forms = bridge_card.find_all("form") + form_number = 1 + for context_form in context_forms: + # a bridge can have multiple contexts, named 'forms' in html + # this code will produce a fully working formstring that should create a working feed when called + # this will create an example feed for every single context, to test them all + formstring = '' + error_messages = [] + context_name = '*untitled*' + context_name_element = context_form.find_previous_sibling('h5') + if context_name_element and context_name_element.text.strip() != '': + context_name = context_name_element.text + parameters = context_form.find_all("input") + lists = context_form.find_all("select") + # this for/if mess cycles through all available input parameters, checks if it required, then pulls + # the default or examplevalue and then combines it all together into the formstring + # if an example or default value is missing for a required attribute, it will throw an error + # any non-required fields are not tested!!! + for parameter in parameters: + if parameter.get('type') == 'hidden' and parameter.get('name') == 'context': + cleanvalue = parameter.get('value').replace(" ","+") + formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue + if parameter.get('type') == 'number' or parameter.get('type') == 'text': + if parameter.has_attr('required'): + if parameter.get('placeholder') == '': + if parameter.get('value') == '': + name_value = parameter.get('name') + error_messages.append(f'Missing example or default value for parameter "{name_value}"') else: - formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder') - # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring - if parameter.get('type') == 'checkbox': - if parameter.has_attr('checked'): - formstring = formstring + '&' + parameter.get('name') + '=on' - for listing in lists: - selectionvalue = '' - listname = listing.get('name') - cleanlist = [] - for option in listing.contents: - if 'optgroup' in option.name: - cleanlist.extend(option) + formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value') else: - cleanlist.append(option) - firstselectionentry = 1 - for selectionentry in cleanlist: - if firstselectionentry: + formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder') + # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring + if parameter.get('type') == 'checkbox': + if parameter.has_attr('checked'): + formstring = formstring + '&' + parameter.get('name') + '=on' + for listing in lists: + selectionvalue = '' + listname = listing.get('name') + cleanlist = [] + for option in listing.contents: + if 'optgroup' in option.name: + cleanlist.extend(option) + else: + cleanlist.append(option) + firstselectionentry = 1 + for selectionentry in cleanlist: + if firstselectionentry: + selectionvalue = selectionentry.get('value') + firstselectionentry = 0 + else: + if 'selected' in selectionentry.attrs: selectionvalue = selectionentry.get('value') - firstselectionentry = 0 - else: - if 'selected' in selectionentry.attrs: - selectionvalue = selectionentry.get('value') - break - formstring = formstring + '&' + listname + '=' + selectionvalue - if not errormessages: - # if all example/default values are present, form the full request string, run the request, replace the static css - # file with the url of em's public instance and then upload it to termpad.com, a pastebin-like-site. - r = requests.get(URL + bridgestring + formstring) - pagetext = r.text.replace('static/style.css','https://rss-bridge.org/bridge01/static/style.css') - pagetext = pagetext.encode("utf_8") - termpad = requests.post(url="https://termpad.com/", data=pagetext) - termpadurl = termpad.text - termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/') - termpadurl = termpadurl.replace('\n','') - with open(os.getcwd() + '/comment.txt', 'a+') as file: - file.write("\n") - file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |") + break + formstring = formstring + '&' + listname + '=' + selectionvalue + termpad_url = 'about:blank' + if error_messages: + status = '<br>'.join(map(lambda m: f'❌ `{m}`', error_messages)) + else: + # if all example/default values are present, form the full request string, run the request, add a <base> tag with + # the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and + # then upload it to termpad.com, a pastebin-like-site. + response = requests.get(instance.url + bridgestring + formstring) + page_text = response.text.replace('<head>','<head><base href="https://rss-bridge.org/bridge01/" target="_blank">') + page_text = page_text.encode("utf_8") + soup = BeautifulSoup(page_text, "html.parser") + status_messages = list(map(lambda e: f'⚠️ `{e.text.strip().splitlines()[0]}`', soup.find_all('pre'))) + if response.status_code != 200: + status_messages = [f'❌ `HTTP status {response.status_code} {response.reason}`'] + status_messages else: - # if there are errors (which means that a required value has no example or default value), log out which error appeared - termpad = requests.post(url="https://termpad.com/", data=str(errormessages)) - termpadurl = termpad.text - termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/') - termpadurl = termpadurl.replace('\n','') - with open(os.getcwd() + '/comment.txt', 'a+') as file: - file.write("\n") - file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |") - formid += 1 + feed_items = soup.select('.feeditem') + feed_items_length = len(feed_items) + if feed_items_length <= 0: + status_messages += [f'⚠️ `The feed has no items`'] + elif feed_items_length == 1 and len(soup.select('.error')) > 0: + status_messages = [f'❌ `{feed_items[0].text.strip().splitlines()[0]}`'] + status_messages + status = '<br>'.join(status_messages) + if status.strip() == '': + status = '✔️' + if with_upload: + termpad = requests.post(url="https://termpad.com/", data=page_text) + termpad_url = termpad.text.strip() + termpad_url = termpad_url.replace('termpad.com/','termpad.com/raw/') + table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({termpad_url}) | {status} |') + form_number += 1 + return table_rows -gitstatus = ["current", "pr"] -now = datetime.now() -date_time = now.strftime("%Y-%m-%d, %H:%M:%S") - -with open(os.getcwd() + '/comment.txt', 'w+') as file: - file.write(''' ## Pull request artifacts -| file | last change | -| ---- | ------ |''') - -for status in gitstatus: # run this twice, once for the current version, once for the PR version - if status == "current": - port = "3000" # both ports are defined in the corresponding workflow .yml file - elif status == "pr": - port = "3001" - URL = "http://localhost:" + port - page = requests.get(URL) # Use python requests to grab the rss-bridge main page - soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup - bridges = soup.find_all("section") # get a soup-formatted list of all bridges on the rss-bridge page - testBridges(bridges,status) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--instances', nargs='+') + parser.add_argument('-nu', '--no-upload', action='store_true') + parser.add_argument('-t', '--comment-title', default='Pull request artifacts') + args = parser.parse_args() + instances = [] + if args.instances: + for instance_arg in args.instances: + instance_arg_parts = instance_arg.split('::') + instance = Instance() + instance.name = instance_arg_parts[1] if len(instance_arg_parts) >= 2 else '' + instance.url = instance_arg_parts[0] + instances.append(instance) + else: + instance = Instance() + instance.name = 'current' + instance.url = 'http://localhost:3000' + instances.append(instance) + instance = Instance() + instance.name = 'pr' + instance.url = 'http://localhost:3001' + instances.append(instance) + main(instances=instances, with_upload=not args.no_upload, comment_title=args.comment_title); \ No newline at end of file From e1b911fc1f3416d49c5ad5cc68587f64ab8890eb Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Sun, 24 Sep 2023 23:07:43 +0200 Subject: [PATCH 132/716] [CssSelectorBridge] Retrieve metadata for social media embeds (#3602, #3687) (#3706) * [CssSelectorBridge] Metadata from social embed (#3602, #3687) Implement the following metadata sources: - Facebook Open Graph - Twitter <meta> tags - Standard <meta> tags - JSON linked data (ld+json) The following metadata is supported: - Canonical URL (may help removing garbage from URLs) - Article title - Truncated summary - Published/Updated timestamp - Enclosure/Thumbnail image - Author Name or Twitter handle SitemapBridge will also automatically benefit from this commit. * [php8backports] Add array_is_list() Needed this function for ld+json implementation in CssSelectorBridge. * [SitemapBridge] Add option to discard thumbnail * [CssSelectorBridge] Fix linting issues --- bridges/CssSelectorBridge.php | 283 ++++++++++++++++++++++++++++++++-- bridges/SitemapBridge.php | 12 +- lib/php8backports.php | 10 ++ 3 files changed, 290 insertions(+), 15 deletions(-) diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index 8408633b..c5a09822 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -51,6 +51,11 @@ class CssSelectorBridge extends BridgeAbstract EOT, 'exampleValue' => ' | BlogName', ], + 'discard_thumbnail' => [ + 'name' => '[Optional] Discard thumbnail set by site author', + 'title' => 'Some sites set their logo as thumbnail for every article. Use this option to discard it.', + 'type' => 'checkbox', + ], 'limit' => self::LIMIT ] ]; @@ -82,6 +87,7 @@ class CssSelectorBridge extends BridgeAbstract $content_selector = $this->getInput('content_selector'); $content_cleanup = $this->getInput('content_cleanup'); $title_cleanup = $this->getInput('title_cleanup'); + $discard_thumbnail = $this->getInput('discard_thumbnail'); $limit = $this->getInput('limit') ?? 10; $html = defaultLinkTo(getSimpleHTMLDOM($url), $url); @@ -92,13 +98,17 @@ class CssSelectorBridge extends BridgeAbstract $this->items = $items; } else { foreach ($items as $item) { - $this->items[] = $this->expandEntryWithSelector( + $item = $this->expandEntryWithSelector( $item['uri'], $content_selector, $content_cleanup, $title_cleanup, $item['title'] ); + if ($discard_thumbnail && isset($item['enclosures'])) { + unset($item['enclosures']); + } + $this->items[] = $item; } } } @@ -246,27 +256,272 @@ class CssSelectorBridge extends BridgeAbstract } $entry_html = getSimpleHTMLDOMCached($entry_url); + $item = $this->entryHtmlRetrieveMetadata($entry_html); + + if (empty($item['uri'])) { + $item['uri'] = $entry_url; + } + + if (empty($item['title'])) { + $article_title = $this->getPageTitle($entry_html, $title_cleanup); + if (!empty($title_default) && (empty($article_title) || $article_title === $this->feedName)) { + $article_title = $title_default; + } + $item['title'] = $article_title; + } + $article_content = $entry_html->find($content_selector); if (!empty($article_content)) { $article_content = $article_content[0]; - } else { - returnClientError('Could not find content selector at URL: ' . $entry_url); + $article_content = convertLazyLoading($article_content); + $article_content = defaultLinkTo($article_content, $entry_url); + $article_content = $this->cleanArticleContent($article_content, $content_cleanup); + $item['content'] = $article_content; + } else if (!empty($item['content'])) { + $item['content'] .= '<br /><p><em>Could not extract full content, selector may need to be updated.</em></p>'; } - $article_content = convertLazyLoading($article_content); - $article_content = defaultLinkTo($article_content, $entry_url); - $article_content = $this->cleanArticleContent($article_content, $content_cleanup); - - $article_title = $this->getPageTitle($entry_html, $title_cleanup); - if (!empty($title_default) && (empty($article_title) || $article_title === $this->feedName)) { - $article_title = $title_default; - } + return $item; + } + /** + * Retrieve metadata from entry HTML: title, author, date published, etc. from metadata intended for social media embeds and SEO + * @param obj $entry_html DOM object representing the webpage HTML + * @return array Entry data collected from Metadata + */ + protected function entryHtmlRetrieveMetadata($entry_html) + { $item = []; - $item['uri'] = $entry_url; - $item['title'] = $article_title; - $item['content'] = $article_content; + + // == First source of metadata: Meta tags == + // Facebook Open Graph (og:KEY) - https://developers.facebook.com/docs/sharing/webmasters + // Twitter (twitter:KEY) - https://developer.twitter.com/en/docs/twitter-for-websites/cards/guides/getting-started + // Standard meta tags - https://www.w3schools.com/tags/tag_meta.asp + + // Each Entry field mapping defines a list of possible <meta> tags names that contains the expected value + static $meta_mappings = [ + // <meta property="article:KEY" content="VALUE" /> + // <meta property="og:KEY" content="VALUE" /> + // <meta property="KEY" content="VALUE" /> + // <meta name="twitter:KEY" content="VALUE" /> + // <meta name="KEY" content="VALUE"> + // <link rel="canonical" href="URL" /> + 'uri' => [ + 'og:url', + 'twitter:url', + 'canonical' + ], + 'title' => [ + 'og:title', + 'twitter:title' + ], + 'content' => [ + 'og:description', + 'twitter:description', + 'description' + ], + 'timestamp' => [ + 'article:published_time', + 'releaseDate', + 'releasedate', + 'article:modified_time', + 'lastModified', + 'lastmodified' + ], + 'enclosures' => [ + 'og:image:secure_url', + 'og:image:url', + 'og:image', + 'twitter:image', + 'thumbnailImg', + 'thumbnailimg' + ], + 'author' => [ + 'author', + 'article:author', + 'article:author:username', + 'profile:first_name', + 'profile:last_name', + 'article:author:first_name', + 'article:author:last_name', + 'twitter:creator', + ], + ]; + + $author_first_name = null; + $author_last_name = null; + + // For each Entry property, look for corresponding HTML tags using a list of candidates + foreach ($meta_mappings as $property => $field_list) { + foreach ($field_list as $field) { + // Look for HTML meta tag + $element = null; + if ($field === 'canonical') { + $element = $entry_html->find('link[rel=canonical]'); + } else { + $element = $entry_html->find("meta[property=$field], meta[name=$field]"); + } + // Found something? Extract the value and populate Entry field + if (!empty($element)) { + $element = $element[0]; + $field_value = ''; + if ($field === 'canonical') { + $field_value = $element->href; + } else { + $field_value = $element->content; + } + if (!empty($field_value)) { + if ($field === 'article:author:first_name' || $field === 'profile:first_name') { + $author_first_name = $field_value; + } else if ($field === 'article:author:last_name' || $field === 'profile:last_name') { + $author_last_name = $field_value; + } else { + $item[$property] = $field_value; + break; // Stop on first match, e.g. og:url has priority over canonical url. + } + } + } + } + } + + // Populate author from first name and last name if all we have is nothing or Twitter @username + if ((!isset($item['author']) || $item['author'][0] === '@') && (is_string($author_first_name) || is_string($author_last_name))) { + $author = ''; + if (is_string($author_first_name)) { + $author = $author_first_name; + } + if (is_string($author_last_name)) { + $author = $author . ' ' . $author_last_name; + } + $item['author'] = trim($author); + } + + // == Second source of metadata: Embedded JSON == + // JSON linked data - https://www.w3.org/TR/2014/REC-json-ld-20140116/ + // JSON linked data is COMPLEX and MAY BE LESS RELIABLE than <meta> tags. Used for fields not found as <meta> tags. + // The implementation below will load all ld+json we can understand and attempt to extract relevant information. + + // ld+json object types that hold article metadata + // Each mapping define item fields and a list of possible JSON field for this field + // Each candiate JSON field is either a string (field name) or a list (path to nested field) + static $ldjson_article_types = ['webpage', 'article', 'newsarticle', 'blogposting']; + static $ldjson_article_mappings = [ + 'uri' => ['url', 'mainEntityOfPage'], + 'title' => ['headline'], + 'content' => ['description'], + 'timestamp' => ['dateModified', 'datePublished'], + 'enclosures' => ['image'], + 'author' => [['author', 'name'], ['author', '@id'], 'author'], + ]; + + // ld+json object types that hold author metadata + $ldjson_author_types = ['person', 'organization']; + $ldjson_author_mappings = []; // ID => Name + $ldjson_author_id = null; + + // Utility function for checking if JSON array matches one of the desired ld+json object types + // A JSON object may have a single ld+json @type as a string OR several types at once as a list + $ldjson_is_of_type = function ($json, $allowed_types) { + if (isset($json['@type'])) { + $json_types = $json['@type']; + if (!is_array($json_types)) { + $json_types = [ $json_types ]; + } + foreach ($json_types as $item_type) { + if (in_array(strtolower($item_type), $allowed_types)) { + return true; + } + } + } + return false; + }; + + // Process ld+json objects embedded in the HTML DOM + foreach ($entry_html->find('script[type=application/ld+json]') as $html_ldjson_node) { + $json_raw = json_decode($html_ldjson_node->innertext, true); + if (is_array($json_raw)) { + // The JSON we just loaded may contain directly a single ld+json object AND/OR several ones under the '@graph' key + $json_items = [ $json_raw ]; + if (isset($json_raw['@graph'])) { + foreach ($json_raw['@graph'] as $json_raw_sub_item) { + $json_items[] = $json_raw_sub_item; + } + } + // Now that we have a list of distinct JSON items, we can process them individually + foreach ($json_items as $json) { + // JSON item that holds an ld+json Article object (or a variant) + if ($ldjson_is_of_type($json, $ldjson_article_types)) { + // For each item property, look for corresponding JSON fields and populate the item + foreach ($ldjson_article_mappings as $property => $field_list) { + // Skip fields already found as <meta> tags, except Twitter @username (because we might find a better name) + if (!isset($item[$property]) || ($property === 'author' && $item['author'][0] === '@')) { + foreach ($field_list as $field) { + $json_root = $json; + // If necessary, navigate inside the JSON object to access a nested field + if (is_array($field)) { + // At this point, $field = ['author', 'name'] and $json_root = {"author": {"name": "John Doe"}} + $json_navigate_ok = true; + while (count($field) > 1) { + $sub_field = array_shift($field); + if (array_key_exists($sub_field, $json_root)) { + $json_root = $json_root[$sub_field]; + if (array_is_list($json_root) && count($json_root) === 1) { + $json_root = $json_root[0]; // Unwrap list of single item e.g. {"author":[{"name":"John Doe"}]} + } + } else { + // Desired path not found in JSON, stop navigating + $json_navigate_ok = false; + break; + } + } + if (!$json_navigate_ok) { + continue; //Desired path not found in JSON, skip this field + } + $field = $field[0]; + // At this point, $field = "name" and $json_root = {"name": "John Doe"} + } + // Now we can check for desired field in JSON and populate $item accordingly + if (isset($json_root[$field])) { + $field_value = $json_root[$field]; + if (is_array($field_value)) { + $field_value = $field_value[0]; // Different versions of the same enclosure? Take the first one + } + if (is_string($field_value) && !empty($field_value)) { + if ($property === 'author' && $field === '@id') { + $ldjson_author_id = $field_value; // Author is referred to by its ID: We'll see later if we can resolve it + } else { + $item[$property] = $field_value; + break; // Stop on first match, e.g. {"author":{"name":"John Doe"}} has priority over {"author":"John Doe"} + } + } + } + } + } + } + // JSON item that holds an ld+json Author object (or a variant) + } else if ($ldjson_is_of_type($json, $ldjson_author_types)) { + if (isset($json['@id']) && isset($json['name'])) { + $ldjson_author_mappings[$json['@id']] = $json['name']; + } + } + } + } + } + + // Attempt to resolve ld+json author if all we have is nothing or Twitter @username + if ((!isset($item['author']) || $item['author'][0] === '@') && !is_null($ldjson_author_id) && isset($ldjson_author_mappings[$ldjson_author_id])) { + $item['author'] = $ldjson_author_mappings[$ldjson_author_id]; + } + + // Adjust item field types + if (isset($item['enclosures'])) { + $item['enclosures'] = [ $item['enclosures'] ]; + } + if (isset($item['timestamp'])) { + $item['timestamp'] = strtotime($item['timestamp']); + } + return $item; } } diff --git a/bridges/SitemapBridge.php b/bridges/SitemapBridge.php index 660504ba..482cbb66 100644 --- a/bridges/SitemapBridge.php +++ b/bridges/SitemapBridge.php @@ -53,6 +53,11 @@ class SitemapBridge extends CssSelectorBridge EOT, 'exampleValue' => 'https://example.com/sitemap.xml', ], + 'discard_thumbnail' => [ + 'name' => '[Optional] Discard thumbnail set by site author', + 'title' => 'Some sites set their logo as thumbnail for every article. Use this option to discard it.', + 'type' => 'checkbox', + ], 'limit' => self::LIMIT ] ]; @@ -65,6 +70,7 @@ class SitemapBridge extends CssSelectorBridge $content_cleanup = $this->getInput('content_cleanup'); $title_cleanup = $this->getInput('title_cleanup'); $site_map = $this->getInput('site_map'); + $discard_thumbnail = $this->getInput('discard_thumbnail'); $limit = $this->getInput('limit'); $this->feedName = $this->getPageTitle($url, $title_cleanup); @@ -77,7 +83,11 @@ class SitemapBridge extends CssSelectorBridge } foreach ($links as $link) { - $this->items[] = $this->expandEntryWithSelector($link, $content_selector, $content_cleanup, $title_cleanup); + $item = $this->expandEntryWithSelector($link, $content_selector, $content_cleanup, $title_cleanup); + if ($discard_thumbnail && isset($item['enclosures'])) { + unset($item['enclosures']); + } + $this->items[] = $item; } } diff --git a/lib/php8backports.php b/lib/php8backports.php index 30dfdbd9..5b103e3d 100644 --- a/lib/php8backports.php +++ b/lib/php8backports.php @@ -54,3 +54,13 @@ if (!function_exists('str_contains')) { return $needle !== '' && mb_strpos($haystack, $needle) !== false; } } + +if (!function_exists('array_is_list')) { + function array_is_list(array $arr) + { + if ($arr === []) { + return true; + } + return array_keys($arr) === range(0, count($arr) - 1); + } +} From cd30c25b08dc5ecd048e54c8abbbecd72309ab5e Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Mon, 25 Sep 2023 21:18:48 +0200 Subject: [PATCH 133/716] refactor (#3708) --- README.md | 2 +- actions/DisplayAction.php | 16 +- .../01_How_to_create_a_new_format.md | 24 - docs/08_Format_API/02_FormatInterface.md | 132 ---- docs/08_Format_API/index.md | 9 - lib/ActionInterface.php | 21 +- lib/FeedItem.php | 742 ++++++------------ lib/FormatAbstract.php | 126 +-- lib/FormatFactory.php | 2 +- lib/FormatInterface.php | 77 -- lib/bootstrap.php | 8 - lib/logger.php | 3 + tests/Formats/BaseFormatTest.php | 2 +- tests/Formats/FormatImplementationTest.php | 2 +- 14 files changed, 282 insertions(+), 884 deletions(-) delete mode 100644 docs/08_Format_API/01_How_to_create_a_new_format.md delete mode 100644 docs/08_Format_API/02_FormatInterface.md delete mode 100644 docs/08_Format_API/index.md delete mode 100644 lib/FormatInterface.php diff --git a/README.md b/README.md index dee69b85..a1e5fdc7 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ https://alice:cat@rss-bridge.org/bridge01/?action=display&bridge=FabriceBellardB ### How to create a new output format -[Create a new format](https://rss-bridge.github.io/rss-bridge/Format_API/index.html). +See `formats/PlaintextFormat.php` for an example. ### How to run unit tests and linter diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 138319fe..8c9dd057 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -93,7 +93,7 @@ class DisplayAction implements ActionInterface return $response; } - private function createResponse(array $request, BridgeAbstract $bridge, FormatInterface $format) + private function createResponse(array $request, BridgeAbstract $bridge, FormatAbstract $format) { $items = []; $infos = []; @@ -108,15 +108,15 @@ class DisplayAction implements ActionInterface if (isset($items[0]) && is_array($items[0])) { $feedItems = []; foreach ($items as $item) { - $feedItems[] = new FeedItem($item); + $feedItems[] = FeedItem::fromArray($item); } $items = $feedItems; } $infos = [ - 'name' => $bridge->getName(), - 'uri' => $bridge->getURI(), - 'donationUri' => $bridge->getDonationURI(), - 'icon' => $bridge->getIcon() + 'name' => $bridge->getName(), + 'uri' => $bridge->getURI(), + 'donationUri' => $bridge->getDonationURI(), + 'icon' => $bridge->getIcon() ]; } catch (\Exception $e) { if ($e instanceof HttpException) { @@ -167,8 +167,8 @@ class DisplayAction implements ActionInterface // Create a unique identifier every 24 hours $uniqueIdentifier = urlencode((int)(time() / 86400)); - $itemTitle = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier); - $item->setTitle($itemTitle); + $title = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier); + $item->setTitle($title); $item->setURI(get_current_url()); $item->setTimestamp(time()); diff --git a/docs/08_Format_API/01_How_to_create_a_new_format.md b/docs/08_Format_API/01_How_to_create_a_new_format.md deleted file mode 100644 index f031e65b..00000000 --- a/docs/08_Format_API/01_How_to_create_a_new_format.md +++ /dev/null @@ -1,24 +0,0 @@ -Create a new file in the `formats/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). - -The file must be named according to following specification: - -* It starts with the type -* The file name must end with 'Format' -* The file type must be PHP, written in small letters (seriously!) ".php" - -**Examples:** - -Type | Filename ------|--------- -Atom | AtomFormat.php -Html | HtmlFormat.php - -The file must start with the PHP tags and end with an empty line. The closing tag `?>` is [omitted](http://php.net/basic-syntax.instruction-separation). - -Example: - -```PHP -<?PHP - // PHP code here -// This line is empty (just imagine it!) -``` \ No newline at end of file diff --git a/docs/08_Format_API/02_FormatInterface.md b/docs/08_Format_API/02_FormatInterface.md deleted file mode 100644 index 461990a7..00000000 --- a/docs/08_Format_API/02_FormatInterface.md +++ /dev/null @@ -1,132 +0,0 @@ -The `FormatInterface` interface defines functions that need to be implemented by all formats: - -* [display](#the-display-function) -* [stringify](#the-stringify-function) -* [setItems](#the-setitems-function) -* [getItems](#the-getitems-function) -* [setCharset](#the-setcharset-function) -* [getCharset](#the-getcharset-function) -* [setExtraInfos](#the-setextrainfos-function) -* [getExtraInfos](#the-getextrainfos-function) -* [getMimeType](#the-getmimetype-function) - -Find a [template](#template) at the end of this file - -# Functions - -## The `stringify` function - -The `stringify` function returns the items received by [`setItems`](#the-setitem-function) as string. - -```PHP -stringify(): string -``` - -## The `setItems` function - -The `setItems` function receives an array of items generated by the bridge and must return the object instance. Each item represents an entry in the feed. For more information refer to the [collectData](../05_Bridge_API/02_BridgeAbstract.md#collectdata) function. - -```PHP -setItems(array $items): self -``` - -## The `getItems` function - -The `getItems` function returns the items previously set by the [`setItems`](#the-setitems-function) function. If no items where set previously this function returns an error. - -```PHP -getItems(): array -``` - -## The `setCharset` function - -The `setCharset` function receives the character set value as string and returns the object instance. - -```PHP -setCharset(string): self -``` - -## The `getCharset` function - -The `getCharset` function returns the character set value. - -```PHP -getCharset(): string -``` - -## The `setExtraInfos` function - -The `setExtraInfos` function receives an array of elements with additional information to generate format outputs and must return the object instance. - -```PHP -setExtraInfos(array $infos): self -``` - -Currently supported information are: - -Name | Description ------|------------ -`name` | Defines the name as generated by the bridge -`uri` | Defines the URI of the feed as generated by the bridge - -## The `getExtraInfos` function - -The `getExtraInfos` function returns the information previously set via the [`setExtraInfos`](#the-setextrainfos-function) function. - -```PHP -getExtraInfos(): array -``` - -## The `getMimeType` function - -The `getMimeType` function returns the expected [MIME type](https://en.wikipedia.org/wiki/Media_type#Common_examples) of the format's output. - -```PHP -parse_mime_type(): string -``` - -# Template - -This is a bare minimum template for a format: - -```PHP -<?php -class MyTypeFormat implements FormatInterface { - private $items; - private $charset; - private $extraInfos; - - public function stringify(){ - // Implement your code here - return ''; // Return items as string - } - - public function setItems(array $items){ - $this->items = $items; - return $this; - } - - public function getItems(){ - return $this->items; - } - - public function setCharset($charset){ - $this->charset = $charset; - return $this; - } - - public function getCharset(){ - return $this->charset; - } - - public function setExtraInfos(array $infos){ - $this->extraInfos = $infos; - return $this; - } - - public function getExtraInfos(){ - return $this->extraInfos; - } -} -// Imaginary empty line! -``` diff --git a/docs/08_Format_API/index.md b/docs/08_Format_API/index.md deleted file mode 100644 index c5b9e6af..00000000 --- a/docs/08_Format_API/index.md +++ /dev/null @@ -1,9 +0,0 @@ -A Format is a class that allows RSS-Bridge to turn items from a bridge into an RSS-feed format. -It is developed in a PHP file located in the `formats/` folder -[Folder structure](../04_For_Developers/03_Folder_structure.md) -and either implements the -[FormatInterface](../08_Format_API/02_FormatInterface.md) -interface or extends the FormatAbstract class. - -For more information about how to create a new _Format_, read -[How to create a new Format?](./01_How_to_create_a_new_format.md) \ No newline at end of file diff --git a/lib/ActionInterface.php b/lib/ActionInterface.php index 4eb9cc65..220dfa50 100644 --- a/lib/ActionInterface.php +++ b/lib/ActionInterface.php @@ -1,28 +1,9 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * Interface for action objects. - */ interface ActionInterface { /** - * Execute the action. - * - * Note: This function directly outputs data to the user. - * - * @return ?string + * @return string|Response */ public function execute(array $request); } diff --git a/lib/FeedItem.php b/lib/FeedItem.php index 3d1b4509..4915c1b9 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -1,511 +1,30 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * Represents a simple feed item for transformation into various feed formats. - * - * This class represents a feed item. A feed item is an entity that can be - * transformed into various feed formats. It holds a set of pre-defined - * properties: - * - * - **URI**: URI to the full article (i.e. "https://...") - * - **Title**: The title - * - **Timestamp**: A timestamp of when the item was first released - * - **Author**: Name of the author - * - **Content**: Body of the feed, as text or HTML - * - **Enclosures**: A list of links to media objects (images, videos, etc...) - * - **Categories**: A list of category names or tags to categorize the item - * - * _Note_: A feed item can have any number of additional parameters, all of which - * may or may not be transformed to the selected output format. - * - * _Remarks_: This class supports legacy items via {@see FeedItem::__construct()} - * (i.e. `$feedItem = \FeedItem($item);`). Support for legacy items may be removed - * in future versions of RSS-Bridge. - */ class FeedItem { - /** @var string|null URI to the full article */ - protected $uri = null; + protected ?string $uri = null; + protected ?string $title = null; + protected ?int $timestamp = null; + protected ?string $author = null; + protected ?string $content = null; + protected array $enclosures = []; + protected array $categories = []; + protected ?string $uid = null; + protected array $misc = []; - /** @var string|null Title of the item */ - protected $title = null; - - /** @var int|null Timestamp of when the item was first released */ - protected $timestamp = null; - - /** @var string|null Name of the author */ - protected $author = null; - - /** @var string|null Body of the feed */ - protected $content = null; - - /** @var array List of links to media objects */ - protected $enclosures = []; - - /** @var array List of category names or tags */ - protected $categories = []; - - /** @var string Unique ID for the current item */ - protected $uid = null; - - /** @var array Associative list of additional parameters */ - protected $misc = []; // Custom parameters - - /** - * Create object from legacy item. - * - * The provided array must be an associative array of key-value-pairs, where - * keys may correspond to any of the properties of this class. - * - * Example use: - * - * ```PHP - * <?php - * $item = array(); - * - * $item['uri'] = 'https://www.github.com/rss-bridge/rss-bridge/'; - * $item['title'] = 'Title'; - * $item['timestamp'] = strtotime('now'); - * $item['author'] = 'Unknown author'; - * $item['content'] = 'Hello World!'; - * $item['enclosures'] = array('https://github.com/favicon.ico'); - * $item['categories'] = array('php', 'rss-bridge', 'awesome'); - * - * $feedItem = new \FeedItem($item); - * - * ``` - * - * The result of the code above is the same as the code below: - * - * ```PHP - * <?php - * $feedItem = \FeedItem(); - * - * $feedItem->uri = 'https://www.github.com/rss-bridge/rss-bridge/'; - * $feedItem->title = 'Title'; - * $feedItem->timestamp = strtotime('now'); - * $feedItem->autor = 'Unknown author'; - * $feedItem->content = 'Hello World!'; - * $feedItem->enclosures = array('https://github.com/favicon.ico'); - * $feedItem->categories = array('php', 'rss-bridge', 'awesome'); - * ``` - * - * @param array $item (optional) A legacy item (empty: no legacy support). - * @return object A new object of this class - */ - public function __construct($item = []) + public function __construct() { - if (!is_array($item)) { - Debug::log('Item must be an array!'); - } - - foreach ($item as $key => $value) { - $this->__set($key, $value); + } + + public static function fromArray(array $itemArray): self + { + $item = new self(); + foreach ($itemArray as $key => $value) { + $item->__set($key, $value); } + return $item; } - /** - * Get current URI. - * - * Use {@see FeedItem::setURI()} to set the URI. - * - * @return string|null The URI or null if it hasn't been set. - */ - public function getURI() - { - return $this->uri; - } - - /** - * Set URI to the full article. - * - * Use {@see FeedItem::getURI()} to get the URI. - * - * _Note_: Removes whitespace from the beginning and end of the URI. - * - * _Remarks_: Uses the attribute "href" or "src" if the provided URI is an - * object of simple_html_dom_node. - * - * @param object|string $uri URI to the full article. - * @return self - */ - public function setURI($uri) - { - $this->uri = null; // Clear previous data - - if ($uri instanceof simple_html_dom_node) { - if ($uri->hasAttribute('href')) { // Anchor - $uri = $uri->href; - } elseif ($uri->hasAttribute('src')) { // Image - $uri = $uri->src; - } else { - Debug::log('The item provided as URI is unknown!'); - } - } - if (!is_string($uri)) { - Debug::log(sprintf('Expected $uri to be string but got %s', gettype($uri))); - return $this; - } - $uri = trim($uri); - // Intentionally doing a weak url validation here because FILTER_VALIDATE_URL is too strict - if (!preg_match('#^https?://#i', $uri)) { - Debug::log(sprintf('Not a valid url: "%s"', $uri)); - return $this; - } - $this->uri = $uri; - return $this; - } - - /** - * Get current title. - * - * Use {@see FeedItem::setTitle()} to set the title. - * - * @return string|null The current title or null if it hasn't been set. - */ - public function getTitle() - { - return $this->title; - } - - /** - * Set title. - * - * Use {@see FeedItem::getTitle()} to get the title. - * - * _Note_: Removes whitespace from beginning and end of the title. - * - * @param string $title The title - * @return self - */ - public function setTitle($title) - { - $this->title = null; // Clear previous data - - if (!is_string($title)) { - Debug::log('Title must be a string!'); - } else { - $this->title = truncate(trim($title)); - } - - return $this; - } - - /** - * Get current timestamp. - * - * Use {@see FeedItem::setTimestamp()} to set the timestamp. - * - * @return int|null The current timestamp or null if it hasn't been set. - */ - public function getTimestamp() - { - return $this->timestamp; - } - - /** - * Set timestamp of first release. - * - * _Note_: The timestamp should represent the number of seconds since - * January 1 1970 00:00:00 GMT (Unix time). - * - * _Remarks_: If the provided timestamp is a string (not numeric), this - * function automatically attempts to parse the string using - * [strtotime](http://php.net/manual/en/function.strtotime.php) - * - * @link http://php.net/manual/en/function.strtotime.php strtotime (PHP) - * @link https://en.wikipedia.org/wiki/Unix_time Unix time (Wikipedia) - * - * @param string|int $timestamp A timestamp of when the item was first released - * @return self - */ - public function setTimestamp($timestamp) - { - $this->timestamp = null; // Clear previous data - - if ( - !is_numeric($timestamp) - && !$timestamp = strtotime($timestamp) - ) { - Debug::log('Unable to parse timestamp!'); - } - - if ($timestamp <= 0) { - Debug::log('Timestamp must be greater than zero!'); - } else { - $this->timestamp = $timestamp; - } - - return $this; - } - - /** - * Get the current author name. - * - * Use {@see FeedItem::setAuthor()} to set the author. - * - * @return string|null The author or null if it hasn't been set. - */ - public function getAuthor() - { - return $this->author; - } - - /** - * Set the author name. - * - * Use {@see FeedItem::getAuthor()} to get the author. - * - * @param string $author The author name. - * @return self - */ - public function setAuthor($author) - { - $this->author = null; // Clear previous data - - if (!is_string($author)) { - Debug::log('Author must be a string!'); - } else { - $this->author = $author; - } - - return $this; - } - - /** - * Get item content. - * - * Use {@see FeedItem::setContent()} to set the item content. - * - * @return string|null The item content or null if it hasn't been set. - */ - public function getContent() - { - return $this->content; - } - - /** - * Set item content. - * - * Note: This function casts objects of type simple_html_dom and - * simple_html_dom_node to string. - * - * Use {@see FeedItem::getContent()} to get the current item content. - * - * @param string|object $content The item content as text or simple_html_dom object. - * @return self - */ - public function setContent($content) - { - $this->content = null; // Clear previous data - - if ( - $content instanceof simple_html_dom - || $content instanceof simple_html_dom_node - ) { - $content = (string)$content; - } - - if (is_string($content)) { - $this->content = $content; - } else { - Debug::log(sprintf('Feed content must be a string but got %s', gettype($content))); - } - - return $this; - } - - /** - * Get item enclosures. - * - * Use {@see FeedItem::setEnclosures()} to set feed enclosures. - * - * @return array Enclosures as array of enclosure URIs. - */ - public function getEnclosures() - { - return $this->enclosures; - } - - /** - * Set item enclosures. - * - * Use {@see FeedItem::getEnclosures()} to get the current item enclosures. - * - * @param array $enclosures Array of enclosures, where each element links to - * one enclosure. - * @return self - */ - public function setEnclosures($enclosures) - { - $this->enclosures = []; - - if (is_array($enclosures)) { - foreach ($enclosures as $enclosure) { - if ( - !filter_var( - $enclosure, - FILTER_VALIDATE_URL, - FILTER_FLAG_PATH_REQUIRED - ) - ) { - Debug::log('Each enclosure must contain a scheme, host and path!'); - } elseif (!in_array($enclosure, $this->enclosures)) { - $this->enclosures[] = $enclosure; - } - } - } else { - Debug::log('Enclosures must be an array!'); - } - - return $this; - } - - /** - * Get item categories. - * - * Use {@see FeedItem::setCategories()} to set item categories. - * - * @param array The item categories. - */ - public function getCategories() - { - return $this->categories; - } - - /** - * Set item categories. - * - * Use {@see FeedItem::getCategories()} to get the current item categories. - * - * @param array $categories Array of categories, where each element defines - * a single category name. - * @return self - */ - public function setCategories($categories) - { - $this->categories = []; - - if (is_array($categories)) { - foreach ($categories as $category) { - if (!is_string($category)) { - Debug::log('Category must be a string!'); - } else { - $this->categories[] = $category; - } - } - } else { - Debug::log('Categories must be an array!'); - } - - return $this; - } - - /** - * Get unique id - * - * Use {@see FeedItem::setUid()} to set the unique id. - * - * @param string The unique id. - */ - public function getUid() - { - return $this->uid; - } - - /** - * Set unique id. - * - * Use {@see FeedItem::getUid()} to get the unique id. - * - * @param string $uid A string that uniquely identifies the current item - * @return self - */ - public function setUid($uid) - { - $this->uid = null; // Clear previous data - - if (!is_string($uid)) { - Debug::log('Unique id must be a string!'); - } elseif (preg_match('/^[a-f0-9]{40}$/', $uid)) { - // keep id if it already is a SHA-1 hash - $this->uid = $uid; - } else { - $this->uid = sha1($uid); - } - - return $this; - } - - /** - * Add miscellaneous elements to the item. - * - * @param string $key Name of the element. - * @param mixed $value Value of the element. - * @return self - */ - public function addMisc($key, $value) - { - if (!is_string($key)) { - Debug::log('Key must be a string!'); - } elseif (in_array($key, get_object_vars($this))) { - Debug::log('Key must be unique!'); - } else { - $this->misc[$key] = $value; - } - - return $this; - } - - /** - * Transform current object to array - * - * @return array - */ - public function toArray() - { - return array_merge( - [ - 'uri' => $this->uri, - 'title' => $this->title, - 'timestamp' => $this->timestamp, - 'author' => $this->author, - 'content' => $this->content, - 'enclosures' => $this->enclosures, - 'categories' => $this->categories, - 'uid' => $this->uid, - ], - $this->misc - ); - } - - /** - * Set item property - * - * Allows simple assignment to parameters. This method is slower, but easier - * to implement in some cases: - * - * ```PHP - * $item = new \FeedItem(); - * $item->content = 'Hello World!'; - * $item->my_id = 42; - * ``` - * - * @param string $name Property name - * @param mixed $value Property value - */ public function __set($name, $value) { switch ($name) { @@ -538,15 +57,6 @@ class FeedItem } } - /** - * Get item property - * - * Allows simple assignment to parameters. This method is slower, but easier - * to implement in some cases. - * - * @param string $name Property name - * @return mixed Property value - */ public function __get($name) { switch ($name) { @@ -573,4 +83,220 @@ class FeedItem return null; } } + + public function getURI(): ?string + { + return $this->uri; + } + + /** + * Set URI to the full article. + * + * Use {@see FeedItem::getURI()} to get the URI. + * + * _Note_: Removes whitespace from the beginning and end of the URI. + * + * _Remarks_: Uses the attribute "href" or "src" if the provided URI is an + * object of simple_html_dom_node. + * + * @param simple_html_dom_node|object|string $uri URI to the full article. + */ + public function setURI($uri) + { + $this->uri = null; // Clear previous data + + if ($uri instanceof simple_html_dom_node) { + if ($uri->hasAttribute('href')) { // Anchor + $uri = $uri->href; + } elseif ($uri->hasAttribute('src')) { // Image + $uri = $uri->src; + } else { + Debug::log('The item provided as URI is unknown!'); + } + } + if (!is_string($uri)) { + Debug::log(sprintf('Expected $uri to be string but got %s', gettype($uri))); + return; + } + $uri = trim($uri); + // Intentionally doing a weak url validation here because FILTER_VALIDATE_URL is too strict + if (!preg_match('#^https?://#i', $uri)) { + Debug::log(sprintf('Not a valid url: "%s"', $uri)); + return; + } + $this->uri = $uri; + } + + public function getTitle(): ?string + { + return $this->title; + } + + public function setTitle($title) + { + $this->title = null; + if (!is_string($title)) { + Debug::log('Title must be a string!'); + } else { + $this->title = truncate(trim($title)); + } + } + + public function getTimestamp(): ?int + { + return $this->timestamp; + } + + public function setTimestamp($timestamp) + { + $this->timestamp = null; + if ( + !is_numeric($timestamp) + && !$timestamp = strtotime($timestamp) + ) { + Debug::log('Unable to parse timestamp!'); + } + if ($timestamp <= 0) { + Debug::log('Timestamp must be greater than zero!'); + } else { + $this->timestamp = $timestamp; + } + } + + public function getAuthor(): ?string + { + return $this->author; + } + + public function setAuthor($author) + { + $this->author = null; + if (!is_string($author)) { + Debug::log('Author must be a string!'); + } else { + $this->author = $author; + } + return $this; + } + + public function getContent(): ?string + { + return $this->content; + } + + /** + * @param string|object $content The item content as text or simple_html_dom object. + */ + public function setContent($content) + { + $this->content = null; + if ( + $content instanceof simple_html_dom + || $content instanceof simple_html_dom_node + ) { + $content = (string) $content; + } + if (is_string($content)) { + $this->content = $content; + } else { + Debug::log(sprintf('Feed content must be a string but got %s', gettype($content))); + } + } + + public function getEnclosures(): array + { + return $this->enclosures; + } + + public function setEnclosures($enclosures) + { + $this->enclosures = []; + + if (!is_array($enclosures)) { + Debug::log('Enclosures must be an array!'); + return; + } + foreach ($enclosures as $enclosure) { + if ( + !filter_var( + $enclosure, + FILTER_VALIDATE_URL, + FILTER_FLAG_PATH_REQUIRED + ) + ) { + Debug::log('Each enclosure must contain a scheme, host and path!'); + } elseif (!in_array($enclosure, $this->enclosures)) { + $this->enclosures[] = $enclosure; + } + } + } + + public function getCategories(): array + { + return $this->categories; + } + + public function setCategories($categories) + { + $this->categories = []; + + if (!is_array($categories)) { + Debug::log('Categories must be an array!'); + return; + } + foreach ($categories as $category) { + if (is_string($category)) { + $this->categories[] = $category; + } else { + Debug::log('Category must be a string!'); + } + } + } + + public function getUid(): ?string + { + return $this->uid; + } + + public function setUid($uid) + { + $this->uid = null; + if (!is_string($uid)) { + Debug::log('Unique id must be a string!'); + } elseif (preg_match('/^[a-f0-9]{40}$/', $uid)) { + // keep id if it already is SHA-1 hash + $this->uid = $uid; + } else { + $this->uid = sha1($uid); + } + } + + public function addMisc($name, $value) + { + if (!is_string($name)) { + Debug::log('Key must be a string!'); + } elseif (in_array($name, get_object_vars($this))) { + Debug::log('Key must be unique!'); + } else { + $this->misc[$name] = $value; + } + return $this; + } + + public function toArray(): array + { + return array_merge( + [ + 'uri' => $this->uri, + 'title' => $this->title, + 'timestamp' => $this->timestamp, + 'author' => $this->author, + 'content' => $this->content, + 'enclosures' => $this->enclosures, + 'categories' => $this->categories, + 'uid' => $this->uid, + ], + $this->misc + ); + } } diff --git a/lib/FormatAbstract.php b/lib/FormatAbstract.php index 3289d651..0304f627 100644 --- a/lib/FormatAbstract.php +++ b/lib/FormatAbstract.php @@ -1,132 +1,70 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license https://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * An abstract class for format implementations - * - * This class implements {@see FormatInterface} - */ -abstract class FormatAbstract implements FormatInterface +abstract class FormatAbstract { - /** The default charset (UTF-8) */ - const DEFAULT_CHARSET = 'UTF-8'; - - /** MIME type of format output */ const MIME_TYPE = 'text/plain'; - /** @var string $charset The charset */ - protected $charset; + protected string $charset = 'UTF-8'; + protected array $items = []; + protected int $lastModified; + protected array $extraInfos = []; - /** @var array $items The items */ - protected $items; + abstract public function stringify(); - /** - * @var int $lastModified A timestamp to indicate the last modified time of - * the output data. - */ - protected $lastModified; - - /** @var array $extraInfos The extra infos */ - protected $extraInfos; - - /** {@inheritdoc} */ - public function getMimeType() + public function getMimeType(): string { return static::MIME_TYPE; } - /** - * {@inheritdoc} - * - * @param string $charset {@inheritdoc} - */ - public function setCharset($charset) + public function setCharset(string $charset) { $this->charset = $charset; - - return $this; } - /** {@inheritdoc} */ - public function getCharset() + public function getCharset(): string { - $charset = $this->charset; - - if (is_null($charset)) { - return static::DEFAULT_CHARSET; - } - return $charset; + return $this->charset; } - /** - * Set the last modified time - * - * @param int $lastModified The last modified time - * @return void - */ - public function setLastModified($lastModified) + public function setLastModified(int $lastModified) { $this->lastModified = $lastModified; } - /** - * {@inheritdoc} - * - * @param array $items {@inheritdoc} - */ public function setItems(array $items) { $this->items = $items; - - return $this; - } - - /** {@inheritdoc} */ - public function getItems() - { - if (!is_array($this->items)) { - throw new \LogicException(sprintf('Feed the %s with "setItems" method before !', get_class($this))); - } - - return $this->items; } /** - * {@inheritdoc} - * - * @param array $extraInfos {@inheritdoc} + * @return FeedItem[] The items */ - public function setExtraInfos(array $extraInfos = []) + public function getItems(): array { - foreach (['name', 'uri', 'icon', 'donationUri'] as $infoName) { - if (!isset($extraInfos[$infoName])) { - $extraInfos[$infoName] = ''; - } - } - - $this->extraInfos = $extraInfos; - - return $this; + return $this->items; } - /** {@inheritdoc} */ - public function getExtraInfos() + public function setExtraInfos(array $infos = []) { - if (is_null($this->extraInfos)) { // No extra info ? - $this->setExtraInfos(); // Define with default value + $extras = [ + 'name', + 'uri', + 'icon', + 'donationUri', + ]; + foreach ($extras as $extra) { + if (!isset($infos[$extra])) { + $infos[$extra] = ''; + } } + $this->extraInfos = $infos; + } + public function getExtraInfos(): array + { + if (!$this->extraInfos) { + $this->setExtraInfos(); + } return $this->extraInfos; } } diff --git a/lib/FormatFactory.php b/lib/FormatFactory.php index d27d7d6a..9cded40f 100644 --- a/lib/FormatFactory.php +++ b/lib/FormatFactory.php @@ -33,7 +33,7 @@ class FormatFactory * @throws \InvalidArgumentException * @param string $name The name of the format e.g. "Atom", "Mrss" or "Json" */ - public function create(string $name): FormatInterface + public function create(string $name): FormatAbstract { if (! preg_match('/^[a-zA-Z0-9-]*$/', $name)) { throw new \InvalidArgumentException('Format name invalid!'); diff --git a/lib/FormatInterface.php b/lib/FormatInterface.php deleted file mode 100644 index 49e36933..00000000 --- a/lib/FormatInterface.php +++ /dev/null @@ -1,77 +0,0 @@ -<?php - -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * The format interface - * - * @todo Add missing function to the interface - * @todo Explain parameters and return values in more detail - * @todo Return self more often (to allow call chaining) - */ -interface FormatInterface -{ - /** - * Generate a string representation of the current data - * - * @return string The string representation - */ - public function stringify(); - - public function setItems(array $items); - - /** - * Return items - * - * @throws \LogicException if the items are not set - * @return FeedItem[] The items - */ - public function getItems(); - - /** - * Set extra information - * - * @param array $infos Extra information - * @return self The format object - */ - public function setExtraInfos(array $infos); - - /** - * Return extra information - * - * @return array Extra information - */ - public function getExtraInfos(); - - /** - * Return MIME type - * - * @return string The MIME type - */ - public function getMimeType(); - - /** - * Set charset - * - * @param string $charset The charset - * @return self The format object - */ - public function setCharset($charset); - - /** - * Return current charset - * - * @return string The charset - */ - public function getCharset(); -} diff --git a/lib/bootstrap.php b/lib/bootstrap.php index dc1c0f04..bc584541 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -12,20 +12,12 @@ * @link https://github.com/rss-bridge/rss-bridge */ -/** Path to the root folder of RSS-Bridge (where index.php is located) */ -const PATH_ROOT = __DIR__ . '/../'; - -/** Path to the bridges library */ - /** Path to the formats library */ const PATH_LIB_FORMATS = __DIR__ . '/../formats/'; /** Path to the caches library */ const PATH_LIB_CACHES = __DIR__ . '/../caches/'; -/** Path to the actions library */ -const PATH_LIB_ACTIONS = __DIR__ . '/../actions/'; - /** Path to the cache folder */ const PATH_CACHE = __DIR__ . '/../cache/'; diff --git a/lib/logger.php b/lib/logger.php index ed1f1179..5d95e673 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -148,6 +148,9 @@ final class StreamHandler $context ); error_log($text); + if (Debug::isEnabled()) { + print sprintf("<pre>%s</pre>\n", e($text)); + } //$bytes = file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); } } diff --git a/tests/Formats/BaseFormatTest.php b/tests/Formats/BaseFormatTest.php index 30ce1063..71e196f0 100644 --- a/tests/Formats/BaseFormatTest.php +++ b/tests/Formats/BaseFormatTest.php @@ -39,7 +39,7 @@ abstract class BaseFormatTest extends TestCase $items = []; foreach ($data['items'] as $item) { - $items[] = new \FeedItem($item); + $items[] = \FeedItem::fromArray($item); } return (object)[ diff --git a/tests/Formats/FormatImplementationTest.php b/tests/Formats/FormatImplementationTest.php index 55c6335f..03ac6d51 100644 --- a/tests/Formats/FormatImplementationTest.php +++ b/tests/Formats/FormatImplementationTest.php @@ -24,7 +24,7 @@ class FormatImplementationTest extends TestCase public function testClassType($path) { $this->setFormat($path); - $this->assertInstanceOf(FormatInterface::class, $this->obj); + $this->assertInstanceOf(FormatAbstract::class, $this->obj); } public function dataFormatsProvider() From f421c45b21c6c98c0cf5893d8776bbc34b684240 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Mon, 25 Sep 2023 22:32:15 +0200 Subject: [PATCH 134/716] test: add feed item test (#3709) * test: add feed item test also some refactor * yup * yup --- bridges/FilterBridge.php | 18 +++++----- formats/AtomFormat.php | 12 +++---- formats/HtmlFormat.php | 7 ++-- formats/JsonFormat.php | 9 ++--- formats/MrssFormat.php | 20 +++++------ formats/PlaintextFormat.php | 12 +++---- formats/SfeedFormat.php | 67 ++++++++++++++++++------------------- lib/contents.php | 2 +- lib/logger.php | 2 +- tests/FeedItemTest.php | 30 +++++++++++++++++ 10 files changed, 98 insertions(+), 81 deletions(-) create mode 100644 tests/FeedItemTest.php diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index ef739de3..1d920f90 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -73,6 +73,15 @@ class FilterBridge extends FeedExpander ], ]]; + public function collectData() + { + $url = $this->getInput('url'); + if (!Url::validate($url)) { + returnClientError('The url parameter must either refer to http or https protocol.'); + } + $this->collectExpandableDatas($this->getURI()); + } + protected function parseItem($newItem) { $item = parent::parseItem($newItem); @@ -158,13 +167,4 @@ class FilterBridge extends FeedExpander return $url; } - - public function collectData() - { - if ($this->getInput('url') && substr($this->getInput('url'), 0, 4) !== 'http') { - // just in case someone finds a way to access local files by playing with the url - returnClientError('The url parameter must either refer to http or https protocol.'); - } - $this->collectExpandableDatas($this->getURI()); - } } diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index 1b9053fa..9886e4b7 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -14,8 +14,6 @@ class AtomFormat extends FormatAbstract protected const ATOM_NS = 'http://www.w3.org/2005/Atom'; protected const MRSS_NS = 'http://search.yahoo.com/mrss/'; - const LIMIT_TITLE = 140; - public function stringify() { $feedUrl = get_current_url(); @@ -109,8 +107,8 @@ class AtomFormat extends FormatAbstract if (empty($entryTitle)) { $entryTitle = str_replace("\n", ' ', strip_tags($entryContent)); - if (strlen($entryTitle) > self::LIMIT_TITLE) { - $wrapPos = strpos(wordwrap($entryTitle, self::LIMIT_TITLE), "\n"); + if (strlen($entryTitle) > 140) { + $wrapPos = strpos(wordwrap($entryTitle, 140), "\n"); $entryTitle = substr($entryTitle, 0, $wrapPos) . '...'; } } @@ -182,11 +180,11 @@ class AtomFormat extends FormatAbstract } } - $toReturn = $document->saveXML(); + $xml = $document->saveXML(); // Remove invalid characters ini_set('mbstring.substitute_character', 'none'); - $toReturn = mb_convert_encoding($toReturn, $this->getCharset(), 'UTF-8'); - return $toReturn; + $xml = mb_convert_encoding($xml, $this->getCharset(), 'UTF-8'); + return $xml; } } diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index 9ee4aeea..4933af8d 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -6,6 +6,8 @@ class HtmlFormat extends FormatAbstract public function stringify() { + $queryString = $_SERVER['QUERY_STRING']; + $extraInfos = $this->getExtraInfos(); $formatFactory = new FormatFactory(); $buttons = []; @@ -15,7 +17,7 @@ class HtmlFormat extends FormatAbstract if ($format === 'Html') { continue; } - $formatUrl = '?' . str_ireplace('format=Html', 'format=' . $format, htmlentities($_SERVER['QUERY_STRING'])); + $formatUrl = '?' . str_ireplace('format=Html', 'format=' . $format, htmlentities($queryString)); $buttons[] = [ 'href' => $formatUrl, 'value' => $format, @@ -57,6 +59,7 @@ class HtmlFormat extends FormatAbstract ]); // Remove invalid characters ini_set('mbstring.substitute_character', 'none'); - return mb_convert_encoding($html, $this->getCharset(), 'UTF-8'); + $html = mb_convert_encoding($html, $this->getCharset(), 'UTF-8'); + return $html; } } diff --git a/formats/JsonFormat.php b/formats/JsonFormat.php index 5bb1a525..dd61da41 100644 --- a/formats/JsonFormat.php +++ b/formats/JsonFormat.php @@ -110,13 +110,8 @@ class JsonFormat extends FormatAbstract } $data['items'] = $items; - /** - * The intention here is to discard non-utf8 byte sequences. - * But the JSON_PARTIAL_OUTPUT_ON_ERROR also discards lots of other errors. - * So consider this a hack. - * Switch to JSON_INVALID_UTF8_IGNORE when PHP 7.2 is the latest platform requirement. - */ - $json = json_encode($data, \JSON_PRETTY_PRINT | \JSON_PARTIAL_OUTPUT_ON_ERROR); + // Ignoring invalid json + $json = json_encode($data, \JSON_PRETTY_PRINT | \JSON_INVALID_UTF8_IGNORE); return $json; } diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 3c3dce5a..984611c7 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -32,12 +32,6 @@ class MrssFormat extends FormatAbstract protected const ATOM_NS = 'http://www.w3.org/2005/Atom'; protected const MRSS_NS = 'http://search.yahoo.com/mrss/'; - const ALLOWED_IMAGE_EXT = [ - '.gif', - '.jpg', - '.png', - ]; - public function stringify() { $feedUrl = get_current_url(); @@ -72,8 +66,13 @@ class MrssFormat extends FormatAbstract $channel->appendChild($description); $description->appendChild($document->createTextNode($extraInfos['name'])); + $allowedIconExtensions = [ + '.gif', + '.jpg', + '.png', + ]; $icon = $extraInfos['icon']; - if (!empty($icon) && in_array(substr($icon, -4), self::ALLOWED_IMAGE_EXT)) { + if (!empty($icon) && in_array(substr($icon, -4), $allowedIconExtensions)) { $feedImage = $document->createElement('image'); $channel->appendChild($feedImage); $iconUrl = $document->createElement('url'); @@ -164,11 +163,10 @@ class MrssFormat extends FormatAbstract } } - $toReturn = $document->saveXML(); - + $xml = $document->saveXML(); // Remove invalid non-UTF8 characters ini_set('mbstring.substitute_character', 'none'); - $toReturn = mb_convert_encoding($toReturn, $this->getCharset(), 'UTF-8'); - return $toReturn; + $xml = mb_convert_encoding($xml, $this->getCharset(), 'UTF-8'); + return $xml; } } diff --git a/formats/PlaintextFormat.php b/formats/PlaintextFormat.php index c8c4e9d6..0a9237d0 100644 --- a/formats/PlaintextFormat.php +++ b/formats/PlaintextFormat.php @@ -6,18 +6,14 @@ class PlaintextFormat extends FormatAbstract public function stringify() { - $items = $this->getItems(); $data = []; - - foreach ($items as $item) { + foreach ($this->getItems() as $item) { $data[] = $item->toArray(); } - - $toReturn = print_r($data, true); - + $text = print_r($data, true); // Remove invalid non-UTF8 characters ini_set('mbstring.substitute_character', 'none'); - $toReturn = mb_convert_encoding($toReturn, $this->getCharset(), 'UTF-8'); - return $toReturn; + $text = mb_convert_encoding($text, $this->getCharset(), 'UTF-8'); + return $text; } } diff --git a/formats/SfeedFormat.php b/formats/SfeedFormat.php index 85d5f608..33740aaa 100644 --- a/formats/SfeedFormat.php +++ b/formats/SfeedFormat.php @@ -4,6 +4,38 @@ class SfeedFormat extends FormatAbstract { const MIME_TYPE = 'text/plain'; + public function stringify() + { + $text = ''; + foreach ($this->getItems() as $item) { + $text .= sprintf( + "%s\t%s\t%s\t%s\thtml\t\t%s\t%s\t%s\n", + $item->toArray()['timestamp'], + preg_replace('/\s/', ' ', $item->toArray()['title']), + $item->toArray()['uri'], + $this->escape($item->toArray()['content']), + $item->toArray()['author'], + $this->getFirstEnclosure( + $item->toArray()['enclosures'] + ), + $this->escape( + $this->getCategories( + $item->toArray()['categories'] + ) + ) + ); + } + + // Remove invalid non-UTF8 characters + ini_set('mbstring.substitute_character', 'none'); + $text = mb_convert_encoding( + $text, + $this->getCharset(), + 'UTF-8' + ); + return $text; + } + private function escape(string $str) { $str = str_replace('\\', '\\\\', $str); @@ -31,39 +63,4 @@ class SfeedFormat extends FormatAbstract } return $toReturn; } - - public function stringify() - { - $items = $this->getItems(); - - $toReturn = ''; - foreach ($items as $item) { - $toReturn .= sprintf( - "%s\t%s\t%s\t%s\thtml\t\t%s\t%s\t%s\n", - $item->toArray()['timestamp'], - preg_replace('/\s/', ' ', $item->toArray()['title']), - $item->toArray()['uri'], - $this->escape($item->toArray()['content']), - $item->toArray()['author'], - $this->getFirstEnclosure( - $item->toArray()['enclosures'] - ), - $this->escape( - $this->getCategories( - $item->toArray()['categories'] - ) - ) - ); - } - - // Remove invalid non-UTF8 characters - ini_set('mbstring.substitute_character', 'none'); - $toReturn = mb_convert_encoding( - $toReturn, - $this->getCharset(), - 'UTF-8' - ); - return $toReturn; - } } -// vi: expandtab diff --git a/lib/contents.php b/lib/contents.php index e173b542..432d9139 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -15,6 +15,7 @@ function getContents( bool $returnFull = false ) { $httpClient = RssBridge::getHttpClient(); + $cache = RssBridge::getCache(); $httpHeadersNormalized = []; foreach ($httpHeaders as $httpHeader) { @@ -51,7 +52,6 @@ function getContents( $config['proxy'] = Configuration::getConfig('proxy', 'url'); } - $cache = RssBridge::getCache(); $cacheKey = 'server_' . $url; /** @var Response $cachedResponse */ diff --git a/lib/logger.php b/lib/logger.php index 5d95e673..e41de34b 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -148,7 +148,7 @@ final class StreamHandler $context ); error_log($text); - if (Debug::isEnabled()) { + if ($record['level'] < Logger::ERROR && Debug::isEnabled()) { print sprintf("<pre>%s</pre>\n", e($text)); } //$bytes = file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); diff --git a/tests/FeedItemTest.php b/tests/FeedItemTest.php new file mode 100644 index 00000000..92833753 --- /dev/null +++ b/tests/FeedItemTest.php @@ -0,0 +1,30 @@ +<?php + +declare(strict_types=1); + +namespace RssBridge\Tests; + +use FeedItem; +use PHPUnit\Framework\TestCase; + +class FeedItemTest extends TestCase +{ + public function test() + { + $item = new FeedItem(); + $item->setTitle('hello'); + $this->assertSame('hello', $item->getTitle()); + + $item = FeedItem::fromArray(['title' => 'hello2']); + $this->assertSame('hello2', $item->getTitle()); + + $item = new FeedItem(); + $item->setAuthor('123'); + $this->assertSame('123', $item->getAuthor()); + + $item = new FeedItem(); + $item->title = 'aa'; + $this->assertSame('aa', $item->getTitle()); + $this->assertSame('aa', $item->title); + } +} From ae53adefadea4b05a419afff1f2ae447bf555132 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 26 Sep 2023 00:27:45 +0200 Subject: [PATCH 135/716] refactor: FeedItem::setTimestamp() (#3711) --- bridges/CNETFranceBridge.php | 3 ++- bridges/MastodonBridge.php | 9 +++++---- docs/04_For_Developers/index.md | 8 +++++--- lib/FeedItem.php | 14 ++++++++------ tests/FeedItemTest.php | 19 ++++++++++++++++++- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/bridges/CNETFranceBridge.php b/bridges/CNETFranceBridge.php index 724564fa..da808596 100644 --- a/bridges/CNETFranceBridge.php +++ b/bridges/CNETFranceBridge.php @@ -54,7 +54,8 @@ class CNETFranceBridge extends FeedExpander } foreach ($this->bannedURL as $term) { - if (preg_match('/' . $term . '/mi', $item['uri']) === 1) { + $preg_match = preg_match('#' . $term . '#mi', $item['uri']); + if ($preg_match === 1) { return null; } } diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index 54ac55bd..cae556b8 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -161,8 +161,8 @@ class MastodonBridge extends BridgeAbstract $object = $this->fetchAP($object); } - $item['content'] = $object['content']; - $strippedContent = strip_tags(str_replace('<br>', ' ', $object['content'])); + $item['content'] = $object['content'] ?? ''; + $strippedContent = strip_tags(str_replace('<br>', ' ', $item['content'])); if (isset($object['name'])) { $item['title'] = $object['name']; @@ -186,9 +186,10 @@ class MastodonBridge extends BridgeAbstract foreach ($object['attachment'] as $attachment) { // Only process REMOTE pictures (prevent xss) + $mediaType = $attachment['mediaType'] ?? null; if ( - $attachment['mediaType'] - && preg_match('/^image\//', $attachment['mediaType'], $match) + $mediaType + && preg_match('/^image\//', $mediaType, $match) && preg_match('/^http(s|):\/\//', $attachment['url'], $match) ) { $item['content'] = $item['content'] . '<br /><img '; diff --git a/docs/04_For_Developers/index.md b/docs/04_For_Developers/index.md index fcce11e3..97bbb854 100644 --- a/docs/04_For_Developers/index.md +++ b/docs/04_For_Developers/index.md @@ -1,10 +1,12 @@ -This area is intended for developers who decide to contribute to **RSS-Bridge** which is primarily written in [`PHP`](http://www.php.net/) with some aspects of [`HTML`](https://en.wikipedia.org/wiki/HTML) and [`CSS`](https://en.wikipedia.org/wiki/Cascading_Style_Sheets). +This area is intended for developers who decide to contribute to **RSS-Bridge**. + +It is written in PHP. If you are new to **RSS-Bridge** you should make yourself familiar with some general aspects: + - [Coding style policy](./01_Coding_style_policy.md) - [Folder structure](./03_Folder_structure.md) - [Debug mode](./05_Debug_mode.md) - [Bridge API](../05_Bridge_API/index.md) - [Cache API](../07_Cache_API/index.md) - - [Format API](../08_Format_API/index.md) - - [Technical recommendations](../09_Technical_recommendations/index.md) \ No newline at end of file + - [Technical recommendations](../09_Technical_recommendations/index.md) diff --git a/lib/FeedItem.php b/lib/FeedItem.php index 4915c1b9..0bc22ee2 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -147,14 +147,16 @@ class FeedItem return $this->timestamp; } - public function setTimestamp($timestamp) + public function setTimestamp($datetime) { $this->timestamp = null; - if ( - !is_numeric($timestamp) - && !$timestamp = strtotime($timestamp) - ) { - Debug::log('Unable to parse timestamp!'); + if (is_numeric($datetime)) { + $timestamp = $datetime; + } else { + $timestamp = strtotime($datetime); + if ($timestamp === false) { + Debug::log('Unable to parse timestamp!'); + } } if ($timestamp <= 0) { Debug::log('Timestamp must be greater than zero!'); diff --git a/tests/FeedItemTest.php b/tests/FeedItemTest.php index 92833753..0e7af222 100644 --- a/tests/FeedItemTest.php +++ b/tests/FeedItemTest.php @@ -24,7 +24,24 @@ class FeedItemTest extends TestCase $item = new FeedItem(); $item->title = 'aa'; - $this->assertSame('aa', $item->getTitle()); $this->assertSame('aa', $item->title); + $this->assertSame('aa', $item->getTitle()); + } + + public function testTimestamp() + { + $item = new FeedItem(); + $item->setTimestamp(5); + $this->assertSame(5, $item->getTimestamp()); + + $item->setTimestamp('5'); + $this->assertSame(5, $item->getTimestamp()); + + $item->setTimestamp('1970-01-01 18:00:00'); + $this->assertSame(64800, $item->getTimestamp()); + + $item->setTimestamp('1st jan last year'); + // This will fail at 2024-01-01 hehe + $this->assertSame(1640995200, $item->getTimestamp()); } } From c04c0a56142de6e2c0dfabd635bd5f1817cafea0 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Wed, 27 Sep 2023 00:07:46 +0200 Subject: [PATCH 136/716] [core] prevent "*" in prtester whitelist, causing the script to generate a preview for every single bridge (#3713) --- .github/workflows/prhtmlgenerator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prhtmlgenerator.yml b/.github/workflows/prhtmlgenerator.yml index ce82aef1..cfedca13 100644 --- a/.github/workflows/prhtmlgenerator.yml +++ b/.github/workflows/prhtmlgenerator.yml @@ -22,7 +22,7 @@ jobs: wget https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester.py; wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch; touch DEBUG; - cat $PR.patch | grep "\bbridges/.*Bridge\.php\b" | sed "s=.*\bbridges/\(.*\)Bridge\.php\b.*=\1=g" | sort | uniq > whitelist.txt + cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq > whitelist.txt - name: Start Docker - Current run: | docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3000:80 ghcr.io/rss-bridge/rss-bridge:latest From f9ec88fb45151712677107c51fc734f5a3c69038 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Wed, 27 Sep 2023 23:29:08 +0200 Subject: [PATCH 137/716] ci: incease max line length to 180 (#3714) --- docs/04_For_Developers/01_Coding_style_policy.md | 13 +++++++++---- phpcs.xml | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/04_For_Developers/01_Coding_style_policy.md b/docs/04_For_Developers/01_Coding_style_policy.md index 4e796c32..9fb0eb10 100644 --- a/docs/04_For_Developers/01_Coding_style_policy.md +++ b/docs/04_For_Developers/01_Coding_style_policy.md @@ -1,8 +1,13 @@ -This section explains the coding style policy for RSS-Bridge with examples and references to external resources. Please make sure your code is compliant before opening a pull request. +This section explains the coding style policy for RSS-Bridge with examples and references to external resources. +Please make sure your code is compliant before opening a pull request. -You will automatically be notified if issues were found in your pull request. You must fix those issues before the pull request will be merged. Refer to [phpcs.xml](https://github.com/RSS-Bridge/rss-bridge/blob/master/phpcs.xml) for a complete list of policies enforced by Travis-CI. +You will automatically be notified if issues were found in your pull request. +You must fix those issues before the pull request will be merged. +Refer to [phpcs.xml](https://github.com/RSS-Bridge/rss-bridge/blob/master/phpcs.xml) for a complete list of policies enforced by Travis-CI. -If you want to run the checks locally, make sure you have [`phpcs`](https://github.com/squizlabs/PHP_CodeSniffer) and [`phpunit`](https://phpunit.de/) installed on your machine and run following commands in the root directory of RSS-Bridge (tested on Debian): +If you want to run the checks locally, make sure you have +[`phpcs`](https://github.com/squizlabs/PHP_CodeSniffer) and +[`phpunit`](https://phpunit.de/) installed on your machine and run following commands in the root directory of RSS-Bridge (tested on Debian): ```console ./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./ @@ -73,7 +78,7 @@ _Reference_: [`Squiz.WhiteSpace.SuperfluousWhitespace`](https://github.com/squiz # Maximum Line Length -There is no maximum line length. +180 # Strings diff --git a/phpcs.xml b/phpcs.xml index 2db0553a..5e50470a 100644 --- a/phpcs.xml +++ b/phpcs.xml @@ -20,8 +20,8 @@ <rule ref="Generic.Files.LineLength"> <properties> - <property name="lineLimit" value="160"/> - <property name="absoluteLineLimit" value="160"/> + <property name="lineLimit" value="180"/> + <property name="absoluteLineLimit" value="180"/> <property name="ignoreComments" value="true"/> </properties> </rule> From 0de5180ded1734a8e141390f6ae4b7db123bff8c Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 28 Sep 2023 22:21:56 +0200 Subject: [PATCH 138/716] feat: improve sqlite cache robustness (#3715) --- caches/SQLiteCache.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php index becedde4..d7cba554 100644 --- a/caches/SQLiteCache.php +++ b/caches/SQLiteCache.php @@ -86,8 +86,13 @@ class SQLiteCache implements CacheInterface $stmt->bindValue(':key', $cacheKey); $stmt->bindValue(':value', $blob, \SQLITE3_BLOB); $stmt->bindValue(':updated', $expiration); - $result = $stmt->execute(); - // Unclear whether we should $result->finalize(); here? + try { + $result = $stmt->execute(); + // Should $result->finalize() be called here? + } catch (\Exception $e) { + $this->logger->warning(create_sane_exception_message($e)); + // Intentionally not rethrowing exception + } } public function delete(string $key): void From b9ec6a0eb4ebafa3cfef0b76a628e8566468324a Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Fri, 29 Sep 2023 00:39:24 +0200 Subject: [PATCH 139/716] feat: add manyvids bridge (#3716) --- bridges/ManyVidsBridge.php | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 bridges/ManyVidsBridge.php diff --git a/bridges/ManyVidsBridge.php b/bridges/ManyVidsBridge.php new file mode 100644 index 00000000..df4996e6 --- /dev/null +++ b/bridges/ManyVidsBridge.php @@ -0,0 +1,48 @@ +<?php + +class ManyVidsBridge extends BridgeAbstract +{ + const NAME = 'MANYVIDS'; + const URI = 'https://www.manyvids.com'; + const DESCRIPTION = 'Fetches the latest posts from a profile'; + const MAINTAINER = 'dvikan'; + const CACHE_TIMEOUT = 60 * 60; + const PARAMETERS = [ + [ + 'profile' => [ + 'name' => 'Profile', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '678459/Aziani-Studios', + 'title' => 'id/profile or url', + ], + ] + ]; + + public function collectData() + { + $profile = $this->getInput('profile'); + if (preg_match('#^(\d+/.*)$#', $profile, $m)) { + $profile = $m[1]; + } elseif (preg_match('#https://www.manyvids.com/Profile/(\d+/\w+)#', $profile, $m)) { + $profile = $m[1]; + } else { + throw new \Exception('nope'); + } + + $url = sprintf('https://www.manyvids.com/Profile/%s/Store/Videos/', $profile); + $dom = getSimpleHTMLDOM($url); + $el = $dom->find('section[id="app-store-videos"]', 0); + $json = $el->getAttribute('data-store-videos'); + $json = html_entity_decode($json); + $data = Json::decode($json, false); + foreach ($data->content->items as $item) { + $this->items[] = [ + 'title' => $item->title, + 'uri' => 'https://www.manyvids.com' . $item->preview->path, + 'uid' => 'manyvids/' . $item->id, + 'content' => sprintf('<img src="%s">', $item->videoThumb), + ]; + } + } +} From 2172df9fa2de2752396a55260f49594606466ec1 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Fri, 29 Sep 2023 19:17:03 +0200 Subject: [PATCH 140/716] fix: various notice fixes (#3718) --- bridges/CodebergBridge.php | 6 +++++- bridges/CssSelectorComplexBridge.php | 6 +++++- bridges/ImgsedBridge.php | 2 +- bridges/PepperBridgeAbstract.php | 2 +- bridges/ThePirateBayBridge.php | 1 + tests/UtilsTest.php | 10 ++++++++++ vendor/php-urljoin/src/urljoin.php | 5 ++++- 7 files changed, 27 insertions(+), 5 deletions(-) diff --git a/bridges/CodebergBridge.php b/bridges/CodebergBridge.php index c13ff004..9515d514 100644 --- a/bridges/CodebergBridge.php +++ b/bridges/CodebergBridge.php @@ -260,7 +260,11 @@ class CodebergBridge extends BridgeAbstract } $item['author'] = $div->find('a.author', 0)->innertext; - $item['timestamp'] = $div->find('span.time-since', 0)->title; + + $timeSince = $div->find('span.time-since', 0); + if ($timeSince) { + $item['timestamp'] = $timeSince->title; + } $this->items[] = $item; } diff --git a/bridges/CssSelectorComplexBridge.php b/bridges/CssSelectorComplexBridge.php index 4d44f853..c09db74a 100644 --- a/bridges/CssSelectorComplexBridge.php +++ b/bridges/CssSelectorComplexBridge.php @@ -415,10 +415,14 @@ class CssSelectorComplexBridge extends BridgeAbstract ) { $article_content = convertLazyLoading($entry_html); + $article_title = ''; if (is_null($title_selector)) { $article_title = $title_default; } else { - $article_title = trim($entry_html->find($title_selector, 0)->innertext); + $titleElement = $entry_html->find($title_selector, 0); + if ($titleElement) { + $article_title = trim($titleElement->innertext); + } } $author = null; diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index 70b79866..1fa5b827 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -244,7 +244,7 @@ HTML, if ($this->getInput('tagged')) { $types[] = 'Tags'; } - $typesText = $types[0]; + $typesText = $types[0] ?? ''; if (count($types) > 1) { for ($i = 1; $i < count($types) - 1; $i++) { $typesText .= ', ' . $types[$i]; diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index c152d249..280b185d 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -463,7 +463,7 @@ HEREDOC; ) )->{'src'}; } else { - return $deal->find('img[class*=' . $selectorPlain . ']', 0)->src; + return $deal->find('img[class*=' . $selectorPlain . ']', 0)->src ?? ''; } } diff --git a/bridges/ThePirateBayBridge.php b/bridges/ThePirateBayBridge.php index 27732db5..4b130800 100644 --- a/bridges/ThePirateBayBridge.php +++ b/bridges/ThePirateBayBridge.php @@ -65,6 +65,7 @@ class ThePirateBayBridge extends BridgeAbstract '207' => 'HD Movies', '208' => 'HD TV-Shows', '209' => '3D', + '210' => 'CAM/TS', '211' => 'UHD/4k Movies', '212' => 'UHD/4k TV-Shows', '299' => 'Other', diff --git a/tests/UtilsTest.php b/tests/UtilsTest.php index a7b41795..f851f32b 100644 --- a/tests/UtilsTest.php +++ b/tests/UtilsTest.php @@ -46,4 +46,14 @@ final class UtilsTest extends TestCase $this->assertSame(4, strlen(create_random_string(2))); $this->assertSame(6, strlen(create_random_string(3))); } + + public function testUrljoin() + { + $base = '/'; + $rel = 'https://example.com/foo'; + + $url = urljoin($base, $rel); + + $this->assertSame($rel, $url); + } } diff --git a/vendor/php-urljoin/src/urljoin.php b/vendor/php-urljoin/src/urljoin.php index ef84fbcd..026b767e 100644 --- a/vendor/php-urljoin/src/urljoin.php +++ b/vendor/php-urljoin/src/urljoin.php @@ -40,7 +40,10 @@ function urljoin($base, $rel) { } if (isset($prel['scheme'])) { - if ($prel['scheme'] != $pbase['scheme'] || in_array($prel['scheme'], $uses_relative) == false) { + if ( + $prel['scheme'] != ($pbase['scheme'] ?? null) + || in_array($prel['scheme'], $uses_relative) == false + ) { return $rel; } } From 3557e5ffd4c2024c36d6a448c520cb9afd5cc2aa Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Sat, 30 Sep 2023 15:03:52 +0200 Subject: [PATCH 141/716] [CssSelector/Sitemap] Minor fixes (#3719) - Apply title_cleanup to title from metadata (#3717) - Metadata: Fix ld+json object/array confusion - Sitemap: Also try /sitemap.xml well known url --- bridges/CssSelectorBridge.php | 24 ++++++++++++++++++------ bridges/SitemapBridge.php | 10 ++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index c5a09822..dd8fe228 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -91,7 +91,7 @@ class CssSelectorBridge extends BridgeAbstract $limit = $this->getInput('limit') ?? 10; $html = defaultLinkTo(getSimpleHTMLDOM($url), $url); - $this->feedName = $this->getPageTitle($html, $title_cleanup); + $this->feedName = $this->titleCleanup($this->getPageTitle($html), $title_cleanup); $items = $this->htmlFindEntries($html, $url_selector, $url_pattern, $limit, $content_cleanup); if (empty($content_selector)) { @@ -139,17 +139,27 @@ class CssSelectorBridge extends BridgeAbstract /** * Retrieve title from webpage URL or DOM * @param string|object $page URL or DOM to retrieve title from - * @param string $title_cleanup optional string to remove from webpage title, e.g. " | BlogName" * @return string Webpage title */ - protected function getPageTitle($page, $title_cleanup = null) + protected function getPageTitle($page) { if (is_string($page)) { $page = getSimpleHTMLDOMCached($page); } $title = html_entity_decode($page->find('title', 0)->plaintext); - if (!empty($title)) { - $title = trim(str_replace($title_cleanup, '', $title)); + return $title; + } + + /** + * Clean Article title. Remove constant part that appears in every title such as blog name. + * @param string $title Title to clean, e.g. "Article Name | BlogName" + * @param string $title_cleanup string to remove from webpage title, e.g. " | BlogName" + * @return string Cleaned Title + */ + protected function titleCleanup($title, $title_cleanup) + { + if (!empty($title) && !empty($title_cleanup)) { + return trim(str_replace($title_cleanup, '', $title)); } return $title; } @@ -270,6 +280,8 @@ class CssSelectorBridge extends BridgeAbstract $item['title'] = $article_title; } + $item['title'] = $this->titleCleanup($item['title'], $title_cleanup); + $article_content = $entry_html->find($content_selector); if (!empty($article_content)) { @@ -484,7 +496,7 @@ class CssSelectorBridge extends BridgeAbstract // Now we can check for desired field in JSON and populate $item accordingly if (isset($json_root[$field])) { $field_value = $json_root[$field]; - if (is_array($field_value)) { + if (is_array($field_value) && isset($field_value[0])) { $field_value = $field_value[0]; // Different versions of the same enclosure? Take the first one } if (is_string($field_value) && !empty($field_value)) { diff --git a/bridges/SitemapBridge.php b/bridges/SitemapBridge.php index 482cbb66..78526e6e 100644 --- a/bridges/SitemapBridge.php +++ b/bridges/SitemapBridge.php @@ -73,7 +73,7 @@ class SitemapBridge extends CssSelectorBridge $discard_thumbnail = $this->getInput('discard_thumbnail'); $limit = $this->getInput('limit'); - $this->feedName = $this->getPageTitle($url, $title_cleanup); + $this->feedName = $this->titleCleanup($this->getPageTitle($url), $title_cleanup); $sitemap_url = empty($site_map) ? $url : $site_map; $sitemap_xml = $this->getSitemapXml($sitemap_url, !empty($site_map)); $links = $this->sitemapXmlToList($sitemap_xml, $url_pattern, empty($limit) ? 10 : $limit); @@ -103,7 +103,13 @@ class SitemapBridge extends CssSelectorBridge $robots_txt = getSimpleHTMLDOM(urljoin($url, '/robots.txt'))->outertext; preg_match('/Sitemap: ([^ ]+)/', $robots_txt, $matches); if (empty($matches)) { - returnClientError('Failed to determine Sitemap from robots.txt. Try setting it manually.'); + $sitemap = getSimpleHTMLDOM(urljoin($url, '/sitemap.xml')); + if (!empty($sitemap->find('urlset, sitemap'))) { + $url = urljoin($url, '/sitemap.xml'); + return $sitemap; + } else { + returnClientError('Failed to locate Sitemap from /robots.txt or /sitemap.xml. Try setting it manually.'); + } } $url = $matches[1]; } From 6cf9dfb7c905cac989aa17c5a694ac2fd4e35cd5 Mon Sep 17 00:00:00 2001 From: vdbhb59 <60728004+vdbhb59@users.noreply.github.com> Date: Sat, 30 Sep 2023 13:06:48 +0000 Subject: [PATCH 142/716] Updated public hosts page & updated welcome screen image to the latest (#3707) * Updated inactive hosts & rearranged alphabetically (country) wise 1. Moved rb.vern.cc as the whole domain itself is down for sometime now. 2. Sorted all instances in country wise alphabetical order for better alignment. * Uploaded latest welcome page snap Uploaded a current version of welcome page, which also shows "find feed from URL" functionality. * Added a public instance I stumbled upon, home-hosted in France Added https://rss-bridge.cheredeprince.net/ which I found to be self-hosted at home somewhere in France. * Reverted the sorting as requested in my PR #3707 Reverted the sorting as requested in my PR #3707 --- docs/01_General/06_Public_Hosts.md | 7 ++++--- docs/images/screenshot_rss-bridge_welcome.png | Bin 73267 -> 48723 bytes 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index 531e5383..9aa292a5 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -3,6 +3,8 @@ | Country | Address | Status | Contact | Comment | |:-------:|---------|--------|----------|---------| | ![](https://iplookup.flagfox.net/images/h16/GB.png) | https://rss-bridge.org/bridge01 | ![](https://img.shields.io/website/https/rss-bridge.org/bridge01.svg) | [@dvikan](https://github.com/dvikan) | London, Digital Ocean| +| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rssbridge.flossboxin.org.in/ | ![](https://img.shields.io/badge/website-up-brightgreen) | [@vdbhb59](https://github.com/vdbhb59) | Hosted with OVH SAS (Maintained in India) | +| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rss-bridge.cheredeprince.net/ | ![](https://img.shields.io/website/https/rss-bridge.cheredeprince.net) | [@La_Bécasse](https://cheredeprince.net/contact) | Self-Hosted at home in France | | ![](https://iplookup.flagfox.net/images/h16/GB.png) | https://rss-bridge.lewd.tech | ![](https://img.shields.io/website/https/rss-bridge.lewd.tech.svg) | [@Erisa](https://github.com/Erisa) | Hosted in London, protected by Cloudflare Rate Limiting | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://bridge.easter.fr | ![](https://img.shields.io/website/https/bridge.easter.fr.svg) | [@chatainsim](https://github.com/chatainsim) | Hosted in Isère, France | | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://wtf.roflcopter.fr/rss-bridge/ | ![](https://img.shields.io/website/https/wtf.roflcopter.fr/rss-bridge.svg) | [roflcopter.fr](https://wtf.roflcopter.fr/) | Hosted in France | @@ -17,12 +19,11 @@ | ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://bridge.suumitsu.eu | ![](https://img.shields.io/website/https/bridge.suumitsu.eu.svg) | [@mitsukarenai](https://github.com/mitsukarenai) | Hosted in Paris, France | | ![](https://iplookup.flagfox.net/images/h16/NL.png) | https://feed.eugenemolotov.ru | ![](https://img.shields.io/website/https/feed.eugenemolotov.ru.svg) | [@em92](https://github.com/em92) | Hosted in Amsterdam, Netherlands | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss-bridge.mediani.de | ![](https://img.shields.io/website/https/rss-bridge.mediani.de.svg) | [@sokai](https://github.com/sokai) | Hosted with Netcup, Germany | -| ![](https://iplookup.flagfox.net/images/h16/US.png) | http://rb.vern.cc/ | ![](https://img.shields.io/website/https/rb.vern.cc.svg) | [@vern.cc](https://vern.cc/en/admin) | Hosted with Hetzner, US | -| ![](https://iplookup.flagfox.net/images/h16/FR.png) | https://rssbridge.flossboxin.org.in/ | ![](https://img.shields.io/badge/website-up-brightgreen) | [@vdbhb59](https://github.com/vdbhb59) | Hosted with OVH SAS (Maintained in India) -| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) +| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | ## Inactive instances | Country | Address | Status | Contact | Comment | |:-------:|---------|--------|----------|---------| | ![](https://iplookup.flagfox.net/images/h16/FI.png) | https://rss-bridge.snopyta.org | ![](https://img.shields.io/website/https/rss-bridge.snopyta.org.svg) | [@Perflyst](https://github.com/Perflyst) | Hosted in Helsinki, Finland | +| ![](https://iplookup.flagfox.net/images/h16/US.png) | http://rb.vern.cc/ | ![](https://img.shields.io/website/https/rb.vern.cc.svg) | [@vern.cc](https://vern.cc/en/admin) | Hosted with Hetzner, US | diff --git a/docs/images/screenshot_rss-bridge_welcome.png b/docs/images/screenshot_rss-bridge_welcome.png index 3ceb432ee2b4048be7da4643723ae5e90affcc87..9a0dcfc58dfb8d648ed58cd514ca51bdd62b310c 100644 GIT binary patch literal 48723 zcmbrl^;cW%^99;MixhW<;#R!46(>k=*A^)5?!}9{6WoG3#fy7zDDD=lxWmoc&-dPc z;Qnydvrg7INzR;^dG_qt6Rr%DLHj`b;mw;jXaHGB)i-b8(O?H55*+LkgtV+2*z(p{ zRYv?x6@>H<cJj_#Oi}F3o7z~Er?2l}=g1DSI?iw2p!fcFydAVJHGT7@;{YHjrtV>Q z3<?h-mjb=q<LrR|Z^W?rae{cu^UJHs;aXcg>Y!fRwXH{iRc)tLtqCo$q%@Ro#j%mz z!69oQ#eMlj8BF~}_7~@w?>Y`1)>HUPHq^c&+3WG-7UaI>X<ocPewml1uC*{XXK?Mi z28Es$pks+6Q3hkOl_vhTh_~kfsbCAzd8j}*Y@zHUfztfD!2bWw1=Bz}I=T@YoGpJI zmDY#aC(q?+imuNud$gM-h~le}gYc1qyKJlNZVxosqAz9Llf$DuB}T;06DNn(H+xh> zzBi)}M&d~6>&o=;O)6NjVK@Eq`?m6(<{F>a<B(A4;C<tNZ{^yW9SV$Ghr;<z{WdNu z7xh|_3O70%6Ut#4O@6cbi5zx+3GDuxk2A9A;w`T+SJpD(0eLQ3{uL$eq*ZNNB<c`) z*C?(_fLFb{kC;(S3GW|n%t5fI<)`+iHHF1=wS=x<Oim@3eQdvcN4pt}Ft3V52<0+} zl=#MNOWm4l@AFp31+?P;d@L(uf8tv(=#XRZ-!1cCP~%KRqzoRDDNP)5!D9Z!tN)bA z4ft7D@EGW|Hk7WQ>b~+|Qj@Vty?7e0e=ECoFTd~>2sFc|*8+W$q1h~SzeSKDLJX~^ zFd%_h?=t~1?7IX@6J<&GFxfhsrD)jGbl-64q-!Bvg~CgKO&%mDIvF19dFcqw6r$IX z$=d{jx?DgTr;w<XWE~&xhp+`&pdOCFBtg&NsV-`=kMdp${#CSuxbM!E#UGuO@kY7e zxO*g5Y)dlFRu%(>oLbfokx23C9C6*hjko%yh9>NQBE+o8zr>4X7!^sqjN7l+7HDHg zB3#h==K2yclu-rh<<hH-$OiEyGZe(jdeJk1NL8&?B@z`wFReSuAXOr}{3Y<;^y1>l zUWj?L<j?;sT1mZavrcCDrzgZ$j<Q>TX@)OJz**ucKDTv~>Sw;wF_A%73zLcWetigr z5jF-Y9Y*AL4+OQ;W60ODLhIJrF750enVq3=6{QGBlrzBs;R=x@--@&_!df4Dzv0da zr*>b78i8c@=it`9g=<1(-_jH3zsq(gegdP_{x%lJcQ412hd-AZrJg7$ZZ9Tr<mz`4 z(n2?<%a%{hQB2v`;KyX!>{+H@?mdabr$8ad`H73sH(O=giaQ}@c}R*{?By)Zr`P$k zd0h6G+%|dNcF{RD=Y1LOqbRj3SIqZzx7Eb|<)#FMwR!uCU;D)ag19U^%o!T)4x7}C z8BQt|G^xPW?B~O-wJu`OUCtIML;5OCpX@*Jo_EgGRt2QV(z<%0noFNSjJ4oF?^Jxx zgBCCC<UsY@_eIRxIcu1Lz6QtC1Tvz(=)9mKLxNh8YqeArEAKa@MtkiIumwCV2~R8K zCF}`NIy?b)NxN9bUS}hJ5jAB*{49?zwfg*Be4U1c(+wu@Ig``Jbu>>dnBPvHm{^;v zn)LscWo6GeBOl+kH>RN=WcKk?gHx5JY9`>H9;7Ntl<o~!`<aEbJ^S70O4U3e_BknT z02X5No~$}vj~B*Y<_oS6V6ll4+8Y}>*YI{d!?};j9A=426h!0fxq5p@lIroSM*O$l zaw@enk5{Oy@o?~^iz)iFoG^el4HcEnAsQ?LMQ=O!x`0Ta0<;}a9}AE)W+)=>BZ0VF z%XmMC)3ZfpxjR2pnO~|-Rv2VCsa-$^>jI{FIG(#heCMh}f5Q8)mp;fZGb%CkKWLX; zJ;Y-Y4@%Hq%PgEisDPHzBlLyNxiJ58o-?^W%`<gB6HDvO?%y^D0_WJtQ)P82$2yEa zDyH3==;rhb>K!VNe9SV>Jfs<um%MAS<5(1YcD9BBH)p4wp*k(^p&&gm*}oBL_B)(F zOE&J1!=LPf8gAerpzbxnCPQmE8|t{rmGKVZLQx>2r&70#e!l9+h4=JoFXJU)t`H{E zw<2$Lzvo0?Cg1t;PttXhO*$8BZ)Qrg{{rOljJL@6&1{ctT7Lhw%q=1X8i9HL8!RL0 zl-qgu5R*`xe9K&ekKl<e;po3!p+o|Qy#!Ln+#AxW3d1d`;aRHo6$cvCE)N_1;U!yj zs#XKl>gh_lZteo?J1#hFnlVi%sC~^sc`m9tg=UX&`I$2SAe^`?87$2lJv|rIe(53( zOlBljmIqv+JDE~{ph*CEoQ*gvyeUk`pnwvM``bFhZ8i!3w#=CXkk4%k)>uo*0<yxU zDfG$0cl9x#<48@wGGPXpioPZR0q{vO?1>5u{&~ubwycOlxgRq|iN8-!U7%v%SJFLf zI-K6mLiNTCLpD@y4;9hpr|}i_r1Z1GSt8E9vzL<cxg|i(L<#m2NM!>8(5{s%yU9`U zlPkZ!w-*LNX#H!Wu-OQR-!wk11gP4@iphyd0Gf51lLkr3w*7n8$6sVSgFi?hEnjAW zpjC1~Zz<a3(9YO!ziOAJ3&82rJ+HDr)p)4BwIU=^wD2W3We2;jUM5r`TmlUCDbxr2 zXcmOC&Yg6gLR7UPJ0DFZB_3Ea?|Y1QtDXMLQ3zXpcKNdIt0pypnQ$NRI6e83k0I3# z2jtQ#a>lLqUvi3b69Y-=4iY(Ie7iE5bjReeF-|3R6ZR(>#gSe(9e+Nd9cD8mdW1Bd z=h(~NoXJ)DGI-B)J_T*}n<t)@*ctO@fs1*TF*pVJ=$|IL^iz+C)#aVN@F{}n_-vzF zH$xm2bdtxtE&-5&q)p!FkWLS?Eef7K5$E(Ui94dBZ~tu%F}s`ZJGxKf+p!TDaN){1 z6HSFsyykXWw?wY91Bnd!KWUvQD(%8k^UDgWp&1a7YWZJH{f+W@jro87z083A^<{^> z5PKHcpCZ)Q{or`*f=Kc)H?~JxEqfaFBvmEfSkpEcl$ywDBGk_9z{$g-ZP9jC$4z*h zp<<uIn$<5$=?!M4(s4L<u<c`(+P(vE4z$?aho~j9t<zX=e_u$xU&U%V-dqiOs$zBW zVAWZ>R`@!+n(kg5_3mL?7y`BAil@%b?mR!uyIQD@IWpx^XjOih=RP+H&6m5>C<}-n z6U2SK9}g$;A}BWnfFy^m{tF`$1m6Q1bCKaGfMG%B7`;KA;4)W==#q{M+;*5c3YmW{ zZ{eB`S7E3B^c+EVmDH%XXeXuTwF!R<Kh;MJAlo>B4XAK0(IEv9r=&8buTCvnNq4eD z6W8}L;~lPZw+dc1MCvoI<4v)L@kcxIq2|kAIar(^$<5Z9XTyhI0I%_ndu$&s@4N@O zLd>zb?)In-9cKZ-3KUS9X@T#5D(3-p^&ireT6ek?ZY>P6AMm4T(F@#SFUQrnoi{2N z19VmEEE#-9-cQ-v#28(Ah-jsM?Wr-e#tmOD%~n#K$N=9La!+{xPaHsEixt(018roW zV7};eb;dwsg3LdDNv<Ooajhv8k&10=8@2FxKe<}w;)eo+@mW>ru{ag~i}tjy%>Yot zO&+OBg6@?q+M+|#cWiT0H7%ph>{+K!v@UBYj|0e9XXy*FVRl-48(yJ7gART4PR0~~ zhcnOV1$h0<ZpWB2N$9-U`kc@7-PRZ+xsde_6uE|oMeD+kO4X}Zn_-fq7#9C#|Mzp2 zprjC>K&z-JwwC~P=0^7q^u~8x8F+-lovxn8E4E-?j)Ih(b<Zy-JdSS1&678EdxrG< zx;uL%7pu^8pXSeDN2h3O&-!mU%=6Qe88QNFY%>@)z#fOu@;zL<IXk3#s1zjx&Kj)z zoc|mkbd}C^+RQsQ(Fx@-P#T%r!Hx9pD-{3c@X?VT716(g>)B@)tA&V+je|Iqo~{h^ z!3H&^nJ6fxcC7{Qc~{KHs%ToEIXwf6P4%0KM~#^PsKZpbt&Z|pO($oJ{zOFEW6l`% zfC-N+8dYrVmzGli4NpyoQ}WCs^vAPB7DBCl%OY?v4V84fFzhso&iC`h?V-mZi+nQB ziLrmrHNn}!DI6O2JuV~7#Ad@KP=N3Lvz2xxqZD1C=l|^Gt_q@&w%(?036Y;g6f@|u z%XEaI2|(_u>;_oSVR-aR7jOA*&BI2^$U1Kj^;pmiKkjJQbV&HQi>H74lTX<Q>#uKc zQ5TC_lC8=yyL9NqBZ$~V%B@*L$&??RXd8oDc6%!tLrS%am8Dnn7sK}xip&#s9o(pr zWpsA+9pSu77W4hml;C1RP^u~LKs$!x(Z2<AR0J#qa`&p5q>lQX%^K(-e2@Zk5CTJw z9cxRz!&e~?gH#<3bcvyr4OJaLE=Z=7y#4_+OJVy}1bL*?jX+$Plr_P$lN6GRzH6Rn zrcRfV^SdZgUVSi=hmUGODo`z!N*Oa^nSxC?^HUxi_({_6I<l-(ZqAa8()NcyctZ@4 zTV|G0>(-S1#7OFFLct56-}tEY8?wWKV+JAi2!j~Q&-^6jd+>erm{wrXC2q{u0d7V@ zR8tWN3F^;%iSbl6YihOFD=*sj;j$c1p$4;{>G6#ye>Y)0Z8xPoYc2Y<E`bN==H$7= zrirvF3~-GKsu}N@W!5me3ng~_4LI&vO?%kFJD<^%0TQE;i4ywE{cyre8bp5=OM;Zm zPU>};@Yh>2^!0S>K!;@8Xg%xhHdD1jaE9OvV(Tg7{x}mN)7vkrnEXzS87WEch9G4v z@%d%GPt=h7F%yNFKL1+~rN+GR!JoGtvwPNI67~l8e(NFeO8UglJp+C2Rf<nGdQJ{n z57VoQN8}S?+CoIk{q<gS)o(?n0N&KJpBY(s?n$d5^$RUyUP0c0r|tyZ2V44)Y9-6( z$C3D#QtmBoQXgfiyg6S-mpOA<KDk}gu`j8CIyc`{y*&<b^{3MRVVNp)9kO$GD<wZE zjK=QTc6+~-&{l8MBA3IL#imcr@P1)nIVD8Y$#G|SfzHv%el;iT@0=w%ncUvUnf_)w z#n*^Sz+PQ|)gYkjp!-;<xk$v{6E&5AH5Nxk{MV;BvmklgIYLx2UP7<lv{p&cMv)t~ zh4PGJkJg%rL$<v*Tlz5nk5!T_xTZBEH}T5nBQnmfuJRA^nOhXvWWL*JOh_O&pw}R> zM4noTj7TTXL!~u#uluf@7pHkwc;#VaNSyS4V1uLgv`_#bZIBj`vzqX@G<P0rfbBVx zP~!#SO~B}br`FLP{Qd2^_>K1$_m0$wT~ceW8f%7*gTRB74y$jLR<{Tl+oPFV_Bz== z7zI4RPLU}g>po%+`w!&=t$fZycL9QKN;g;6$gcK|G}Y)I@|6|8M<Gf(8MQHm@^LmI z9Q}IgJYKJsFXfe?-cuG}k1HLL;eOrxW!#UWx%oWx`-_5<L=%+k17x{G+R5UB?Ax>n z|6iYOT-rze%J&BzPFK4XuKTO#`|=niCp(RZ_d}E4k4DsODf2LKjt~!n>q5EsSks1Q zSb;BpTTOdBO-zM-oYqhBjL+}hn!`<xa5;3&Zz<{7gW?1}zAUFm?dX%-&YMjwqpNpt zFVAjupHIWQ*k__ln_$U`ubjA&Eoy7FNHFf#)>m3dazeJP44nocn<^^DKR!nnd;%e+ z8nFfwxVd?;Cm3Mo%4!CFN2ie;pZ=P1@6YRVB!25q!m@V;w8QJo0)ZKyz9=}~DIU=L z$f0<3wubf+49*1+6x`4f*H)1_FEVhHv1R7rT45icbj%6-(Y>87iodOj_+=RL5iQcK zb1VL7pLf6zF}mEaDJIml>Q8S+FKe^YokdQ-ouK9yPC7epT?)UM1r$ox|I6<H*94Tl zVf?wM=kdo2YWE_JD&M;;LO;f`c=VeAEc`kjZ&gIl)EotD-(X!!$L89r>t><8_qj?S zl(s5jXIHqDX=7)N-GOLRF!S}c@&7qjKYRj#n?E5(HvLEsb`9tl4ERmM{Hl%BHAH)1 zlD9RuAa+!kv{>F3{6&4feD6)PnhyVzq+9NIJf?rGe8-|E`}@^<Qw1VnaLd!`jrvOx zUhQw?V(X7FJFTCtUqZ^p3?Qtdo`01`RQHT5V`z70=7X)x;cWKw>4$Om$R>8$9t&0> zdfOQL;nu~xPbo!T^`L-(TVW%0^A@&z%^pi7!myfS!)rzIK91e>OZ`ofg<^u6kQxH5 z+!<x2eK{<s!LmF5QQw(g6(J#W8FR@3R<0AJhLax^G<yac^T+SbWi<vkQG_u(Gu{}+ zQGOk!z_Q<8+O63eYeao+=D@4l{ao`>$)TmdBA%lo8q)29)pQ&g6=BzKxhcL!MWcMC z&JS4KC%dz?7<5X6AC|P@uZ#|Ql&X|muE7Llc~g^R(UCl1v3YO&9a-aQ+So3Xv`QwY zvTQ&r$d-8ja8D_={CpI!jQ4xdrCC}jH^yG4i`iE2rgAPtm)y({5xh}yNd1%543U5a zX<AX<EJn2wUe?z5*~{T&xNGfcK(JR(y$L+Ksr#+WTBi{1IJqHSt(*I)Fu}ka$$)on zjDcvy-@+al>N)M&h5MIP>bxz+(K#y@brJ|(J)k6JVM+4w7PV~IZF6Lm7reysh%y*W zXUtD3@67j-q_6RKix7K?!Adv|>Hu_%wTf{)?sP6^Al3^Pc)HFrwQN?arSFvR_8pHE zqTf<{iuj3(cv&Gwp^&|Aq2D5dKbGY0BysU;v$RFh052t${N^g;W>6#0JDb@J!yNyn zT+P@?Go5<>CGCKQKsEifQ=`MPv5l=k*}zZn+2)4?BLuH>{%^;3zN-fnb7+wOx?Gqx zcn-iokSY2L=$kxYkyIA!dK#7K;GHU<aBq#?lHX2n6U#;#1!|(pN73AuC%c3q)>RXS zyQ49D-S;Uxa|m^Vu$m95l#KC{VW!$^V=uRBk(tbBp;40mmm%Y8k*hpGrtsWB-B?|Z z2IZnT@+m26Abw|GTxvE2tQRJDYBL>#VK+_@Se<3m&WUyV5U7nJi%!v{z6$N}uq9~D z6}?L0a=tWexPrarIkvl(v`g!kztx7`+FAiYjvVqi9iNB^A>BaIHS2zxZc6^8E^Btr zfrm7oGbt<bam99DPsw5db@`^=yiaEJG8gOX0^wHadZ1v`{?jd{52&Sw(w1-UnTcM$ zmn4Wdc>_JO{PPl5h;fsN<os_qT_<q$pS4d<to(qiqfkqxZIw%nJXi@72!mnJl%grC z0dRSKe&R6#R<i#ld^#)&OV$CqNY!*?TjfWShujcbaE;GS6;lk;Qe|S{%4-N8$#7ni z6aLOUvLK}!{{uj1OP#s9pstas4K(s0Ur}+~?OF!jWOqVcSA*F9UKpsQd+dW&<5ZX7 zUhH|Tb1M92j)32by|q7H&J6{JllDIZH&j{}TA1RJfF{jB<01^#bopiTae+4WP|H~= zMyF0erQalr6P3^~tp3%*ld4KIhNCi1MRfs6qh4VB(Gf;~9pwyld8TLFPV)?xMV*kg zmD9||!1u#~v?|~ZF1F<@S)k7+LG`03Ggb8XezH}HW1?ZdYo_OJPRo9?+n&kdR)X@0 zYE9l$CKQ<+R(abcNWrJR0H7{=!u_tToawCI65LXGqJwT;qJuu9R}uQ=h+lnp)@oHk z$8~1h(sqX$!hf^DAu(kzQI`2lmg;ATCOtp5lAnX(J8n^XJyl`hNeU^!1G;O0Gudfj z%-6|`pDPP6xWITob1p>AlEat(o%K3oUn9a5VH{aCHJv%8!!k?FB^j6?gL3Lw<>BIA zTEZ6Pqu&@J!6>)8kxxAA8f)sP9PrhAz?vo!Gwa}o*V_OqL#>_&l&$7I<lgSviYOm1 ziDNyzX)Tem6ub!aj_WTfzHrqOI%8qPnpuDt8_>1}@3jM#0MM+IxaomiQM^wkd5sT0 zdrGM<E+*_Bd`x5a;07nMGZ26#<USJnoi8MEHW_>IsSBCQZyf~4rkaQyp+7^`L!PZR z_xE`QzPgyp(uY6(p{+HU<#OlXURO@95hIgKQXg<Z-6~ae5c9ta-ke;MJr`{*X$|AD zf|d*LKFw-M@kR3V{Vk1sC%7k*WwSLoYiO2}8!+o`aP=XfQ}*_Pz|R51wGBs`r+YIw zxFFpYrq-CDKrF|CD81Bcb{L3GaKg&`9x(t36lOJ)sLyI(J;Msx@j=uwFj!wraM|MN zpE*~Bk*GyAW>~(oynIEip{lTGD=W~9lYr$6h-Y~F*H7=jxP%JB=~BP490fD@w+B(* z5ulcIx)ePOuv}y%00RvJUg>!^XH5yz*UXebe{=0}{P=_9zy~;=<CNtCnGykIs}(u; zP0kTj(*V@~{}vG<!eN%awGq@T8ZCeCsd~<85@cT++hcjHfhpS8lhD4F<!R9tj{6Rl zb2cNQ$G)N|-e~ZgZmEQAo}39<kw`5M{^+pG!_`~n^*%`<t%u(sR<=35&}cQ800+_x z*@h|cTkTr!io%zey}8ZXK^F|CX1pb_XSW_lfMamOoBwEPF2ZnHYg-ZI;&*%Ve~?L* zQO|r@|AI$n3k(f9DE|P}8XO1g7&keyo?}cjh3jpx0Y{aI5YeA`Iy6{3CwWBh!hYLj z{3O>KN=sP0>8G^2CWwLYD}96WJxyYW9?u)6TCu{fGaBbPw(>9E&CA@)t<F!<H`_Q1 z+|9l0m3mp$(jVq3!A3S=@-Ktq!#OK)Pu<tU_C}d^`{g+1Mnr;_7gzllO!f3<HJ)@T zen+a;u3uy8FsCVe7e@Z_R)@dt-_X=a?J48IfJ)ez0-Ll>*}>K8O2*Tfo~ncBYRXeH zN#go?#@&45aJ5Z@|9QlnsLjul$q9cj-9=&aB_IL+yT|DEeRS*&k8sMZQwy4R9l=`8 z{ra-1Q45pbh!f#zQesd7@hZe8+w{Gn=aa)uKyHR{4O^R+0DwQsuJm0a`HF3*<JXNX zE$(1E3I%b^d};>g&aI*|{c=<0UuVW(0Ep%~V90y*r&rBrjM2G75m+9@f4z<KqS887 zUG=rm0F--X2k`Ooy|Nwc8N3Rc&2T)xLqeJz8R_FIvAFf+Wk_pgi9Mi+BLgb53{6=m z%gUbu&yc;^sl>j(i;y!KGzdO6nL+KUAC1qOd9N}y|E~RPXqOAzP8R8e`q2&&%#Vff zBV)LM^<4Q%DD_W*+Bj213abLfy`57<v-JGz5f=6>%Qc+AMkw45cqPScq94*DTftPw z=>oHWLf0|l*+QV)&M1so>t6NB{k!4e=wHEbWQj^BY|V|sqnJH<HlEFne*}%q%vl;( z+`b4DdAd=2z2IrIYwgaGF)1W=-AlYu??N3Rj7%;HUiS<{?sCkF0CuoDJv{t@)pK{} z&mrZUqOBrW<G`#sRO(K>wvWuDQwvoQO8fN{#dABX16Af92GR0yWA(6w;cJb~hK2%f z(+i>nkfSTKhI3bQ-$404`*mH~5Hx`6T@3a&WQknEsd57PtOv1w!71h;SYp_NcKF(8 zlMmt*k>F$qJHCFtY9^2p?V!B5kFI4iAbQ;5F1q%<l2^{mQA+44rx+@ecD73+k2%9F zaXN?vwTQUw*T47NnwF|6U3-1UeLs^q6IV~q7UY7&1Tr@-iWI4U?RJO*@=%8|lQQQi z0?%jJgU+C`Y`6#Y+9jge_qnn_&4rdn+Kr(PwyxKPMKvde;bI#>)>Nxd&#_*OWGjaF zSpvW3gg+s-<|IN!;U8>#shSVq_8FzhB^@CcC7%n^VibpP|HJ3b9yA#W^ni`k3B1{f zl_|J^&TYgXSS2Oe#Pcv$z7^Cq-ImSrz0$Epb0zHuB09dPORblTeC#z%%rYEUj2}tN zp|So=j~P~`r0QJT8C_RB#}%{pu}}o6T8D+_&LD5$yl7I#ev3GtNjiEM{P{G=ppZz@ zd-&Ikyh&<^AjWPJO2fB)Fg$N&Db`Ch-&sLIahs!&nE9zL+zNqIeP-d`w6Rq)GS*T% z9W;wnV4$Fnll5_j<5GoYO3lVGh22iK<gIL{HQ^Sc*T4f!QXJ#dfMt`k05W>fN9SFw z=8rIHKJ3NL*5&-<5LS!Bh_&L&Q-z`=?;Ne$P#8xB7u;A@-zbhmzKiMq(Q)s=qSb+q z2U`GQ(BQBfkniP%OEt};WmX!|<dSfCb3IC-az+7ldsh_BNVUBH$#D0i_6dr7n;NZP z&e|r^u|Arv-B>4%f?D!;s;mPbPna{9z?-Y1rw#!P<zuyZOH>nupZe$mnBU@k2~qj^ zldD9;iy&#mu~&sAo-|4gQ84Xp;CqKxY#dnpKR!B2G7(ysaOr({WBvi<p_fp2ZYX3^ zfvx7p+sNB1G1K6e=i=Gu$BV@Xq8daoF~271`rT@SqEjozXe!szTH9~PYYm@oOx26P zqD>1@39~Z*!udI6k+frCxejxU<G}^`=9AHqOFP>g)FYh!gx0RVrVYXCyX?p}XExUk zcVCWK<Rvb|Pw;WWQ7rRWyq$rW>f`iacU25J2T5RGdJ!zw4Xg+D;$3ue$5$JGo(`LK zY?TO#JcZd*kZmdwXP+xJKT&0MvLbk1S^TNokMGKI<in-)E@=>9)JWs=)cLjSDL2)p zp08gyg)EMEj9j+6e;UYAxYAzMo#$J70r<7^Ky`C81najFf@qxMOm0gT_?HM2Qk~=3 zj!C*){nto=COf~+BM*L27EP3~Uatimob9i<+=c!!y^GYFB^G2{_>3h~_CAzr|4jYd ztVj!<ro7rdv`@j3#k%FRbLdfv!VmE+rCQ)W#1Wd2rL^RjyD?UEL@`Jx@l{BD$uMeY z`(ml1s&hz&6@a=D&%>KH>tRocpRX<AvZe^th{n{mvT^l`YCVoK$yCc(CX|U7Nfj^t zfxB!ha(M8vv6zXiC6qzpPX3B-Ol$1Xr%Tw^C2Ht1gFR@mm_o>ZD+S}GL^9Q7G<_>7 zm7P9)*!kZ7_fLj`qA#dPAPT8U(^l|g8o(mhLO3~_qfc<uMeY1J^PUTBW&^vjBvIYi z9?9v3pt<nOV<2O~QCAg!R_#nrsR*SB_qnZyhFYcyQ|ysOVhc#5vl(KgN<Ab}#gn61 zTAsH40OhlQc1%59&>OK=stQT3twRH2gI7jdpp+`vQOWD=G{@}%Pk(W9A2hh)?z%On z08H7EBF+Ztjj`pScgrLKj-M~I389+YbrHH0mMKNIh2GLAm~L*GF(}!5*t06g=~VyH zJp1Q+p9CDQ(H9@yr18z}qnh3tq!JtC&{45Q!^g_e9>Qx?NC)b|N6UdDqqSo49!`AO zJ!o`qu_}$kaPlpnzuiicBK#Dbm}S(ucxLE*M@c;RM<Kyqa(-CTKS`a8&$RE0%`>@K z!@JL^=3G2MG%NQmn}pX9#Z1oS`t}ENz42inZ=W|BJm2d&8hk~N5x0ciRk1b+0Eg`g zlO;qxVS-|HU9P2!gmHz;tv{6YGtL+wS_VsB0(|3-$QR^e-YnVH;Z$APsL}%{bZ(WK z%GY&b^?iGB5ebzIDi&$oow!Bt-CR`1D-BC5^ytAuU5j8tBU3GG#i>;SzY_<f`%;C> zFH1XMZLH@7PsZNOl}We*xqTaz(dDTS+C2}it&>rSyL;0d7cx)2OAL~HCwebcrpi?0 zD7;vnnXwo(c*s4}pc*^z|4Rmhee^Q`=-IduxLUHsk8fUpqa1Rj3Y8#`vR~T&`e!4x z`&raplj5V^B!#L;tCr>}d@V0q_;D<CD|tqyB$YR`FytBP(RKF3$yyecr7<=jyLo2B zBCzq7FGfI~cD>=V*YIi95yZd|8VF#5)r*DX)2FemV;H2wH?t_z3!>@G1XI)+T&mTc zOM*Ft^NziLV)v$wUG+6PdW?te5p;~S<<u8~s55u2z}0Q2MMm7DuU()pTCmtsOD%Az zNomzD<BB{^>;h)dP5s`PE_q?ZTy=3*Q`xomzIjJE4s3x{gGnw44X+!lb6#!=Ptymw zq(a|6mKIO$e8@%xFTo?uwU$;vQ#nf??Si0FLiAgk0|9=n<2l_d$0TP&!_R%;p9*A5 zXdp{rb;8^NfQ;uq0dY9-af}lGOQtZs3$OKL*yq;4EwBBX2X@I?D4bN02%DGRUsxD9 z#W>d4=cEVd`F{V(z!E8K?X0=kyGUVF?bkmd5JeY1ChVH<;3%5)oc&@}k*^qm`+Q=J zvnm0&R#81_u}<E<pD34ceK4^%ARfRtb;U4qkBwXvLgkr?Qp88ptIDP%)=ac5UQ#6; z(Xf{|MeFc3gK-okJZIy=<Wn>92)MI8I@t}5A^_ynP9)tu=FD$7T~se2#JBx0J;5^J z>s9C<dba3Jd)F)1UZ}M@!$P`p!$o&^FP)PbF>gFu*)9}#|H1Mk^mes}SJ=}-%#lkr zd;E}QZsYIMV>F=l%FPt7p4X*E+2QtXGYm??MGK;dUl5@=`h`JIsf)TWVQh75H`i#O zRN_VD=<v2Y+_0QhC4a<<?>UpKE=G^yY<}=EK;*jTJ<bf@lLw50ufH_MHvg@XUUj_< zBtYr31h=j#$fEqCbg8Fcl<w8I;^T#I0e@pmTS85(nn`)onM8U$&p?h6BlXv19uJO< zVicT`YPB1@MXSZHg4`fX0XO!#WluBUr7Zuo3wDgVoB$x-pnk!#1+o=XGp{<=%QMU< z{(Z`L=-l}U=V<HIOX4CzrY;XEvu}(|OoG1nXtbPI<FMMH5t}N{cvn6W!bzeTo7?61 z>FV%$%4j8_)6<_)=Xp{S1xHuB4YrwQ4}a8sYy(>JA}mOM7Ce_N^%!4!a&Ib(ISu=* zO3__u^_H8A@R%Hes*PdzmVa#E5|E#mA+-JF!bX}tZ!h7*y_wJ6%SD9W=35zp<L9f( z;qTR(jE6Lg4>9H+7^Doc=rTUPd?8JESR7%l1R;*kYHaeIh#~dt5^l9ztQ(S^kVh>m z74|TKJ}0z2VQDaIjhy|s#@Nk8nZv+fF8)>3-yMyyk6F1PgfR7cxI1uWp}%$3+^X!N zTZQlF*Qz}}Z6V-a_bI+K{GR6ewsp{Xb<}VFQ@b)eLnv3WvW*)VeOs1MemaIB?m7Vh zxX=m5G<b!AEvH?<H*AGlGD7XP1wqZIO-^`UgGhy5=3A3IJu*8bVfA_Ws<xuG+2h~M zFctOh1q+@UxtFa0IHBU^2IP89O5X?H-X0+s(=-HBZk}5l-sL$c`D?E=Y9D0<&SsFC z$%JuXGAP^I8)Cma?A>J`I=hQ%4dJhtSsLNMoZC_?$E)aqmuE0?SAsorKwt@7que$O zq_nT3L#~-<4n$E07j?1|?;}5dDwJbD<TV!nPfr`I73z*%>xqAZ^{gG{h0nDJ;?Wh4 zXz=*cSUddGg$qOTi(DEAv#>LsYr8u5+>Qc<?fKWPZO+Hg&Z`1`SrbvoN%oQp60QnY zxl%%ih2+nG?y<@Gm*x&-D{C8|2_Em?-7fe3c8Z&_CklOxEj(!!(&ji@$p7kCY{jy~ zKGYTWn!9tfboa`iq@txMIADtEUY(~c``nZ$Tea!0?AL1=Q@#*H3tK(f=7uJb>@%XD zFiuf2co7)==)1X%Ac4Vm2C=+2&BUFO2GR7}`3g0^5%qi&chc&)2B~Z-=i_tygdYzU zh}t<vd9>FY>~vJz+FD%<f*fZU`zevo82<%cE6#*GliRyB&HYeIp*B<3fi5#L9HqT% z$gsvdk6wqv3NMa*L9;L6b$c<*l>(Ne3xLG+ZYj;5`ALatyMj3^S?u^{r&<_38oWJX z`oOv;sk)>tXTF@^uDDMyLSmjfO*Ln|_8U{4D5n3yNX^rlMJUXb-!?u!>;;#VaWCpP zBC5O%_Su^L6I-nmwFX+Op>OlFe+i|(SP;tYf~l;gy*c;Sk_1eA)hp466+U(z8P7ek zh0gf<1Jn%(!bfFWzpj%J>DyG@bMfutLAPN_!-rjFld9zk*8dV5%(eW_Mr4gYYCig5 z@?8U$Hpf2fe?=G9{jZe>*^IZ~1er3Vj^0m8(&%;C`rF+>u2QR`%C&CM%}sGb16-D| zx3#J<)tgQv2>N%pj+^@NDaObv>U_6<^U>BzJMG)5HS6xl(Z6N%+I>zx4M@AR<$JEJ z89<MIBM;UxD4i%bYki=qCi(#K)wsy$@Zxu~<u3GvtQL!G%{THDF3mHR4}5Il#Zp^l z**9@kwydT=gBksB8U5h>^$k<4*Z<n>vj83`8GtzSedtUT0TOtgP@(-OP+4I?X5BDL z5LEbWFA^tbDK{J9EPRxU-+<PqWWXQ7`$kcNe)t$NuC(ioV1h!)Tsb7o1s<o|cV_s7 z^f+50u8x!;@2f=<Djh0e%I{k>s$kF&G#r_wbTrgGvTk&y%pB3amzZT((fO1(xR}$o zxu0^`gC&SKtPHZaUd4Xy!2O>MV~7$m766UMv8fz}+wNs!rl0+Yfy1E9F&K3Q>c<p7 zd?TvQz;zyj7sq}2{)2<n<gwZfxuZ+QaNN^!dE$@0E;sudgYs`yjMG273UX~FHysxV zmk)<JATxSF!!VX}aDKpUWxu17&^CB0Wj#>@Yrshs%zRC#0?%Q^KlGjv#qZuwdtqxM zwMs#V!PzO1tdYV6Tb)rECmOo9y(y&86l0~&hq>5k#7*Xu9JQ}WzZ-#LUJ<+Pr6!78 zkp_w0?t9c^1kIWrvW(FKAnVK6?)b7~e(K{@JTc;|qKlsin8cadgv?#kn}wZy`wWp; zxnX3@T#4<q{SV?W=C^GQZxx`&*3wzCii&h#3fVBDO|c>M|MjM5qSVyp0E9GVX5ltl z>(&$#jE$eu0)<mlCWL<-)y>YV5*pGbi`BK>iCvFQJ5Q<yz=vzx4zn!{TQjOGRVDry zW0G>-?QbqoG9c5MxB?T{FzKO39-uyufA2L5m?Y+-;!AGl6*_3wDKRw5j!J=aFg`c! zTCsizD{zTTTxs%5C^_Nq-*g$<91Ay@DkoQ~#<E%o^_v9raD`8wL)!8bv$v9L$(cvY zyJmd_=(k(u03p+TB?~g}(CBFmSBJesY=7rXR0us0_I<d2J1p?=S8JHUckKtjqX-0P zOj@fq55F`1_?bKASqk6Z`om;T>d8$kKm&dTUxw#34XlMk%wPrNe@!1*D=~T^_FR(2 z;Pn}kTT?@{a-O&^*;R80dxPa(i0=M7^YA0TOObIR#SbULG_^XxmPH|Ip;qj!53S$3 zmMDoPf`M?w$vx>8+}@WkWs&Da-A`Tpkh+g%S-0e1)v{4;2qB9Z>O&yYrt$-vC+Pb) zGJD{6|B8mwykk+@h%WRd98CxIq7i81N5o_Rs4mKRL^UTy)Q!O_)6Qr35B%!^4e5(s zYKzesG9oD0#=p~G{Btf`!ud*-5?s!DX(!%6hLK8Zi|9*sa{OLwCpU<uQFdqK)y?E$ zu2)urmi4Gfp;2OwF9?e1C1D`JsLyV+oRt)&aZ;RUaz7rE<h(v!`7K()PR$9z7QHd` zEOu>TOyj;JH9Xjps9zrZ8K%C5$r?uc5*mvWQRy~?C-6jSq<!yJ&03<<Rjd^ewn{3| z=`ZEzh`=;WU$=W(SY7wG9K`tmk$IQF@soDX{fhu5rWQyX5ZeYPnFY9VudvB0I1NL2 zofe*j*6d(SgR7$=Fe{mf-b&SFvJUhVrR)3W8;hZ3nvZ=yRWK)#n}s#*VlxwSIzPP> z(?Xp&tg->Z1jV1wVj#}qqmDps1e}tp4_dgfx95*C^Z{yGQgFU13ntKurQT|r=^Pmr z{_?6l1$vFD;A%s>Gu8O=Z)GzanErzi4ng*NZ5E33Z=J=vS{7U28RkS@b2H+OBKc;V zvQL7(&_>O+(%yf%l06O1+t_D~wXbaNTX6*sQDU5qyle0JyoDRFg7A&07pCYUQFcw4 z{;!Z&YjWYEeyCV<qb|h%=UbB2kwEOeESWl-SBLb6=}KEp+x-d;+1OPU-l?pl<SBLo z27d?cP%ZAh%sWnkB86>PyWv&H=n<XXJfp->6qM%24y(lbn?d;n;q=#C+rwCWQsH7F z#Z3mXZ6678G`Kdi;`xn2yRy8{Y0(Za6!eAFSkShSMu?ozb}jRD;sgv+g3Ov|vh&Ds zfi=|1Dn<gdW`lHj>vL~E+O+ss6s;ZJBm-3!Zw6b-37$(!S`n~w7BO*xScoR>eWshM z00`16MoU`U6hA@jpRg6?^qF1LLV>FO{`J(0MO6VKUet{#q8=mf<F(LA!Nf=Jz#T)1 z#SSrKNS$95x^AscBo@O4u6z8Imn~FJVT_!2N>c32G~n0JdC(?I)l1PqBLIGElmAB1 zLi2EIU68X*_<Bo{n0gAyXf6RhME?ArhL?j{m&6r2j*>f@*_T+=lHE*CaeMafDn&m# zaitHd7A6_HSF<V}t8*kxS$6LstmwBcPEd_bp9k;^eHiGiB6(8km$xP#@;O@SL)09c zt$g>B(3NjwXW30FyyxWrrWQe&S6xJ47*~IWP~tu-*B~9<?<$@*8)1fqe%WUJs(!f# z$34iN^l(AvdUyRd85sCZa(h<jqIgXq;F6hzP79)cNRx#W{#0<3c=41<rUqJBn+)&G z3(Jza8duMS#iQ$=bC=DuZja`Nr^pnsU)WV1pZA5p0_P5;zmC%3wpQ*;{f=<%Npi+c z_~N>5@2PrVz0Nvld1F|2X#5Y>oewQr@ub8u9o)aI+&(9@B7{!EBqtv<#9Qu`v&xxJ zDF0v4p1!ph{a4ALapsh-_etlMq#IP}7l>>?KHe!Ih15EMAaHzJmQFNO6{C2B9(`NJ zVNMNbuwQPU_s%Pb9yM%WCUHU4>T{;R@Q?W)*G{{nqbc@+%6I3{m@!aq?KrkmouakL z^Pp}-(Cr<!f5BT8Exq~r;fr~CCv*D<_wK$jnVhmEqvSaP;g&qQ%VeO}<9n!@dsSj7 zo#jJ5g+5MvrjB!E;#J<WmrScUK&1oF&Q#~62=DagCaLTTm+eP_H!z)FBv0}+VD}=g z+MsgYvHW{=lHb@D3wxIj6LSSF4d&ZkBWrrM`=C@QF(lnNwerCMZ|O^;w$^z3YQ> z)6>Y7<o|+BKQjSnW&HivAV13u5hnZ8w6}t7+<e+Ir`>&E=HA0VnHcCJh6Ll|Jjt<9 zs!vddsN1+<<M(_q#UIBY9FQqB7g`g?@}2DIpG|Egqd9pyI7l9)X}z;W-O~564Q+jd zz0~F+v+q@M!nvPh63HC0jW7B{1sJm!C5X!9OCM-Pe5Th8u#=l(<WVN_fC(S;J-WO; zVzueVgGj~~i*CJE7%$(gA{L2~Gv@3Z(CCHyo|^$|+^gTuN!$hG5x?D1D62L8c%iI3 zj=MWYFL@<JCAGoU)31me{;K`!Zh)h)Z1{zxa72SUs_U$$MuAX?+gW_+-<$+jl_G#1 znO8^$hT8#8hJCRM6b3ne9{M(mP!Wc$=ALy@PjJKHaZQe<B!*dZi4Fp#j8_|~-*|7o zcm$5^W6=5UwJQ}lYM5DMJA+KWh_S1v?splL&$lcWt6Tk<BWaoC@KekV@%q>}h8JBj z0;BDTBA1}ujPfkfi@wSr?g{+ALBK%GV6^qlu2LI^#s`%}D$|dCd#$BN?}t6mT@x6V zZI$?WSFNBwcHVyO*(>hsC@6N$lEvh$)(}8uDrDj<v1kS+*w_o_c3u>3ebU4WRl7_L z0(i=lqd`LS<77)3RONRfV1n21U!oaDvzV@i>@XHNH<!4i{}|a4{vZ4Hi-#73JCWSw zsaRLewzMmh0*9!iWwdi#p3uk^5d6@s^oMIfrY&p*CAk&i9yRE2^Qeb&!_M*oo583` zhE9=SVGe*dJs6(D<>zELNMcXw7DNRAUoORw{I0F8W8~C(tCBN>NPbFaQe|-X&Aeqv zX94;sXV#Y0!R^OVtWvaJ>R`AskF@~BCBrM$22W<YcaZ6uYhfMGSN)cHuI7ohbfFxW z>M(at%jz^~)86O+O+J5uF8N&o1$-l4YS@j06Ua@ay={Hl`UyD2Eimj^G=+z;Kij6n z+{k_F*5=i_zMRp14}My6;85*gn0$Xjsl<{;EHyhld9VEesO(ywbEZEL1co)ufN}SD zl0(P;SRhm@7(t4ffhla%F5dB%x90An!dZ{VH9xv-kW5`1+eK-Y7(V&xgNvV-=qH&u znOydH!2#;0V%kmZCMZJO?54LKPC$J7w5lc(>fkDLDsR#mQnEnugq|YIm6xb13m5lc zkk*Lm`L5{nmacUEfGq4GNl79p%R+W~;|<9VNh;2s^l+)sx%x#n6uy@Y4*2fJoa(xB zo|8Gi1(`nKD8hPo<(<a3;)@>nM2FYsDYa)x^kIAg!&{3N(dY9Q8o`NJ!@$HN!^E%# zrWa)>J+H;ruwKa*KFz_y-c$o6dO29rYjK9+r0gE_5yIH&CIX;cZ|7a8#CJRzrYVOY zIx?wz9r|@^K-jNDkQHxqDr@v)eA5Kq50Bg!xUAMOym&6@bz9U{UzQDFqfqPp;apB| zt<O%_0p+lCb9IW(U89Y5IPUJ0vo58}cpB{pRzUbyCzo;;B$}d<ttE5t^b*o=f9$cl zE+5W&bv0}-5XAQTYBHw4m>%p!CSlZfy(GawXzjjljH>?bF`VURcz?Iaqr^pH9m1#w z4R6;teZc6AX)H;Mz-YNGS23yXUfE#MdAU1&@kq6u{2@;Riql&0B{VOt{p(_)+hJk0 zoOMpUt5r_62aWpia<Yiq$I{L)waX#$tOO(K7I`LJF<l!w*Mz6%>zIZX46}p-BP66_ zwCC^UAMz!oDmTMr?_b@%8w!#-Wox*kT4bp;-1;_$NxbLIq~*ZKL88@hFjN=622x*t zbI-!f`ec*eA9IM&=?Fjq?ZQ-*FZMDI(_wYXH@g&Qsnd0LqhiBa)w;1Iqtz-XL+V%z zS`h<Ao`3VL9S`pECSjr<>G#;2qyJH4;HW&sLmJFc#ddqwLUUOn@dcz9MeZ4{G@tzl z2O2$ELsOd$N29edt2F!Lauqzvpy3>Qt!hb5Dfxts_^tDxkt$i!?Ia>{$d)_%79q7t zr0KL9D0uxrseiQFg+DoG2zTT@A;Tm{t;=_pz`+buA%uz~0bf8k3Oq)}XY=QK1hIr` zp@o8Cvr45gFTqP<o`HjC2xqQ)9&b3s#Ns&P$0S>GPjdlcA-!hcE#<h&Q9b@+sRrd@ zpV#i`vPUj+gVXl#FN;Ia7PKtz%Mi(+_)j*ox#@v>n#RRyL*~|<?6HlitG@>6wQXF? zXY$p*2>iJ&ZhIc*j3n>IAy>Cebg)KgadsGmTx0-mi$+W5U+d6bOlbyUCmq&oScP+; zw`Dk+C;BbN2=Cs3sqyIvHcI91C~vh*Ef-_6ZuIBMq6#)FxS09Po>a{!c|b>^pe3Rn z1bcY`r|O+PP$tiUj(X#DW){xD&0GRf5w+6f1N!Te9h+N=w=anUJru1B57Fb{qZE+g z+fcD_u+}o9Ibuo@>VZK~CxV4|-dkKQ`fx5^z}M0&Y`_=Cu&FxCPvBftTBU6xU61o? zmf}Mh0lxmd#c7~-eOD{XbMH|CZ0q;;lDMMWpQ{UQ_!kqxl012drb`H2Y(}DGn#teZ z@fPIq*$4!TOt0@&csY(=(KzE!=;9n|5|@wIjH1wswg5XcT5dv|N~)ra+pD~ww&!=^ zs8=Taukmbso|+3{2v;|-%yk4CaU|zT5b{$NW^>zeu$a0S$~Vh%H^hAT9Wn)rYtL8c z!6RGL(}~m+3?c_0Qewu~6RO1%-0<goBh+kK=OBM@D3tqlzg9<Mn*9=l8>Gqi{o_?x zz(#qPIDK>|P={Pv=S+ua%8W5AFyG%>MPWqzour`KmsMm9wq{bfHk-;Nq1W8Hn4s_p z!?gnxcH3FNh?@YHHWV}p4H<RdAgzFnv>4ZU)0B4(PLqVPG+%P~@sS#1ux84eC?kI> zd(KVlOJq)g>4}$wv#J!SfU5~43hzTzEpYZZ9sE!?n8+ac3D7OcTxVP$3{YK1W$3fO zrkI%8uTB+=Y1)3LZ`7(OLNO`!(dPyl%LK@h5)%}yt~W_igT3vQ=$*>g+a>_K{(p^M ze&yAykY5X#YT%R_me7;WR2!b0!^$QvztQPQl7Su=+p|R_`u$6C`PR0_<6QD2cT#`_ zO4QyisPz9{AXfxTfOHc@FiZB9`^9Sou{&0)_L*1EsH=HBl|}9J)z_NXg!NOF?QoLf zNi2&lKCH$s`@>yd-PK|I+$4=9r;Aq9by9(1_hV<46-0yATMfOfQk_;Ld0Ase&2-bC zGwK}JxO?1~&BU6W{mG;|Dw)df82r=lifIP>aodbWy{o@pl?o>|aO1cRQ!MqpiOmtJ zBlI2EJ&+=p%)qZ&VD%ck%Rm4bLdpmc_brJM$WpW2f2U!tx;snx&QX{5l>f;Ksl{$z zpB|8)7z^&7y|xPUaY<YXkjo_%v6{n;#GrcJMo5-)+k0P5we9jx&UP@nVXft<f5RGP zwc+JJk#tf$L`Qq%-8)(B?+&;+|LJXz>v7&DRh$IHi}0D%38L`V807Fp)T(fvD(h(N z&&uDGu-`@rP3&<0MKMqPzwwBnxOeoec>%zL8d&NgNi8+^yz<aGnZ5f%B4qbPRLX3h zn)(pdDv<q|x47c21_8K^{V=pQ#pjSsfT8qV<#{-VS`GvFbES%Co)_Jeok<Z;;fqMl zr!GstFFlsSNgaL!!{s|KNRr;fW-k6s7(V(y(P7`k>a&jufI@I-$ILT}fG=52$fBe3 zJUie{FC~;2Wm7|^5ynOj-<3d*8=8!K6>a#O=XX`~Sq6+pL%8+@kY$r!&g&R5(6urm zEKBO18i7pJ#H7=v?5-CV2P}`5>+N({%4g9+KKj6^wWD8enVv$*RaY^sG$A7zefSB~ zytCs|tUx=r&tx)qC9eYHjc=yzy<WQ;9_i5_5>0&PhiEDq69VSzE?>8y+29DZko?es z`!krJL<Wlz?Z0t2lYs)5Vmgj>sG~lKsP4qE29bKncx#oJn#0^_ja)j!)G!=1jd|g& z)QM!c+E;cQZK(R|$5+F8-mnNTeP+eoH<~p+F+R->%#o0!^|kv#STh}Nf94<&{ajIC zpOzPKjwD(JMRYlZNzi^iB{O{CRD-!yh}IGXI8J(n@YPpi91%9io4q3ivdvPmiVfm- ze;2GuPQ8d>m{sc2ZKx&1O6Y$8>p2W--?NOG7(8<nkuYgE?QR@mc9j#1rwJU5?5J@h z-%r>BZ=&!{?DJ#^N<m724o`ECv091Rs&3Oa`?`Y*6y3sZUk+`!{Y~Z_F^ZCm6!pqN zrDU9w5^*Y3Q%;h^$%g%$>s#gR+#SxtxOTbN<Pk!htr#32saf1K72c{=S*wzN%O9dl zMY~k|PtxM>|IcvWT@qHj^!%zbHo6_a$%C8xQ9)L=-45Tum+?zpqoJMcjBlOG2bBlA z;o;wKO?rFJsoF7~8giFx`F&mjl&A2$n>p2c)9B-5VAHLPEgHhHL0qCK9<}Soo=jg; zn1mQql%3Z%^?YZ0QqdsnbEoDk%Rom#-dOC-;0sA9^lXrgF~5did@pex@*D$pD2{j0 ztzM>Wg!E0B@!PHyMSW{<H!r9@`Jgrp5N~2XpNgF0^e?jRATAlsSO#jX={T;qzK9K> zr`=2##K#S@lYp^Zuz}IORTymnr3Zz3@j`it_)*_uh>*gg*ntM>L5@=Z)}4p0lP@o> z1z7uS!a#U(5eE1A4fgoF8Idxa4qK_Gz{jhItc$+B_qX5iqxNEItG<>T{8?HBhE?kG z+>zx!q<Wc$agh~`_F$w9O$F7OxtmW~2%_V%E+>-xNq$)%Nib^Vj4{GGGr;4w#3iWr zLi>JeL0&=vo7ZOgn1h@EO)nF-x(&fcmpt>+Y&5lHi`|zKZ8sGD1{PssU>a<}rfC_; z+XA~Le<&|EKv%P+a%lJq!Q1rcrH~5#{J&A5JeUmU&G!QB>^s=7;BN~o_*Pw|6FwQ9 z_poV2C>g)&nj(F`a~u1X`A+2Sz*pyP*&Cy(>I~h~yV$VEYO<c6$fG3#`b2J<#@7T_ zGQ>jj=rmz7^OErLbh8uIgg+vJ^bcA#FxLMFh5SG4y=7RGUH3Pv($b(J(m8@iNe&EM zLk!(1pi<H~NC^@m5(6T_Al;IJG$KfMiV8?b3(_I+?D4uUZ@upOIiB}D-v5X9!}G<# zne*Ix?X}lld+im!eaKq&z?h*Q=UMy(S?Yx7ZQ0M%6QiUq?coK_Ua#icuZFyZ3xwnm z;4TWoxVk_=hW=DyS+eQznXfTtY1Wf+ov-PnDJ7Fk-{QNp*e?>GTQ*jr9xTm<)SuLs zRb#*Eg`jIWX|LYMhaBFvLEq6*zRJ;^YqzQcH@$f|_lDt|j<qf`H^Q2Y7{_O_gubbE zC%x!(bkQuoQX0ir%W`q^rIZwVk1yq-FPcs_^%hX;{>5Ylr@kkqGX>%(kq-^GJqlk7 z^_ORwgVGOw@0%Zav)KhlAE=IYKGfFd#||3ms=jIoc^Y!MUsY67`ZarHeC^Q#P5EwV z-N)QTjpiN(w_fgivmfLmv^(nZ89|6S?6tS8Ye~x3Nq@AcJ+`U#ZXbSRzrXPP#!~(L ze4hKt_4kvYex?Ap;jn*>=<!4HFzB3nhxVn^#PiiNoyW_ol#?rIS}Pyj=%-uC^vw&y zPWmnv(kLbjl{gw^#B<*xS?eK=P+MVXgu2P(T>N=R*t;ELrvM>%G?ac;&*nG#e7g!> zCe?Ci6=r!>1~V->L#dZHfP%4o9G(J0V!?GWt0jx_LP-+o@A<?o+uc_?p8s$t4`HaA z<ZTES<ziQ+Ca4sR^R+NpK<V@#kLPnnmJ_qp&8aFUlU6fxoq9;uw}j}4y#-7jG97*# zjR5gegol4P!{ml|@+~8cXst|Xh4QXE{el&dN~9_5ZT7nRJQJTwM>{EQ?D5(SD=Nw7 zuNr)>GvEE8eHhTd2g>>_1Db@6`Yy+gliaw4RWkt73+&VvR$Gbxid~Q!qLXlx2_eCg ze9eA@wx<M%jggc0KU>NWZo>~}Tle3vBI;Oyvwh|bH}Ol}qSShA^VFaQom`$zFujn{ zx8^Ati%#jYkMr!_=v<|j(@uPb6xiC$T*(dxvAI3TH#uG~<I-m(HTI0>qZJ>6xX8?< zmzsWU3C1t07e%veUnp*Urr|(0@_8*BeV|^|>C<(Ty_lFsIgX|2Iw9jTzrGg61E0;6 zzw@rGW(&QY%^i`x1MH=<neq=HY&X+I^X;pW8cts=?QXP6`q6+8E0ba=RqxZL<n(OZ zIl@Boj(nXhi>*Tad_0;v98|RY3JV9#R@0?!u{YEle8kV5626O3?r23XN*$wCS0uPN zJQQw=z2rV#CGKo8rfJWP4AzBE>9apIA~8>Cw7IqNWUihx2OOigjp$Dgvdi46TJ<Lr z{W#5<o`neDVFZQx^!pnl$Z~8gA1NN8%0}<WeA}gvjv?O)lo2DJ!$JCS#kUDJC*{-c zN3Ej9l+)GF&3^s&O=Y5+qlKumH!P-N&Wn#JLB{3r{;Q<3&#B`3IU`L|F$Ys)U?OIn zwpNoVOBtY3CS<Pu;nTY&E<a8$_-~f@Fy2CXEgYkl%^lRyUfNsc#VQ#EB5e%jFW+vS zKJZ-uj;_SXWZ;1OtoGv~DJ@s%t+42-GNDIsUkTkQO0G3zjx#6-&uMopT-WDeKI(nC z6EIcu;w#>G{=-Hu(+hW(>ytrZF6=NsZ1~1Tw^@TQY1yuG>GLjO<7drdaDL)6_o0>W z&!>4_QD2DKDzbx@w;FAh(>G@u`l&P*KapH=e83+tN$K-?XrZB|H~Q;$D#bE?b;lFg z36N<byw+Jf83j&-J{tGUP7M6J=W1y7zixQvd{^o?iH%}k-Jn#xf9p{M9o|KdS)0-H zVD&{Sh@S@ds?1J1DRVbKd?)8be<*uq^;{wu8k7ucM0#Z0(|NU9JS1@NiJ_)@IeB-> z{jqiz!6FI0*cC)GUyF;NuES!)jdUK@Ap?BHthrH(c`hlcwgl%E%9_jjcl;|^PUcoo z)1PV2R&+OcOvx3eCGXj9DUL%<t&Cmgi+e)D+%)__YJfNyPfh5<Lyw_|Y0UP`f@PkV ziMqgI_zGp$s(drGWh@?d-{&F;dBJ-wp_lBu5MtN3@@vOSFDWc>k{3%6*L1ew{0SZ( zSHgSzd$;lxT=LG28yX!ZHAok)8jH&)b)pjUA>WuRdtm%uHzHHYx!g5)yc4U{f;#fL zy=xzI@$c%XDzSBH*KD_AHKbF9?uG?l3slV+)=r5u_ctTwzlq7P5W#whyspp-Azs8? zPkHx%0#{08*SKoSgR{YkKjtJJR}#|bB5_SnlS<V2xij@JT1eC5bz%Mm&RwtB8+g%l zxqNZK9NjLc#5>ID`CM3Wa~n-2FPB^!#|zDB1g;A2PhMlp&U8c&XJ4yjw_*DPw|Ea1 zxSZ#?9IBP{TEmAyOO#~=n>r;_OL0ZO^M;b6db>(rS1P02lcXPp12h+jTBn<!mi;D( z2%D<oR9*5Vs{~y(W5JiNMZAwKmdZ+hNK2|azjb|5Z1slkn}+D}D<M24GmHyDwAMUe z!h%VhW#WAw<vrR={6K@|X}2oY3$iP*-hwX$K;Sp>9enW$Jr9L>nWUZ;;UbzUNsr)# z#O~w0#W>fl9OpjP^;?@m;P4q(A=hcW%0)p+FNuv7_-e~3vw)3pKSQDlAaP$tZt=Gy zxjD_MI7Q$}u5RuV>bLraj{YbFwt6kD_Dhpovv1eQ!vnmbW<zh<ib!jvKkw||Qjxs( ziY|;`Aa7J-UOn0Hh6eOlWYy{Wl^!}*^?jOAze)l_J?|b33K1X$8Ur0|QY7KYG-vw; z8Om1R6~jW4<g2vM4d4K<aYuYvK>3wd^g@$s!Zx)8C)+gRuqd8(_ISQivPj9R0{m+> z?7LA_&GV%7W_P1PxEC+fuB;}H7LwI|dqbp?lRw3Kg;Rq)*5?AI9|lJ?cT)DzTl_06 zj4LP>yxDAy-hR|mx%h$Ndy(xV(h4_L3*&b*Rsx)~z3!^qH!?zel;<=*jnR6&^6lH! zJI-v|6gXq7pm`m%aa&5lIHlfgZUo}`bv3mkV1a*8F+Mw{HJ=D?q>j`5g3N=TBir>Y zH<dlww^ZNo9=F(fu&CBs|Hd_hW$_Gb(r=yhZj$MFxjhV;`qKNOVLD%FflQQcdivT1 zeP^fd`S~9a3bWTI1fIX07QRXFvXOhQvT#<i9?>8<oBL^~=|k%A`<wZ0PO7Ir)H>a& z*qa{Q!MIt}uev(6i}AFz!oA>ZCZv-rhMGO~6dm<HGcmL`w~`a{##v|Y<?)A|l%Wz8 zUwS^HbH*08jMYAK&htLjFi3(g)=^HL!~&ojVzVUKg^Nj3pLT2z3n)hWod;T#)tadi zi(ePEJQWxHvbK)2+c)Y9F4YwYH>nu?xbwNACRg3>{KER;$P376D)aV*6;}S6jZK0< zOy%u|1wQ?=!B=lIm2(KC516koDqiAGc+MR!>2EvoK+>S$l7+~RRW$#jw}tPbtqCa^ zTqI=bOVuM-MO|js06vrKx#rGWPpVMgKZYEYWHwx}OuO3W?}0*63-e&}n*-%DY`O2f zxpb=WW}AciNpPRNYn)`Aq2xgujkDq<*V@Rh5l&{w@g;8_2dz<uH{OQNb1Y_By?!bo zhv)(*J<&9eMTT4Y{N6mCb(s{gpGpdeOF+_mv)G#d8hg_BM$Vfp<`5%6bV)P^`9`5} zn!#ze*g=vIfHRqH6gsTmZ8W_)pw;VpVHx^%=JiL*gM5SKNd1BzJcOM4jw8pdrAMk4 z_k^~KJ^3Nwb~WpppN)CK@j{0I_DMXP#n(R%Cv2X7G;6()_})|vv+Hzoju{U;JOHCo z=RTV!hc9+97*IYE(yCZbXy)UM_aA?}RPI~81>+go#2(HL5*q^b>yMOHJg0`9-e^uT zS4F$nvMxSwNI0M#<o=QI-pL6rJ6Yf)PnwP8H<LQWD0IVXjFYhMwRV5)N2YQe^4M5x z|J|dmboX(oSzWB=m!IR264?q^jXcRc0K=@~bqq%~C>t;nzDBas-cvMm;rQWNG!E`8 zKtJ-ca(l3|zSE}Ynmp>I3b4XV_naM_^%vIEHpicaT)>JbZp81&`W&K5)gm=}znkTU zW&NCxB=5xqP(uw;HYIAf{fBIL2N*W!Z%wbL=VgD!;;S^PlPs_%qZPN)-u1eCTSJYv zu>Z$?`;@>HfK+pQ=9%^K$;!kXGhFK_0Qr&=`Q+6by6Y*gSr>}1Mz!Gn$bx&;xZ0B} zHS1-T`vwukb0*{vszunBdQU^N3wc+cWxV{2xw_!`e*TxvYJ1{WxE*~tn--eaKzYy+ zFfgc!pS&!S*c(*I0LrJ{N!NX_e1N07CBW_We6W26i0&efj;=DNRIr^ky~!RIk7+o} ze>X?&v4>;b(;bL`BU7S@xy%VRM(>RRe1B!nn-bMh3p14#T6%8jFCOcY)(?0*Bb=Rw zx+$%=&t(-xf8Zj)RVW9DI7dq1(E<G`j5aF`B$w(qyNOA^s!%#ViaYwwdttScQ7%V@ zd7BC-W%~J>=jN7&K)tTVekh2=m`3hi!`c2oXqpQ(On5jvIZ8u9lnsh&PuCW7#B;d` z9%uPPDP?FGI+hAYvQ&JwKPLEz*j9C0VF*|g!0XwB6rt7dvL1Pn5a{o6&a}P8Y}Zh` zQt1wF#PCUNET)`ldcPjIA@qgo0+V$k2*G?LSj#w^IRrI706GSk2F9<&w}AU`&=1sU zBpW-~6+}3uih0OHFO$*(?13LRGa${s>HBHXXB(`(1V%Fe8P^W>uf4hQ*u$Z8aB}g^ z!e%*g&I<>KYBSD4FT<a`OLA>=KX@Q)kPATawn=~r(SB>@>hX?TK3}}~76N~n7sw<R z6H3(`y@=106%5jV{^!y4{*BMK9Z0Zic;}_RYOz(L+c+Pdj1tZ7J<bxcvy&urQ92{* zUg)GByVn~ORe0qjJ=eHgsQ<>g7zCfo-ygy*2Un?Ct3;U8=!^kGKq<5w{-#)VgzU>Y z=F`5&t){71y=1qnXW|_3q$hKC%7(dnPo|#|33dAy7^9?cW;cL;E!Q;V_`7^800>Bt zy2rqzC=M`g$jrYRWWBcYT8^8n(P+D$;g%gy{C*Brc;?!6bY19PfZlS1jqPag`$(_F zRP|@>8jnntG4cHMuT5u1qb?3d$}NHsT*tKz>fF3WYb%82XWo4T?tA2}tzep!#DY9H z5_U=5kAvh(DuwZ5zq|TeNJYL8jiIee`8;9{$qD-=lZHy}8mJHnl<LHL?!Y$iXiYFl zhhIO)RfC(1{qpd{m^uDhK`#1snNHi(*ZsS`dS{{oR5f!+;zESa7^P)TwyH)Zk}DD= zi&U#QWG)J>*2OAwM%#HmY$zZ>+>d0tJOR&0d8!%gwnhnH>Gp~qi1Lz6SE-NQbegp6 zP9$l#Z|L1@a9U>^_d%{I;mWoI)w#O0oILY`+cB-Sd<_ICR{DZoY$Z_~0@rmAt(T+c zk+_uYJJHiS*7WBO;~ulfD+$OvRRs`?n>gGpdiINUwq&_e#dpguzu<eC+C@<PE}6?6 zV8Uc3CYTFTAh}FNq}8mlv|uZ`mGCO@R52ucCS>M27T%;scLGU+hph;2+>Zr80y>?S zxRuZE64UV7whbW%;K20>PzeX%6bStDV9H*jZZLGB1AG@ILS7I7R=leb&IYCV$IUL4 zwtB@5Ak_t+lq3qXNd(9rgM*Hrm$lu-LLh|sz4_FOqpq9-fZK3ptG$Ld&c<CWN!6`O z-C&bbzmrPDwF6XFi{IkM){kLZJ~wWaHR^u@Li8<ZQu@x^%kV1!6}NWPJNCNfzn~`` z?0)VB$X8Qh{2~_*C=a(l<^8l)Qts()+x)4wF5CMfkbOH-f4{S$<cS|bm;ZD-S$m&h z$6%78I{_S_KuYNWl~7ybYmU>&n-No=hNBhTLv4x)89LH9YQy5%_wWKd-CUPp%WV7B zBsn_|^g(vc?#aaT=hUgAkW}@!V=vn4$J5^?zNvH6Nu+Y~)QJ7)*Kd~$kY8E+q#f(v zK}<r2l+a!Ol%NozZhqy*$kA21AJcDg6gCh%jW9vidmkMIk}YcMkAi5gRrxZyxCeZP z2(u+lqUdXGoH%HW-_(C0;$vIw%Nnp=bxFue!W19^&L;}7H9E^5m&5Gd$3=P<*B>`> zAttCwG;-H`jh(w(ABC#qT1`Y`?_0Z!Im_SHX7_f^!QJR0SyMLOv0&FvnpUP_*ZZ6- zakjU!cj9`*-J93+TBAxOZ!w!mO!cvlM1hf3jE%l{3Lz1@nIKs$NPI!`0zek$OW_Zi z8wB^I2}I+-R-Oe5hab(?8tj)(pWanmQ5??&Nvo6H0aRC&C%04hjY=IsV&MLpRxGT| zw+8vXUsWcl>IWKT-@kQUh);;X3+~eBzRBdgt^H^`S{^GnfBjC=s^MmPY66c}$pMS2 z@7oCWXRZN7J3)dj^9%;3KT<FAFdJDcJ7EQPs=ta>sQCE(86zJF-BHgSKGv`?!}?p7 zmx!6w9RTP+P}yw2eSNRUk#AQa{g?Oe)+QqAedXr?B8*cV<Hv@qCrHo`{BUDjLHn{V zcS1ct=XrOkWFb%zjl-w7)lQ_q<9FSDHt-%Q8+h&Na8Vs{4Ba>ZOEm{2DjRx{<;|78 zZZN7-QQmlxl(PpGE*gS;Thn;YIC`1L2OL37gnP+B%Xo^Ql>Fiq3h~mflQ+ym_l$hZ z7XVtt{*Y_|Nv6*wcYyviSaZTkjAS%JF5;xWRpKsn&&YLpaNAsi?xRw1q2%7jvg-hL z=o+r=JRRN^tkAq!bj9oeZ>hI9uMv^RE3ZfvxpVGA#?L}ravJN{4jW#<X&>1=CN$IN z6(eg};me+hNzlRS+F-FKhdjbstN9}T8mkWnF>!B9aW+9)kI=IrK#=4tIuGJ1jp!i# zD8Os9Bm+D~#(OK&X`sk6C`b;3CReDFT?4+CDJ?e?j(QA5QRt@)Pz`E`(1GIuQ~#1E z9uof^tqZ6aJ2?ooCjE#vPQ%?7D^7y_ZOw-#I)>#u0EPY731BaLLn#Q6?R8-lPS*%i zK|(WF7G;kKK5u!G+~pde5?$Ro5cI7O7EfZRxv~B6b~e#`an$n+Rp|+N1C!^4CFW=Y zkAS-;Ej0i-&a_r6xu>wGwUmjV$3T6fIiQKp1sw1J+(Ce!n|SF}JH~)h#EHrCt3Ueh zKjFEGU>IfyxpVUN#nM}DL1VYAa_4>Lqt~Oc)@JWxpWoQ%-63iNmPNDoOA!hQC*!Tz z$r=D+LPhi*9KGvOVd5`09~A7oNN3z>_SI7u`_?&Q!0jvYqmgiw?s`X&v%+Y_lB(Xx z_oGV}>aO5PmT`x=j&F1p`(5{%k!7=?TEppQRo0g@JD%}~0Pu5#xH5Uiy3t?BZki-f z%1CS&ksFp49e=Y=D7e+W%ODJ)dU7|Kfyg6zfy$KZWct$(rat@aicP=s7I7w1Qpv-% zchje0LjeDEoFq(0A}-kRE%)_7uj25XCkv;CD_5CED9<;$`wUObHciK`(5q&g52#`B zt1K|)em<7a57fC2yt+_dbm>XI8NIQG`9grB3WM;~5VONmJ<(Tox5>m89aUy7ec1}! zswT-@>pTYTDY+I9-YDGApQUhZeU!2aO51n2#6ohOg+^I-I~miPz2+kNI#Mx98ti5F zzGn86i)6cXW>c{xY<K643;M|wvr&fQCN%Y>i~CW%)%YzzGdp9D__q{;KJPPYii18p zY-Hrt#9xhM=$W+!0JYV1l1TS}gV>lCPqK8F6&W!b3mzZSOCyFy=}=pNY9A+Qfhfjd z0#M~1a$CZd>$Rl6cT%@-al}3sY-iuzD*s`X3~=$$ijff!v7Px;w8v3(a;791ycU1* zd7hk?$@IC%?mL@)gP9!<&6-$NPbw%uCG@t4`$UT<-{UVSHzU@|hJNVDw`p@0gvdfX zS`7ANnX=!r26yjWU!}RKx+aq59(O$&c($<o2G<))i(XyPgT;I^(}KE)?0l(~cx&J| zJEA+78oO4fuP|G4?XpGC1C5se($tq_T~KNOzERKkFs12h9y}A}Uv+Z=mSQ+S_3*R6 zE7&=>fRXoPUU9JMCE(+;o7tL{eh<|Y068-?Umi}Mwcuu>>~J3UK5M91kNxZt%v7=q zr?m4Mve*K}JD<=$EE$bcMDD9yx_la1X_Idtv9m|$+;#C~wNAoy)-B3h6Ed^X<2RB` zhLcrpIcEhik4+D48gnH9Y@yFv(Y%IOEsN`JM@I*tpg67D{2;X=g!aL%UCQvRBDY>* zh)K+V4_PbBO5TT7{R0C}(`~ML3xRR3_L+SylT9V=0R~wPtU8iu?pB-%3r5_NH)BaG zmns!I$Z<R<n7&DKb=(tbwO9jivuh!A3Pe4Sap~4e)0NCA-0_q({Hi~OBPfa%9%g^N z6<<zH;BM2BYLdqD@a3+(*Nr(HXL`I_vr^@?XNR*9EzMxJSQLiD64~BK9~>8kXkg2A zZX2ECz2%XDL?uYPR=B}AADnW!8*05D$L~&|nvXrZ0rFYX!k#i*gsq2P4krq6(z@}0 zGUmdYZ~DWe<yp)6Vne(y+xMcUw;%8&ybQnB9q|r>KtY&z38=E>7#0%XqJOX*M`k@H zY8;PNU~*-2KC($5;*`xcJ+`<aL=RnFUyP@yc<I$r^$JC1s59CBCYRAX?JUJW8Nu`k z?78uXYlBd@o$qO%yT$A8p@Re8)EAj7zt9s>NtQw>uHHNyX7@=o|EdvL@IA!HiW|CI zg;n6Vx9wOn#8fF^D{wZqS{W)!@wyGvFbvdBm&UITZ&GlQUF|#lQfr;Vdmbl((4sx) zcvVpbrqX4j-YV@H;Si&T<kmzE8<2SzxV)s`8(-_)=lTH(!dRJ-FZj(gnbJvziVS&` zXv8g1z3C`QR(`tuVUlWv{>JgfeOnWc*e9Is+g3Gmc<nBmTy^?i*`s{e+~bf_H><mA zCL`EozMtT!K!|4eyHD?i#knlRle^9Xi(ko?jBtyT#*(*z_iZWPUbGV*FGwa`zNWgj zgFAB}vc1l?y{?ZzpHTT3;`nZExh(&o3;CrM4gLX!CeM-=1uiGyi5y{KsoeF*>m!2k z&qD6f&*B-Qf(DN<z+=UF&E(1;>sayCR6u&c<(OlOaV#q~>`+=XYn$t+U(#DC+J|4v z?x`h&6SY5N*X*ZziikVR+OEp?5LzM0bd_G*@CKmWzRT%A;kgwRJ>Ih@^_~sqe=X^@ zW3roWV@*E#G0b%`QyDV(qn^{KfOBLRe&*tKQ8bgz+NUm@XesNh(&=u(A!{sQbf^k| z9>6-`!yg!r4xJXc;(CSV@|7J$$IrRrp}x$cAoqs^C*6?IzSOpmdb|FU=r}tW=jCAh z(>j%mpt1-j2R&w0{r~|T@$<Pw!Xv|wdy*d-yiF!w(e!!l$~$^o5g%P)Wqh?4o!Fip z)VQKbc(Q+!BfPX(Y$*R-E8MEt3NLd;^$>&5)@)&My<slU?F#}?*B&_;OdzUQJ22&v z{VvR{ouqwVMCC`e2Jn+!4Fi>}{i5A{3Bls(*9g5x@11oIn(Z1z#E^i3ftWo;c5G<A zZxJ>cJD?Kei%TM1UV~8avBnZ*VjIGo3g+jxwzE4}A;gaBl|5e-s}!Tc(1ml9IdDT? zJnj*}t;3F{avuOOL>Vf}`?#rIFp7+?<bzw^GwXc=m%*)pTLg+HA{G2E^|#^g_g{@w zm{@wHqt*_yTA;SP=cS#`q!Y@OA;j!o8<Vk5XP>!SkU^IjTN$3pkSuDiW=b}Q-c)Y` zd0a5FK<j1t&Xf6^OOF#|&$VH-B4i;r3MwUZBmlhZyze)HR9q;5D1#&~x7W@SMEN2r zMMn4FEgc8#yXlgsM&COJ^LJ)Iemw|utlV8Rtie@kaYS;B--?qoTV~Z2HuPc(N378I z1~^3))0{p=xGtbbroREBP)j@TWDxi!(dmeWT3i6`cG9&lG8#?S=w+zlBSSYiRCh}x zV#6$*`&9*Zbh_aDvZU?+f2DzM15(p^=6<&f(_(1;Yg}yRR{J!b<ElF|AfHvzheP>E z`anRVkl!4tR>SaJ30PS?Zx7TmP$O##tquLMpFmWZpMR3=Cj6K3R5C?}oB`7|w{R9_ zx2x5`b=#9^SO#QZa<38MhH5p{s+IIt0wDPJDo%RQnT3!Zv_&uV%9z=V)A-V2ZmUBi z0(oZVH?8KHSBS!;D7l9JPRepv8xrwc`!0=Tu`m}8d7iw(%B_gmPdpmC;#QM0@}k`H zc|4P7`QZlS^&Jh31j}I!NWRBsV)r=89CrPwfZMNq8U0^wf4o4m%H_l8n{LI}0cr`l zls%I}q40hPLCiZR!)t95hx4^9DqSz6SI)h!vABNT<-wxc`=%P+p<D3}zsnp%7pr^n z=sLYSF8S7pLv?Cs=9Mq-6t$8h2Zs@e#8v1k*Th6>qrdulOn)0YDVUTaG~1W+Z?*ua zvS!;-qqm-f(28n6$cGRTZie;uBF5lA=bVhdbUHMU(gE{z#uWXmDoO9KY5h+&=y{%s zS;{{-s^|pZ9x^c5PT?nyEE?i!TysRo`&7j#xU<ofQys5#Bz9sWSteVO#-p&`eCr2M zI_FjTE_z{X&5Wu2Il1AX^U(Zz@J+wVXjUo6F4s+2*oReA8me~mftJS99mY6(k0pBL zcb!M9Woe{O7f0X7;Zl-elVWR@xmJmg_p6Ct<CdV-rk-8WRG&X0O?+P$Is{O;e{3j4 z=Z<K$yMApLjEg6<Ua@Iu8DKt4Z5$}^%3^exkGOfBt<hdS>t4Puov^!Bzs=r3I+;6x zP<CuODowt}Pne)}TKid89o`%;!!VGdAd`G)IspSM-}wBEg`h}2sp$`IvxaMFlE*JU z=}yuUB=pquCX$E}W^XT=+*nN+AsJZ&H6Ab((pA=ov%?#f)#uD$w1xmA=Of9@XI+Ue z;RVKbUGLJw^r)wJ-mbpKh>7@u^t{(Lw%1q0Nvn^7dLFXLzwV(bk%|c8E`3IOKBf8A zxcuNL2=+VsWTgyeU1@u-LS~i`EI(j=XhQ;Zke)sO74YFGj`;KiNjJlydm@$2j88hk ztxN|)XgM%*O5YliQX?_du(rQpn!oNmu?+`k`JQ7WRJFLXgg@E<Bm#02pqmC+?8ilz zL$wuoo#K}C-(-&#+T+fe`jN|tR7rZPo{7j<^ymn**WaHibnN#a&aOqu0<rX;GpJU9 zt)WHDUU3gmq|dh=uieQT-I($Ts@cdnJ+)A&_dO)sXk>KghrFs5ro7s>xAS!R{wwv5 z2hMs8J~pgg#=F;EFqq}ii;xnrf{jgc$puTAmV^^P(p`W2Ha1+LW~Ng{Jpf#?u(ARY z%_W~&`HTh{mF<)WFXg_rmLRZnkv1p0Ib137VUhuM$ETGmF|^G5HP^?6<NdQW{{XF8 zUpPTN(2*cFby6iDvMS-;a8`L<r=%&h{G8M$4<6V^#fo|m$lbf9N`LF>kBO>Ftap2B zhLjHGVut`gb8%2efw4LEE4sq^3PyeCPG}kL*72TTt>Xb9dM8~lcKvI>!HO>OirSWR zA-yC;OBvpg6;KHPf>y3-vO(JVhF4uyRi#q35wdB`tNiM5QIw15ly53eMsG9?5#24a z^<~&W8k^1hhMpkg5-HK0c_W!_+Tjwt4v$?$2G7*JM5y_ZhEQqN=SR0Hy^jT3FkpJl zInFq^&*J@EWhZN{%RoW#2RI6n7OOP+-O-{?-+J8u62s4`Zf=jU&1C)vKKS;u36!v+ zYxsQ8+w3$}Q;)yOs%*fUoIEadS=}}#%UA(*Ueq*kUeJ9%)izMzdp2K-E%4YVqt1Y! z1mq=sKHQlR6wc5&{f4Vs$YYkjC={~om4h%$I^Iff@$sm<IthGK4PLs(0Pp_JqY`m$ zuQ}A=e$nHmd%CJuyi`&dO`k8-!TeT~@Axt4Y4nd1Jwnup@Oi4Yc`O;f?2wkW<eB}F zmrgnrpn1%^sP~lde&tGI4^3j5b4TeBKuWjFF-RlquNC*~dfduWRVcrjS1{svP~20e zm#_Z0dxbmy!x3!4t4P(~^1f<g05e}eB};>az<%>+?5CI9pUay{Z>xKL@7u{8YX|jl zO3#w%qL}&YV{2bcqJ6rzL{@P3g9Ha{zP?|IiF;|r;M?BS%HxE}?a}cDfxLx<1CATo zTi4W+KfUY=SmxXvD#B@4@kor}s=nIjsVkF8=3bdBoiz`@r5H?;A54RcT=V$a<J*%+ zU|r}7y`OGienPp^@`|886C}}mzQ?VecH;6vgS&4$<7=#wN%Av8RStIm30;vq%wS9d zv(+KY2hP6@BWmhuZP@i(OEOnEfC~g%s|j6H-Fw%p)yv;w(4fJ^8a8^c%wk#lx1=Y| zcQ4>_J=c4q;%*~YI$JuSE~o6gP{_{hR%owFhwp1-tjo$JV(vQniG%`wG5?s1zAJwJ zz4Dx`M}-2tSp{E&WzEp3_ws}_^@Y6D>+l6-aIW4=L23t*xJrH1G}dQ!7#ph+Pbbm4 zko~S7qOwuV_!%s$?hRN`PaskwfNP{;XW#{1!Mg4!mTS6}FdZ}F?fa7hrIkTB9E@uN zmq5tWyO9g!srHao6Ir%!Q9H<E#p$BjJV3l&(Y@4JkBkT;d4=m^9r@LuxL#6I;yEW3 zyFKM!B7OJrb2^@~8R>8L>x{%-eJDy2%8R+nN${db3w3i?JOf3N(>(wO=Ze0MSDO}V zX5Bisz<z`p9C};aQx?2Z2UKz5b)hl%w}MY8_|bPSzCVwo&#b-td5==I<2VS|YNDC& z)nV1wNCn3+n<E2u;o5JNU2&h(9aVAKx=c?7*Wa%cQ+|HexLMq&WWm~GV`vj_XXY^L zb(v!U(??`h@4$of3UgtZV}KmMl(ikP*oEblb-Et61h+a8j4q+Sh(Y|o8_)e(CIYVV z0e24kzrM=nDgJym2MASsjHI`QFUp`tY?m5U@R_Wi4z%zs<L9BE*pp%)(s_UjlR_O5 zc?=yZ?8}2VgoFa=gf+bkTEKh|9Ze?FAudfiZEYrv7T_85dkX$vFGD(OeJX_(I2&lV z|1lo4AP_|l&5fe(>J$%x+NP%en7B09kU}P-WIPzWc4-g`nnbDr>4mtH+LKI1V6MKo zi2Qz^1Xe^jEsGX#8t^M5a8KOLkOP!`Tw)p}gpmmc(SMF;N=Lj9UV1u8HmO!PI9V%! zA<bg|>yF=!hH(^qJWU3k*DScuXaTQQUr=tp?_w>U@e)G)XCpzQGNg@{cLqL_!CEO9 zKqry|rUeqU=FlZ>W9j|ni#VXEVJXs_%b+P~(pC~=AjpxQjfe(4GAwcH;!U#9xvSqy zAUn;KJXz)mIP5mZP<{34UA_;N9izb;I$3f5dK_=*R0nU8iq4(E1G#*oq&`E8k0~UE z8dtX1o=j+|xt0W(2JwFcwWUuJLs8XIL6sKceZC}``y1|?0vakV;F?hf-4Pg*tq7(s zZy3OWZQdAxZ=ah?vKu(*m#)or*L?fwKDe={+v3Zp>~~DJOA+Sjh%K>Nh)=qhd6W|8 zrHi%W6d=YWzot9I&%b*6g|_lGD4=c1tkWc}T{L{tl6?=VZUwe*nGRY;meMC}NE=5P z&gGfZ1B$<&`lPGQ4yWo}+^+5Z*%I$@mk5_t`ICNRX%6DWOAW{&-DFh3pkMJ*rLM28 zzaFY$OxST&e?Z_7H$6C+cWrXU&Yj{20%SR)=W+(>oQGDs6nfsvUAREOOGE`C+GMt4 zttFH^YWfU_5ZA+*alvfX%-8hg`KSe^5xBpiSEgN76MWyzzig?Kbw?d@V^z`(IY<yv zTNz1KqCmReng}SlHW^kB+lQOA5uwBnC1?;pbuMvZMEXke_im1Cl3-}(_!5au)Vkd$ zl)x_zvt|~q7(>plWRV2^`spb`hExCwm!tTb*1#mKIL#MO+fy0zyuCYKfdcZy_K#Ls zNP1$<u<hg>%-l0y_5a4f23n$S*wrwLqTh))OG9i4)?(?hz|@fFa8?Yxn^=`l0wC%3 zd8h|b>r$gi>L@cs5PD-e4T9!x|Ni_ED>+;NlC8z@tMw?v1?B`G&4MHFMg=cM!sWYC zNc1aIkM&<v5?uAS&y6>M5-gpclp#H84fx56ATD=ECMFtrCXF8P`TeU~oiAXNF98_% zcAT@tW_2~#O;49{*7|}22KW|kHhs|K#lvYLVarm}fOydVtINqwT69PwAoyc<ON8Xd zt-LY6Z!Zo{;sHWulIF@XD_HU1o+r$p4VMt$CYHN_!S5v-ki0h5axmeNY(Qv|CQvG* zE3D?3Vlj`_tw-S2xPj;RuVs`aHwk|MLDVNe2mMKAAS?;HcgJ})n!*gZnb7;gfAQmu zx&=D$rb0SmMf@*DUa3WCLSm`k;^T|ZS(j~pVd$BPE{<B&Ae1{;Ys-q&4!`BCPH;_S zqgdhD<4+$6Hv`<YtArXO&KY7Y@1^Q$PRMNmLYI{g%NkC1T`s#SgkG25DE=}8zfKaM z#yLN6-hQ-`_CG0Wct{iXUXKZ-wUJ7<C9tg(iYt7O=47(j%biv3aoKf@@AOE2WGtkP z);-}>g9AE5N5Nx)WDC*GZvFTVrh;)3S=i&dY(WDBec7+s=it`jk<8|#YHhlM2^!I) zGx3f3RKMXxkshQ(aj)NB&8W7T!&Xj)XatVN^N1@Mr0bCJ;s;S_EU8}azLM}3?*1P( zdnY2Tu|M=&CA&jpmEhJE3n5?lgll3dB8VVC59u5s<cv0J4GRi1;yH{WECnXe&d>F- zRh|zJ9I<0*|KMxV`J;tQ`#A;f5vXw_e$2JICHB|adTuc#khq!+Vqa!1jCGF*zBvf~ zAqXinb+T4xui3MtQV!xR&rW!SS`+hnJ7yQ2BTJBU@t*{vqztOBVAZ6+CM`3hmzL5l zChwp0vAXnPO;}I~M46^c%~)KRv2zNWSSj!<E(GheCgfVdZBH=Q|3Rdo^n$;H@rlI? z5f(#}ncWl>A1N8>oDZDsp3pei2Iut6DI*8;d_f!_h~^O{$xCZd`sZRUA3#5WM)W{6 zgsO@Fhg6lU?#U6nPCT)o`>co-#usp#R>LzCm@-yDK|9Tk)ReaWF?ULJWr@^Dy5MC0 z!$!gSc}yE(Hqq?CdsQ{*n-e$_s~H%!xIZF{+8+ak*4;}hS$%)$AnSB7YLLGpTossS zYRl^Vn22)dNmc2%a^1*tSZ7u+t!mm0lhS+XcKSS{|NTWWY6Z5p(g9NK&l3hoA*V}7 zcwEiK*cYESMh;1qtE}@*?`GWdz|fohr(WspOZldM+zSGE4%Cl+shWSS(sMuh^sV;o zR!3tsnAkrzCg73^rDx!7a7g3A({5@rzL(Qy%CSdR%UxM{M9A$ER-R=^2HVp?878Xz zmjNZ6Uwbbbg(jDMmvy0giPJbzz&V^<6idtmPuEJF!v`Ag$X@$j7dqBX2#i>JHBHpc zd=H;^M2fSa=7`d%_=ndlt8s-b)$I#2?i$3zSro2SD!B;eTNnR1fviAwNbK6Ks1875 zNEV8pLCA#nlAnE%robL!mrm=X6_qVfqD5WtWn+&abAfgc=FyH&*CchMUIy3H{6}kK zGRJuT=-~f;<&O8)TKRv#iJp5<4pn@nMmbWg3Y^~=*%JVT{{R<G3Ij=8mi=u3lF6_K zgDd^O9ZA3T24L1u13iEA`?eJl(}~Hf3;VfO|FwhwNCPO3Op^GuNn!4=wB`pw1}=m8 z_hy98gu<j}k^ZJIfdN4T)TZC>o5*3yBdYrcOAv_?VPG@;vylZ-Kv3~NDcC$Oc`(>N zv!ei7>M;3dFSFnpz;FOo@F&$t_WpjS_#@FSzVuU(Qn7LX$>=#Z8r$~HtP<zU_Pqeb z3-8@p)ZQ?Y*_IJBg+StLBUt+MY(L0ZqeLy4-8Na$pUOSOq#gY+#s`7+;If@ghsSAr zMtJ@NVb**{@DWl=<!bniqc%ij${i}^Pi&Tnoy7hk<YpN&s;XkwL^J+#G-S-&w^=h3 zdCNPOSd|#YN1k@D8}kx;5P;@4JCJP&Q*>RCjY~fWqKl%{{O2?toHK?bC7V>aEwgS` z(_XNDPxfwgnN?pp=fZcE@&nU%-Nlgq)??Kv*`Z}W0ks@`M7x`7o)4PUu#?+D|J=-n z8~!V-%>UVo+dX2b_zxG`6k3VJxeqO4oEv5=%3YUPFWsA#uD~IS1NJP{KV?VaEt_SU zg@Z@=;!Hz^?Gwa>@QoJ?6fo;T+Rk=u$nC_5c!zw{&r2KsVCwOC9l}clDdINlAK!~; za#AeG=}VPA$5+^4%ptgxFqe-C<RGteIM2#73-l65$^1hSl@zTCX=RLn<NgoP&Z;<Q zj`nv987%2%h^6riQKkH+96oLNtbr19{0TtvhlBz<-TI&W_#a$6$GKqjw1zE7+g1Hx z2OO$`69Oh-=ns<vn6wQ4_bZI?_5TM>Wcetak_H^E-{fiZJs~VHKJkUtKBMDKcXRaN z+Yf76B8)sB_|yIvdN4Pj5^(bNtB#WUeYffqcC#UUs!gEeexK#6!?ynQ<IL{9!9lg* znT(-Cs!ihmcp{%hBuQ@kZdjzUxfw>g|8{5qhZY)cLPf>#E4e&8vo!xW0c;Z|`ebI> zM-KnDW<$H{2L~(PFLy^C;Qo{HmOE2ROCM)<zREwq`hAOe0<~?N;dgR(zwdIi#oYMs zlhsdiM_}4HNf=sNCF18pJ}vd<Y2$@cbq@`lsc}kUeYn#f(ys~W6*gJ8-&nUXoipWJ zeGrsrGILX`pJ@Es57e-?PLc=uHR*q{4N!mBj&3|8t$j?AM0Gm!1h*hcJ2`q{^vO2Y zC8ii<&klE5eqs6Nv#nOuYzJqh`cLJLCA_rm&ocZM9&JZcs`2WIYN<mix$IZ&*o3JL zGu=R?t3cpss^(0<`}3OUzi{EyrSeE&2YddZH&MCp_Pd$>k;r<o#e5W_AS-(wl@?D6 zq`X8aZRzr6czoIZyI{4tf{uZBX2S@Sh*n50B2Gty45aw-|8ur~4lIJ5SBf?~l^04t z&%!R3!x(XsXXUFEvsH;rA}-S|zZ~{N$h3u4l1MRh&gp*WU8>`|CEi3ITFpA20b0T7 z|9AWQ=Zu}YaEo4mAm7=Lct0r!bPsHlrlqvh8eItPz~>W>^?QFStCMI(_}%-ghm;Lp z3ph`eUBzT0SeSA|=3*PPXn`$F{a-}ncrK>^HE*CA4%ba4J;s;z4vty46*7%ag^G@t z1C4YQBdKd=RRn4h@0DjGm~Z|e63KA%4i!IEbx(3Y$}9R-YT_9V_Ah)&;ng1>e38Q? zU>38J-3gTqCYpKkD01<LRa^HsVcs*8{RN}wr!ZVdU1Pwf$XW89&k3aH#(LoEf;585 zVSYRQ-XDS^br^3&{}S>ZapXV>(eI;}*l~|r-?zTv^0&~&8p@Godi)`rm02iv9*A4e zznj59TD%JV-OLw~M`zUuCW)CWL_ICOi7+>_ULOc{!1<#m9ne#+HR9hriOQ2!lKejP z<vAX4>+Qn}m-%788p_h$IVXI7Cr|}F67kDWvqwOd>BZ%D4x~Rm{zcX6%dkrba%zwK z-&p_vlqbECWl2e`+kUOfSZZ0)*k0hUi2c=IvMOZR>=QPU_xo&rmS0Nk3{K*s%*^DR z7Wr%*o*|N*w~LSHxswBbAMzLpItwq*ZYNOl<(Z<<EJTbnpePs&e!X7oG>wOcUw+3~ zj{(N0ml@U<`iHu@wvuJdpI_(T=iXG3){aXEONkMRJ^FU$dv?cB3}~_PmajVC>TuI6 z)1{ey8PeFxJLilTh?+*=bhv-pmgURKtdnzeI~E$!(N`{?%6(K^c@uEd*UMAy3v+rq zsMbA0akSUVlNC@{a%1SqPK>@ZY53(G=jF7pLFgZ}<fNnA&AkS!0<k<rqwRrzS2OzE zQuClR=?juRjTPeG7g13oLOELt#CzAjlGN^tEd>RU$%H?mXQta30PT)k<2|+{%R#KL z{xWtkVsqvh<H&QX=R<ZFHt;rHxk`U0C7w^4G>xRMbmdP0@<3<#Rt39zFCR-q;{lhk z9dNX|X+`|baXm#yHahWwz!-dJ)T7Bu&u+Cj5cqhB_c)!XC3h{1%pte00}8JGqCia( zvMh1`6+BL2LmX%W;0N=Fb<t{5g$0f!D*qvPfS=3XhYrpC5AVRP0ygBI56*h>jNi5E zdO?`ZN*2qxU&jXRL3pk3I60iBVJD8B2;6o*5-45nlP~wXrne76$n*naEhMHb9C2X; zuE_rD?2wa-N|}~N@-IgR9B2Oh{t)m)TIJIl0Li8McSqy!p)&Ne43eJ!_7dYqyv710 z7)jO|l702ZcGRN+xsww8tlB-!-HNfnNG@&=zI4XZc#!dT>*a?WY?&xtMs0`(?(cLA ze98x=SQfPd8aog5RNxoaz`PB-i9%4s%3(MbMUKYqiI01d4TgdL@7MDH#JvPPl;Gry zG~gik@2-YYwaNk9AD<p+HDE9PFLBlLpg@XXoSg#duYNKFEc>5M#>vi~hbg@@1oYjn z!;}y)I>g^`HH*s3TbdLXBMWaCZC8Q!&t2}9OT_^R?Mzk8!0;H?BBltfm4Se|{^j5V z+wfQlB94S8OHv!-b=I*}0AABS)dHrJfd7UDgl^`S3c$EAV+@0!%lLmD{(68{Nm1k% zz12^X$z<+60+<c-&S1XCf6dqMFuY3VZ-QRSQ-C3XqtCz2hhRvoTAis{L&4{PcmA3k z1h}V%VzK^a&`<j==`xrG=YLN_0ZNXu#+`zBX%k3!I8JE05gJIzlfphSpbI;H(Vd)u zxR45kXHlH4F#TMT)U4945JN3DJhT*l9e|4fc)AwcJuX(6KO38rc@?vGpL+AMc73-| zXFop(=g(M~Nr2{O{(BB!7=o8rsR_-zXC6(To$fdNnB4GRwa}NMS`Kt2V6ryL47-I# z{nfe^HAvdRh%37dJ!b*h0*mG1FP@k|Ig)$FK>E_80aCE`jS|}+T!AwJ!LJ1$i7S@# z5Om)J)8I7Nk`zq8KpV@N`7)dp)5fhNY-_oML0zytC_rvYfi+F^i!ExVkn_tQGEh%+ z-jNMRaZcF52X+}lVtgX$`#3ucAaiD~lrAW+o=;S|fXw@I8&gw<MCi!8abiq(VswE& z9KZJNZ=FBFwk9gN440Clh$+({EdtUDFz^2Ie3pAIvBoN2y<no%nf)w~UY>Jl@DcFp zGa(rZ)51hmn9duso+pP(wW(99?7me-k1YRtY91pl)Cvnj5=`s}mNc_)VLf0XHwIhu zh>_1LE>Xm6YP7X2U?k-?KQo~6JVq74vMFLTv6eJ}_(MrxJ-XoWSPcbv^;qnk7-h6O zHVAEiNC9w|I6MYEc?nZkHnuhXTZ1oJDpJwZBuGXeW+XDC&Lqe$n4ZYBNE1<NnA!Qa zchHay9p7oqJ_?trtzF+Y(*mCT%!l+!m;$2#EE62qu^=QaklyU3+<&N5Fa?tpbXhH{ zQ|7VCQVMu!iiCJDK&VoCdYC94>=+O`AZzW$3_=mVndwGDaH)MKyyZP9QY<qtZL&al zy3-<`7pCDgUwef8uWjTH17_Dl3yOA3Lk0fabBqCr((h!V8A8a0T$XP4$&fxGL54dZ zSkDtRT@3+?5nuxU%egZ=0Q&O4gW>87SjpnR8~|;KnPK%8ky1rzVJdjACu4t@?Lsg# zwtI9*8XguQ%m_q>$b;!mxpv45rAgCv|A+#B;7GoXve!iDlD~NrJ<UpH7$tD>yjhwl zk^oIBkfMp3zk}6z`<DoSt$<?Z+?Pl6y!cG>_)??~%$8=#3H})5Qv^l+d_L;*XuN5~ z>=P}LmobFQGCizg;q6;vbx3FUt1zdPEFYX-;>hkkq>~*dO-dR9g6kP!?>Lw<x&PRx zMZsW5Q-zSnU|nEjU|8Wdb)0u?`R>rF1KK(ZxT*u!+rhNmJqH_sS{w*66L_z%!s7Ka z@%&~b|5|WABS;WvrxM$(?5sKmoXKPmivosLhh83973^%=*M72GLvUy~YaBpfmeg-O zUa!1TYm8xw5YnRZ?OVW(e??phVpG{0NG8>@;#43FJ3}h9bI5dGQ6Q)s8DW|{yx)PM zpw7P2`fSjc^y}!7^vi?40y&t*19JhBLe%;>*r;l!LEE<B^a88FOjN<8;j9{ksGYZJ zXaP`<kI}ZenRQ0JT8<X#9L;B2-8{m>4W=DwQus`Y?8xg5*I1h^BFn}Fld$C=od&P) z99Oaq|FWt;{BZOdw<bk?ymEG2r!{kIxL7M!#uea`z{GsBgXyK{tWDmZi@ZHnPjc=9 z3LXeffk3@_xib_fK$CZ1?lQ5;7_CHs;cOn;WK-cF);Wx>-;pwmy3WeTOaNYbtIB?! zy712Dv0!gOWHqEQ1lqZ=<fs$1Bg{Cdj?vCbdBmXLsuB~4*0R2oKwhcGdY&aNP=)n8 zEL=Y`%y(8L4|V)>U14;IWKl31II*cKv9k&ZYxe<9@})`RNs#9}z;YkNya#lWf-THi z8`3NL>m~h6u&JbH)`BUxd46*I#u6W9Zvj@^jivJ=T1s!P{M3N4hJU^&xFj*2qT`y+ zmi7ZljFHb6ma8zvu{1mlp#7+A?~L4+1_XYlGzZ~JOuSXvs}5+sp_|!}NM$xgKboaR z1kftBz>d{al4>)cc7n-M3<L{;c>iAvn18hHGpemD1?5B*Vd$!$bN7OPMUIVJavzW_ z$c_no(tu<H7cmno{R|st`2?=R1(*638R@k45j^?W?r)0I>cjxI*#~6r;lN6TFq*5{ z{<m;;{a@ki^~}rp*jc1lleu*6ikL100K*2b?B$ux!wwMgU=#zCO(5}rbO4=C@-teF zU0Z?7XOs4F2ZErMi8pW&>HP|fg0zbSSy~+YS2P^N?EenJS7uQRm`=#-yZ-gK1|h5i zy40M=oL`alS7@K%XK4Rbny)j?WDRC9h0v?sSW2&YNFPqObIz3Ew|oJnT64Vt(=nNS zThJI-#6?$--(RJLErc#Hr5;=bIw5#nmw2$D;@V{JYNi<MU&6)!8_~<e`x?5`j6R_G z`H+EHT3tTXUuzN<skAFY+NzfJuiOR7S|`D<^>U0Bz=0hIh0%{iY=9($drR~}^6n6J zz#+XIMlt!FK*R#TXx%2A?9Yi>{|X*|Aw`M?G<~&B1yNa^!G@^c^D@l;%G3Cjx6$-7 zk0U&KP*x}@3~UPcwLBouze$V?WZwpdYJ}`Q0u0eDg#Ia95o6}1{on?psmj1J9IC-& z4uGA)8&qh553*Tej2#JtRv2T$fnTQnzdzu;#CqZ)iRmSVh3IA|CvA2xm>IBV0CoR= z=%aTfi~r}Nc<F2iS#kF`gc^oQ*O2u08aMzm88a?GG^h^QLu&r4+7-PpFqDh?1~D|a zs?CaNJ?V{&v^K-d2-??(Azv<cV3mM(6weu*GrW$@YPjtF`DeCBNX%1($kdBQ6rwh0 z0Y1LX-xEWCEoZ?P#`u^q>NSEhu^Xd=r{F)yXvMP4TCckt`5u$;5%!ajWO(?j*(mQi zFk!$Ei-SF+`AA0E>93y%F@8osuhY}6#qEzEtpU)&^%Ue>{QWn^S_j|Ni*hFS+VNoi z#4iK>eaz?PLY^vnW&`F=rdDu2+0Rd!Ukxhf4*#=}zl(|&@^m<}|93C`-6ME73tjKH zyOHemvT=_TYUsw;2(M!~K6AgGe?H3UT0yF;RqC77XuR%{B?0Z`K1Dn=1wz0Ce*gm! z<7t`P0;lsBA{u;x;9rk}oMxc}qtGV*^$|WcH1@vpN<Z|D+Xq{Z0qfT>VE*s`8Mcrr z(UcM3sUrioFWj3x$Uul%eR#jjDAVx((?`;KgZBJ_45K{Mi`FrCYVRp%V?cqOBxQ=v zs^_j@NQ@8&vv%}Y7Fog2I^LT+O||>k(PoDa<-|N%2p2zZW#kgs6`{y3d<xiN3yY+g z+~(&WuPYQa@W;tkvOa!ft|y>k1n1^EUvTeTna@<Qq_T|8WqBo9=T6Fi@*C&ERLLfE z@x+8j4DB8kl}5boX#R|YVu^bm9vHGq(vCx+%vp`>B~hgnp_{k3pvlZ>g&^jfvttv1 zTS^hTbhwH*D>2{eY7Ij@$E`VMhx(9fZ22**QVrYo&6O)g<TC7!eh%<CR6UsZmIEQ- zr{>QqN|zm4IC4hL<=^(4AQ^k(xk*f9cRw9pTD8J<d`wR$FJZHV`jbh?1?RX5IV?{{ zg~AEMEXzH85n9&BzHBDI!eR%t>O!QAsy?;|0r?7AEf4Oi-R>zxks0G~ubWYX?$i0& zE+yea**NxWRwm5VLHJ3A4m3HzoFlgi+oZLY@<^&?UVCU^MO_GEcK8$_V@m#{;)WHb z@2Xo{8D`jXeN|V^sa>P)(}spu+$&w-%SXttq$#sMwP$Y&sr$Te&s{)FL+0GIn#~CT z1jIQ`KtxMS7@d}`M6u{o9}@EnNSyUuTZ^|2lUTcqcDO=QLLqyE{%Z#ckck)P2-s*1 zCtR^*l$o)_C|koUr5KgVJg`zsjT6hI9kq0W%a*=Be>|Ej=khhw_3Kr8NKBU67Y8PT zPfsd8+~?1)piwN{|EOqP%CeNUGAF}Hu3JpWgqRJsW#>~{g!*w$DB>r9BXUR&R~xd1 zs2)}_$3M#Jdp8m0H=B=gz~aywwwT9pC=K}_$}+&_wN?`aPQ#yl%bUy6Qk7<F^=$|x z=yX^})}S9YT}+^_YhQuOnC+Y#FNi(NUHIAMm>c}Yc&{?uN8X5=#2|{#2Deua>Fjlc z%5(_)95(qd&LaQ!J@?3(bSvJ+%?O+(xMfcPUy2el6R!2@W2dl@zRVB0<=vGDYWF$S zHE3Id@@Z10G;gtF$kMH06?yUWXy?^PEwi6yktog&Medm;?p#{Kr?4a(_(*wsRa}HX zE}cDVFHIcvA*k|Vzn&E|oQ(he&l!HQ45Dnt+yb6ZX*#sE%r&t~VI7|76qZ)qkw|F1 zP{Mg;qMVPZl;!BBBfP@>SeH~qQb-E=<J~DzR3<*uxoZeklCKD&o7WzpVr<!M*Gyh1 z;sd`etIXXiR<$No1J4Mf(1<Tq=pE-?Sz{%M+QmD@x0drNicu!L+-dJT1;4B<-@xP& zXw%#tqUbdfAxOF#*VZ?R{97LQa=PyjntIj-4HFy}pe~NNnQ5r(!Q|@RK|2_r@KPt8 z39oZ%jDVyZp?6tqFB+UF_<LES(1pABOgF&}2G+l7V)vDguIcc?1r8QfB`j%+=ch+m zM(7cX^aM;^255E7hMf$7AtC+Idxe{<WzmIua$tW3&@z}&d@#ZNBov~?{d6y$oH*61 zFuQr%@%yLpw&MF%XzPtikGR}yJYv^vajE8zmgTe~)dG}rX`)V(!>|O-&t~VSn;l{L zUt?<~aB^j7*vr&!q>$V{sHb7FW(S(w`YvBi@tDP@Zm+&iyNa5k2ZR-mP4R=)O0%Ej zJ}J!&w-Afc3c^d~eUs^H7HS<UWpVGJmNnJs$Aq8?`%m=<8Ftr8*C+LuoGGY#$`N)$ zNuF4q)g%RNw;uYLh<J9A<@S?MK{kW#QPiAMtrm$OHF2|WJ<_<?$Bz)E_+Q0+c|6o@ z+xK8Fl4*vJeK1)nWE)FpjIj(uwide>St4bbkYpKKb`oO{lC902lC0S!TUjb=WxXz= zxF}b>=jguf`+eSj-sgUv`+lDO@rmE>cW%ddoX2r|zsGUne)h<sD7CD6!<?#ED%dxW z?QcgI%|MbHN~5?iA$_U?R)Z6DbpuZw#0m)(E=S&>o`kT&_9Wm`#%ZNjfnKW)nq`86 zFpu+DZytOOh|JM=&c@i<4N#AV{vvR35{P~xm1<P#9UnEGl)vh@dBShG2R3eYEvv7U zd+G}T68&thrBHCoSn|7$wQC;K>+{K()ItZuF8745CdXW7oKfqC`%x9;{IkwGyr;@3 z*zd0cyxDUF2RU0VY3Awzl|>LNJ9=<IQyAABus54kT-opnwocxV?)t~;dch4HTplP% z$g!1)H=n=F>V`r?Y)wg*B>Mgo#MgX`e&`ImaR@6_4;vFF^)ct0^WzThRJ$k^mD$Uh zWjX_kXlu(!!_unCyGsRLg7_AT`GUk`U8MOP*pmaME`q7hH3h<JdvyiBWz*2&%xpHa zZaq2QD4DYLUJo!~8OO8-AkisFXZu^+2j_SVU^R#v@+r}0Y~RU^{}Dpsqx8$DBVS`v zi|cBl*dI#sAWdVraTh>H5|@0eUrt=H#qa9=hJ5pRWviO-rbnEgU)E}%A6yrE^W_st zoJ0l<EOfUyidqlS4F*?V_iG*jE(_o;)_}55;)cTcDWw8`4CQ=o?(RQ7z49OsL8yCT z2>!|-9~(;0?(Fc4reF6r%&(o3%b#F3UVQFwqgKo@s{PYqewOSx4yynEWeM!r8vyhE zk2c1?^Z^l^5_R;&D>s7_x{(JKSZPN_2^~(yY-<<Gyk2Rp9BX>4V^kNgWZ5)S*uSV~ zCi_LsuD-jRI)yQ0t@T|-t>tT9`)a@{qtBrx+SJZDI6Q=cJ!X|FaXqJE)tjp<t(wa( zr<W`Ym>bREtz-dSO76Aao`&27N`Hi8Oq~xPh8SEpXfIxN99mnTR~lI=aA{ae&FUE` zx;bQubGbMK$gvZMXz#KfOEr;>w;2{PgC-WXK-<m`;tu%+^}{$1U|yzd$Iala+4kR9 zbNbm?JQ0yEsoJK^6FK>;)fL{AyuUffD<gbTvmtqiEkXvj#VoS?+V)eDT??LV9RG@Z z-#E@Y2j}oGY1_}SL*a^dzhi5lHiRT|Wwn0O8@fFpZp2M*cNHsv8fzm5g3#}T)5?k_ zhcBiHub=gO>wA0Bk_gphW;)_cZgo{_9PL8!X6({@$<Ck%*1nl1LcPLi?(vYY!R_^H zF;SB&Svj=Z;-pL9==s6ydarNSd1QaVdbaonH%z5$*fmP#ko|6)>m189dxQMBC-Zmh z=bzAtu$l6QLbzg=^crEoU6)v`jM4inTRE<^x=@}j67y=mskZdMb1EP9l9W68Rxa!I zfP4@sD_dKG)HA&`C1r(0k>kavL2q=K)0$OeF{UbwNQ6gq^?`x--;#a%YkW%ABqN<{ z3Bx&zG4&TtP)GULo86(1lv^@sYnSKQrNo?ViRmiStKF}^7@6vhAJu?R-rk1U)03JV z)wN2@2f=_|?G1?KUf-5;$1QQ8`4Ry~R_|WteJLGGTYD?y+r<P^W2Jg*-ik<a#|3Gv z#ALhJnyYA8G96;0s2R=Ku%G>ti+nxYI|tmCJVE@lDKTE~qqBaZuHtN2-|E;GPWkT+ zv{LCft0~x=;?9&!aHra)is*^>nF>d3G<$p^QR`H#!%cUnbr0O|1kE&vSkqWRp;)?S zG*jMw;gkZGgy|P}%A9cLK(5y21GTKKmCycLg@(94XCOD7|B#F#e)3Sp{`P{XW2E)E z0XF6V*hK?UxcY@+Hm;)91FPMb@zb<Q$M>WTLuect&o@AkKz!J~ylUF2V<3cA-efB3 zOhwQGQ&}*y{&&ieNhZ=@sQWSvj(75gS)HHy9XD1G39mRXH;Q4_BCBgj9StI05AVnd zk;tKsidtlh$0p{^UOk_~dT%r#{PI(0XopH>743b2m8SE^qo`GVcwJcGhqOt>*|c1) zag}Vu;CB9OMtx$3^e;(j!$6LA9+DrxrdZVF58>3eZcdAyUrU>Gvt8EbDfVzSRzp={ z&d)#VX*h7QI@&4GDGD8VAHJnHnGI_L*MVlMd|{IdcfjtF>o@o5!^t}9G`8A5*-lre z#>1K^YLF{lLQJBRxB17MW23-@Mfuch%Vm9R5QswVKr~4BC^6ASI3#SfVv}<-c{JJ6 zk8K@tHlEv2A<SpHSnlo<4yqMn^IsN#jB_ZD_O5$%UKc7(7Z*Y$_b`cOJbzO=3V$bW zPk;_mxR#|VaQpMOeOBhE9i5|$xg}X%vrm5t_XD&fi}S<%+IHAv+%B0C5`A+*+0g54 zas8QYuHfaQW#w^5N7agFbzS?~1Fd-zZJz_j2_=;4-Rbcfp;7m5O~T3w-yf{6sHDB; z9S|;YU`M%)ePb<;`cPTvnxnpH3}whfLy%x?;s+^`8AF)CoH82_zuXtRmgK}{BS1uY z^lR<b^JXD!cFC`{u*Ym#qlm=akf=I$ocZHRF8amB>CKA!^9jpidm?e1Zn1CUU~}G6 zjyoz}LXh66LWq;qb#`Ml<~?1)b$RYSKZfnOZMbs=*o;)@G(V;zu4l(O=!pt13w1i9 zu&YV~$=RP41tm4Iy(D=u5ECJ{EroR3OqXD{LYRSUzrCnPpMQ*xa@q*k;Ue37GFr>F zR16Z#l??Klj|!R1XO|Do*{1Z%F%<bY8#Z-l#BB|#oPIMfi|OtDcUZQ7`1xLeS*fQ7 z#lsF3TWU6VNub^-i?2VRjT%<tP;*8NGt`b6hJc{V$q^ycso9JRI-Z;;X2eN9^2>5L zYg_E9O!%~L`I_1|Z>8O1rnvbD*rq%iXx)S~+Qrb37C!TD_550#n$`k~Rx+7aF+;j$ z(Wa+otEM_?UfKtSPm?Zw_wh;@P38dV+v#`V)_DP-O|om<U{pJXbv*sLC|t4-iNfdb zdr?x!-~i~}jnU)?AW6=|UADh~^RUQMC4uY@hP$={0ws&129>_P7HFiB^nX7}^1t*2 z5&i6b*>5TU24Pr+n)`B=H7H4^j2L*Xk)f`Yc5Ee`V+ru|VM{U|6`v6p9dW=d;uaTS zHQf|$>y$1AX%7;`cMHn%^~Sd;0U*_&*p(oaVJH$f$~dCQhoLt%$<nxge8;E@F!3GX z$+25HFHZT6miBmvFiaH2N0-hCt`y%h$!b#=zrrVHi;h@dL`ko|^B>H-+|yxpiNe1z zqzkvJ_tq^qs<WXuG?PVXsj;Wkywc>PE+&0b@vQj$sqY^-Ci48HQBrsFA-Dn&GZ55T z0+6{;mjv@mkbiBNhAk7A=hsGu#c@m8%qWIA-xugm3u{y#I=yaGXs)qt^0Z#;+N>cw z|6@H?x#Zg?yGyY-$e+K5{rr9(<D~GT^RkYoC1WnYgcx-VW2Gl1yTh~PBO*-+@D%N= zs9L47Q2cK#k~h1YW7}OTELPYzdwn7Zrh!r!QOT`NRN}MGFW#GDxM8gn#I>xWIoq=Z zkBFg^c5(L!3b=;R*XneU9?%1OlPY4uk_SCm3)^jx5%s^=Z?Gj%9nXfiR2|Juo8!+7 z5cHmm?sd#Kv&*&DLS0!A&G)H@Ejygq2%n;kM+BqOd`@y3Ln)|t9sGG{Trb|=r;VVK z&X=DI$VRCEr-~0yLI~HJRg9?syA#``(l9$#Z06@@X_B_Oej&iJ$<MAJI1k#qi1xmj zgLI2BrEaHXcPd;#^QP0bt4%0b-Y#d?N)}k8^{=9VFTJ<)8CRBswzYZV@jPy>&?Hl3 z&7MpR%;8YWJhVp8?J4Gr94{lWG<Ac9mY!SO%z?!>4^{tJ6jaHP0`Bk(-OWVQ#5>f{ zWcX=$j3DHmM3C&QRM-1R2Y%d)V=8jRH=-hsRkKMr5rdQgZ7J<Uda3tKu^V}d04dK8 z*;_JOe9DJ1=YSOI_tuiNSe}!)B2Hvo6Kd1=$4R_OjSfc+2kjs7cDX-K)j3}l_)rfc zP&U9%KLhH{x4g1iKw5(9i^AQXV%DxJgd?%a4{Z~3H35+LYh{Y>uE&>SIBBTvfY(Tn zJoEqC5tM*~Dr5dIp*wrCo|+t45&}Oz&oc~g8|1LXYwJOA?@%0rR|K|SwwF@u;s_SD zbnstlZvPKh^#6lRq}oXQjBO76<2L_v(~s!44}WNjIP}4@UTZvgp8dVu>1(t;jJg-l z;(|AQeS5&(EE@k76WixGcldfB5_bVqfW%#hy7PX=Fe~rxEF@faB17MMUtVT2lNzd< z*dt1clRCkolQ5TU)c?jg#9aWCbKC`%zmp#S{>WbpFVMih`;lklr-9#pjN&+N{KRZZ zt-L(&d|}_6t7gf02qk|YQ%XV&{Zm?jrM^F+bVS+9xXw~0u9mSCrysp+$;ejo1c`xr zBcP?2;fHx9&XRZ}Tt5XEWM~>xjN()oa%akjrFzdO`F<Jd5s^3IIlC>_`ct|KR>?sx zR1?;kZ)uIf_E=dbpfva2*sSMi#ynN$;X-QN9g873ZYO*IDH#+MCa^JT0I6-jP&$W2 zv<3~+Z09?bii!g?o8hkhiP27Q6%m{F&HJVa*ujN4k`4!WUlmsM<<48Up)QP(X$GTm z0n(<9rXBfE9us|5_|B*J)uVN`r5Lzhc-I-NYLm(lmn?U`D?o=OeG*`9rF4^je3|Si z{A-E9A~A*--+l0|vNs+5=#2DOXM|oHxFBQ4rBuJ*Z*QIvlIZ0SJ@GSm?ef9Do$V7X zZXZ6C^EkMKPhtzSeOPWh1T91%<|9D-wui5mfYHuKs0mDWubo6R%`$3lQV;&`3&npF z&?qal=QGup0|G_@8p;627DiRvo6uKHdQGl5$&ZqLbH93~sVf(u#Iq+cB&Y2@c6m@& z$ze)ZzEp3SQ<0KNW!O135`cP~>))5~yW^=kyqa9B$zmheW<R5T{16dmB=P+_YNZ+_ zcu&@PmbGoIO$9DG{k%DF?pTT1WJjT1>;(xs08j4+vQ^BBO854u4cYXn0r$+pmH;me zCjM~7Qf^%mzq|hK>h$&Pqu4nk_3+$uY!I>g#r5wUzJ@VQT9GEwW1m=7ai+vU5O>(y z3`R$>1)xEKuQiLl%R(P$)l>zVqahuHhHIrNt;c~-CA9gdzn%2N4(j28WY1v)HEvot z6`*vnAV@O2Pr`oxRxuJYa4^l{NIeNz8ISEaNPxa%N%4^CMrA}f2XG&kAhwh&%=e0! z3n6hO41<Sc8^_0VfejW0<EOVrnFb*qo|#6OLlKxX;k3g%tJTr`$2EIiq+G!2#6?+e zmCOhL<_Ke3!aP|Fx>|g-oWY?gdiQp{?=-wQ&7KY9)@@vg5JvEz;^7rM;Mokuvs9(J zB6d`9gqf0PR6HA!1uqou^xKPxLwV>O;*qO~up0V)G+=l!Oix=TmzVjQRi(j+5N3&s z9Vsv=j*is$18b4$un4D1_s4oscQa<5d`a7-pXD|#h+h?up#k&~<S<fF`6A9mY#<UB znZ{N8@BxY2Ohp<)_vaEK)2N~yPp%5sAL&|PA);>GhF71F3})lT8D%gU1W4CE4LS-| z_3S)I{c!oi^!a9;hkRei2ZFB|HT`kDVf@AQMu(gCU*Fc1+6bu!*@X<LG?D}>>2AW6 zRp5zL-som*Q~*H+_n2>Ty3+aQ+JhtRhf#HEZ}p2(Vb!u>lG*m<;rI;1sXS}DRPSDz ztHU*YUINT-eLt1^7f?xs8gW_3{h=|gOswvMD^_%W^vvC-X>b#Pmllsis;zf_68}76 zux&>Z$LdHyk*kOvtKGdd1I-e`pCj}oZW`+3dmuhin$6xlV}#p{1mY!ehPzmmq=jSR zmXt+N6fo{ck4Hz6eR1~sDl9-yz(AzfZM!_+X((>_aq-Q7boV2YF2o-aZk4ji29I{o zbk{8JXG(Zmm=s;|DE@fhZA8!WV8&(u{3{J}bSlcg9qX^2p}wyhX$P(^C$wbt!qYg= zC|Z3(l^((r<Iux|OMS=dgtIm<BYEF!W}KYh-C^<ZiZZ!hD^;xbYer#Iz<_4_5%>Jx zv+)_<208!##U^NXhnH%%0Q3#M!g(*lG58~Ilm6bfgi)!t%tDs#Wd(${JMH}@*{JrA zUpUrMQ#Pyb&z->=Z@%wg(#b)nr0R(W!+?jhPbaSAtYfhL?|U$^hD3yLq)abK31MUm zh;GJG8S!`t(UdM>Kw?Qa&X|(!=lYz2>~aCZ_@7vqE?oDZuAnYTg!L(j!8|Ft9q`O6 zpc6a>N>C%@JSvdUP&_)KU9&vGx}x;<UY9f>7AJ~A?Ik#vm&&{`E3ynCUA^tFOCsYX zl4ucYl4>jzzZ17lNrFQr&-!R4y57u|dQNGPQO+Jv=6G#;tzYB1h6(jXk$~)x)faDu zD`SxB4CI*d47r{$K0`r>-HkTkvRyo<Ou!yY18M^>DhMkpd%$$+P6D51Jmt2|Z6jt| zFi7q?(h7R$OysMWoV{4^7XdksBmPE_w{<dUmx>ZlsSF&uC`0&Tap|pav$MMHd#Ly@ z(j(qVmlJhSiLiuLrCC!?F>4y<MH}ypT_kEzcu30@a&N?z8x$Fe#?5{N*!joIvtP5O z|F$lFLG6(#`+h~BA)eO9_{Oe3$#M)K`~zfHz0=Y23r^=p7<uUs{$Gs!FDbH^ZXE+f z!okR0fPu&OIX7TIB^q4Yo3uDaxkS*jI5oq2Bc5xA_d3b<YrFp^xDUda#`kKo_9pfm z;~Zt3YyLA~>-2Z`J5S~E0B-;RdU55;v$5vsyxp1)mD{g7`EP6O`aQg%iTw?A-}Z*w z_{Ad^vK$}RZ)SV#);q(wZdfo)<z1go;5KY2nb;h0xG`D?=ORyJt9LGmjE%2(Mx}-j zati4LZ>B-xXW50W;uOH0#=RS@A=YbF{W&t5<NlMhQT1n1m5Fa>@+46;%<gz>OisH( z1T!viF@n3A)jXroeg?88f#6EpG~gjx*{R&%rjay1U(D&+%MYV8M0iG&w>w?gu1dx# zQ<)ksem|`9;4Y;_BUomYm{_i7JC*D*^1*FaN?am=s^-sDcMo8~ZKjvjovn7)$LWVF z`3@dIn&-l+nUgkBZK?Y$4$GSCa<gaa>EW`>+Ddx3%87Xq_a`^WPq7US6cHZQ!ZYKP zsf<w88>ac(FzGIdh!(ioCUo7TK?Jrcsp?jkxr$MX4q<!NB7Y(Wo|pO8kjL3rO1fdJ z(6*M*Z!bIJrq7`c$??b>G2WG5y*#f*P1u*VKgq0G62ikYUZ<KC;UEx{I6eA?l{qbw zZel}5X{DQc9ga4Z1^)i&)n&bl1sOAnn^%XNvp8?L@kdnTn37z2y<X>&eE@Vzp_N>& zE6jUCnOgIeXB%v>MO-3JxJ~Q~zOF*x7!KeiXZ=LCx+JPkgm<%uSoZ~%K6k9JYf++@ z=+qg3k+Bv&iTLfc$(-&M-b-^VZ>1_FIc1(<5HwirMK@-LFn<q->Q&xYk5A{&YV(z{ zvjwg4{A86Y`R)Uw!f;UzXj#5@+kCJDVj9jq{#DD;>UPgar*b7<hc2<{ywBv$U<c?0 zxL*bLF0KdJ>=WeONBX|DJ+s=Ir`pjfuRir^qNOCqp);j+t0TUO)XCT8^f;igo~z9# z?Y!5;%M;dxjGo*AE47A8n${r};~LfyD=T$FC+tqdjyh+BJCvx0-_?s7HD5k9TcUkY zNO*RA^aL`!u#ZsxNV4w!d3fkBmVbOD_{D{9=V0O>gAz=~5oe<g-(y_io=BdL8<LTw zhk2Y&yxbkX0<Ah7Nq?*C33~DSk@q$A0~D6CK1}_^4vK&ELa#<yE$l>LF?Vw<(WmCn zsvh|~nffW3r-wDAZ<vN?49L*)iINn~50Pe`-9k=`p0JGi{L+~{L^Sz5^QX{(uYb+O z`0bu^f{+9Wsosx<&?$+@9<KV+JczDr`Kc6-`X!~?{o}Z8hkLZS8O6G~i>xW`>XjLs zWT}`*EJ3DD>CE&i)7{JU(d0T7GS%CGT^<G1P#M3cb_}4E(BpO0CTeRO?#mE_F2*4` zhew<~ATEUXFk|GSb#*KurF!*pxv~~{w_c1>yaN~=aYC*)!+nCBS(CsHjdPyct*vem zxMo_Q=vpm{itK6KVp(v;q?s$#DqqP2nm|dM36va&?b<f=aTw>t3M>AE9ZZ2v)+xpC zg;@#E--_m#aG`Q~-fXd;RmLt>_OZH5>W?d~4X<tC8$6Ff+v3G$OEzG&UlcRbFutM1 zFty+uKW!Gb5w=;Woa--Ecc;RRRTK6~$2U}&MxTSz6VCJ8b@@J<W~%NQ2I<d)wh3g! zpj{pzM!Gqt6tXFC$ATy8U!Q*F?sNEs`mIZCv-AfK%d^8BxDAbw!~%lMz3kq0-y@P? z9kmCzXeF}H@*2f888~f@D2t17z(^%u2{8;@B41-(P46C7=Z2xp+tzTVq3S(QajxXi zFD~o*10o#qKli<AUIPk9k)cE#E{r~F;hQv$?&r-5kO;Cs4m0j1AZBeXqe;>G2l}C5 zEAwxgw|l&Y3DS}LyuXnD0MGBC>DN!hY<a_%ig{WSZl?at3WU-By+|hG#P}CAF?`VQ z^toD(G`OGo%7YrTu&BraDx!r~jK&@I+tbg>C<Up3`YV1S<s|{<2lKVZGl^207?2?Y zN>wo|Jt31UdES$hgdusv(ar-YL-Lz*jdP<Tnw!<p4T@n4P71+I!(4F;E7+3_Xp3%i z_iDn2N=%{hOlt|HPwEFI*>{wqnD{V~$@Bsz>MHS4OvpVUyfYymut#vqdv2~Eh`u5} zS{&7_^!%FC+Hv*#;J5M7zIu$<LO8a`g8l4~1<B=o5n!=W2@Ly?$hElt=`1S;b2D^b ze%s3qwcL*nwEga>s&w;}C)P%?)9PM6y_}hB+ARtC^UEtJfdsH2Zh*jHyyMvex(^n) z64)kgzCada5QDU>C(fKPVxD1qS%u+3726cZt@7$;e7NfW^07W8`2ipI%pDcGC`x1c z%5~Lydb|Pyj2xRi7q5FEuCnrUob!DyyU?J{JL!;({x>Bb0uHVpyI$^p=av5ak_-;E z*Jn67`dyVDlbp*Z?-|A6V#i(mH(ugFt27r3SG8&{TNux8MUG1`A(IIRoIS%jmTL8v z>g;FH?J|n9z$DG<2@#C6)6k<AIPuf4O}yp&Fe9%1Uw3)_l|)QNcSG?2c-H|U3v$6J z{k~hP6aJ$Xh7%jiw6-Gsgn-m`8~41!vmo-$UL5!`H!S~QM1PjN!@dJI4f4xkC{GNO zHAasb*<yVeq)1@gP^YnWEk#1$A`7-~vf{=^5Up1Y3j(lmrCT3g20j~n;5@&|br4tG zWVjF@#fj4IT5zPmEV1N@&DENv9Mv>a$tctnZ|$MmT3p=I5Yv*Y)}x0IRng7W@x!K{ z<aeT<f$}%$?ULg1m7Mg$@YjWAI1TLj;@y<F#~S-Mpy}q!5m08z)XU&Et@5J_L>S{b z8l;j_1XoP(BWZI8o`c*f>7vifIz^dkoXjk3g&z!*2)pG1MeM%jtM6JwTjhqsifUIE z<kwuVxQsW+EGcwaB--AXoove^pJr+s<hA0gmT-TAlDY!}LidID(1mlC6hCa=*ON;W zk8Q=gacC)l!FN?VMQabAiaN@ESKM1s+=#f~e3Uy)q0?3Rv7GJPII7w~tj5!rsRV_h za>mKh9<|qQrn-v|aX2G)u(P_KL)q*So6dz~NiZFs>P&4+)NUz>V~Z1|hm$|raQZ1b z*I*uoV;h-MMb_`e6!lk{Tzb|jX)`g(mxTR}NIj$UWW{rz5|z^cGZteqAT-K^Tr~y$ z2&3Nc4*P?~Apw^qA6*Dy&U2G%{2;H(=aENRJ_s1TAR7&?osBzIUU~Kw`&^LJ)$@Lc z$wwK2Z<!|H8j!Qf+_+Dm3InD8q;v60wyC%^{&oeP;~1C<o88=u!h;t`rnO!x3@iG{ zODG1(E_xSdk5Z~&N?C~U;D$F_pW4V;cvS7BD||OLyyvx*Gyzz5#M2J3&I@TBcWU8e zK-v5?p+Z$(JDuAVEZbSFlhg3)*(YrB(@HZ+lghYy3^!D02HZ>;2YwhA;A8l%Y^^*M zu;Rn|)ySs5#AC|ZjtsMF9D3?{k-rf|q3KnQOrF*+$UEgDf*3&uh_H&2bQp2WU4z7` zxC4sA@`-sd7Fh___ve=m7bYw%EaA|}s0qQO$l=59iZiP#Wfe_ji=No?C`)Oqk4nJQ zGS%kTT7C!hA$mq`pTIIRwK>=<iJXzz5~oF8Ha9vN1`6lx@o;lkpOxlIpeX?i7zAON zr9HNi%4hS(3}6|96b|4>!@lbX>H`*zAme|)6ZscVsu?>OxHA8A0Og<4CUcz08@S&u z#*>udg@K}Q)e7k%t(yJV3A}&fv6}51@HHsH-lbQqk0ZmbPn&EfaBn6rDGodUK9Gsf z(Ge~G)AAty5hw{dYpN1BIfjIM+-?VQ4~KcobB45^?`c<%TrX|>QTF8Lx3k-BHBH<R zUQvi)^4<#|%u$vLu)sM5^*`SE7X~Fd&1EBWH|V*iG{MoBnwinfg^y8BQ`P+X_(t?% z+U2`puU^pg?Y^RPa2+Y?Z?(Ql)2HG@(;ecD3A8;8u=KIPE=#`Z_`07m4bQ^1evwHO zi;Z$1yUkS9w8_UWwbit(o5V;JZqKHcidcCR+T*$(0B12y;rO@C=rZ9&p>6#G6Je{- z)7>ZtonjiK6=<pL{QP|WO9ejLU6+lQzTVH@sb*lqu}Ey3vxJZ$^h-)}<sbEUtV+D6 z3AFK85V{(jg|&L*jtcMEUYpbMXNybpNEiC??CZ_=3({5PS#nzB&~1x-TItPACfQz3 zDX9|xq(eACWan`u3HC=y-zZr2gd-^JBhSsM-PkKX#`T%Gj%54O$33g~!EMvm+BmKq zjx1w}4{zdbHG-ZH(`|FeA+GO-!aC%eEG8agDQl%Ftlyn{)C+1To%Vqj=+<|eQ;=9G zve4x6!0~$^;jo4gQ+)xkENKl*N9xtjT`3py7Ob3jZWc+$sJ#kT9F&l>H+#^4)RdA- zY?6V1p#?$l%1A_VGxuV;l)N@mrL0@4E5BS2n~T*3zqx$HTa-IpGIQJ1ZAMLWru?iR z>EdUnsrpCO&c+`B>y5z~4P`1O_5mCZ#hHR&KB?t1M5r@x2#2St8}OhqNiHg>kdqq` zUG?99j$)pZl*)Zbv5!d4TxAAKy%RgPz8Xh)ukC^yTk7v%H_UOO+-ye(Ms*%T5*$q9 zcy+CT6?B$%Oj4F9$CJQ#F0w8(8y2@(o`KNa-z;)z*Fd6S8A8Mf*+m{AqSD8&Ne>7F zV)JJ0GppvqCxx-#)5dMCoRX)OKx{eg4loQl__4jU4>E|*_+zv=Uy$;8$XC+EnU)l6 z&_IPFYkVH4&$*7Q^12>4a4Uk4+_dhRWg>?1d8ks=*X9_nU`b>)B@JS?XBB6q*k`CU z1Q`1PjDt-)yAhN=6?B1frs|_0#^O9(QjBNd_?X|%8z-(l$vM71{QNlQ)SuIxVWVf7 zn6=HfCT^vU9Veso=)o|_hc;t*3on%-eimoUh!cZrtdB~cT&dQVcC-A^^-yMo=hS<% zZy&EQRo`$w`^w`edU+@DS+wC&AqK|H_3`+S`P(eQTD%0Ol9DHn__BYUqQ&Fe7Iz+9 zQ~I(or_86}ckLr_`F<}t_Qq1?g9k%APqLEwr%syOQWe;F|E${4Sh`gb1QM^`x~|>$ zBXrj)H~@)wEaiJ6EwLpE{Qmu|L(fbjay?JV28$$+Lofc;%5+Lsn(68p->@-y@0Hgd zBn}eR0|K}q*gIy-ki`)K1ae)I15yBTGa+A>(2#JQJSgOKX$A!H?e7oSakU>m#RT!N zKd}_5GiV5b5V?)snrVureBppVq&SMl)RSY|TMdmNkki8+&C^aNJP*b6o0X1y^rMPF zAwh(mLj2IBCnhL|6XnIj<i__Pz_$7A;>N_b$8#2l)<X>sDgQ;H@lP}*T`#_<ct|~Y Yy3;%?3>|)e1_A#F`eyh?IOoX!14y0yQ2+n{ literal 73267 zcmcF~g;QKl&@Jx1xI=J)1b25>EG!n>J-E9|aCf)G-Q9u*cL*Llc<@K^`@UE2A9z(; zb!&I$-reawJ=1+oPlU3fG%6Ay5(ESUs;rEJDg*>f6$AvdIs(-DCk^GlX5KH*PGYiZ z2nYyk+e$y*-x4@WYC3=ZYUb=_=x7RIZuiyJl*!51(bUw=$>OW?1$2)P1Oz#Rtc0kV zd)8@}TN?3_!}*JP_IXYtWYmTv8jK7El_+a?dI4QTQ9-heeEmnCX49vRsr#|Ju6mo% ztKEy)gnjET^);rv^jayV4D#yDltF_Ef#eY){!}7+t(Hdjyr3;I!G>j6Gc)smedhiL zL3fDL^WZtbQm<(*@G%o1DQW1)?xni`Y3$$26oGp+{LlaURv<@f=tSh-H~$9kw*2P% z@&Alw>O{Nw-&+N8oRDjv1^;`_`{={<{{LUM2DXv<kL;$~s)?$-`SH#a;@tn-+}rM| z_n}+iJeIO9quOy+4Ex1vP9DgR_Jc%4KSks2DkI=$JN5QH^1$Xt$xw*0MY5xRvotTs z+uq)8BMo|go?uolE5s>j)BQ!A{-#ccmqv8Zx5y9%NdHv-oVGxbB61^u%JbO5%90L# zmx0Q1uDL>5`+w6_Bzd2CjuR}1t~c1n_?$>-B-gvKlb@Nq9Nz^AxRCq)04<tmc*Wp| z*}SF<3}RhDj2=^6=e6T9ZsShue<n09`D<EUTKs9)mxJcx5wZ$FZ;pzEy6?SXt#CK) ziXRXRXlMoz`VRIPFHbarHJC=j83Mdx%Kka8G<+$5bb2KgjHSYx($m)S=W+zgoGo4r zpmO^-?X+r)Fp&fk0-OG)r3S_K@+vT#&5PpK-|0`&Z@c7;I4QE~B)j?foqp+n8Z;G8 zvw$uXX)mFH3h;I;Gp|;L4bM1N3jT&KD)Km<8gkKHPPEB2{1*f{p9kJOd}u=x4MR%b zvGrCn({X1Gt>H?!JAQf@(g;Nk1y2rFr*GtCmg%et79k#l^xO#zrr^Q&{`^dR@X~TK zF+}$dFT7sAeLe`}G^D7(sAE0slHp$P#c?*9n9j~>L&652mfx5kt(qR}_-d=M>S`q# zhAR=>PT<zf83A*N3ZtJkK0KB0eJjHX_P_j{_Zg!+@@ax*Vm#^fXGlmJNy9(;m~-gC zdkUC;DvCtk(0IJ7X0<n^<;!=<gDN7Hs9a(y(QmIjDZx>}E>e#sozB{(5+>aBQqmB9 z)jTUNWMKVf;{7Cs<Dby#=icgV*yAxExe_tS`XADrW2TXB-KP{K`hshOK|UjX$r8bN zNKknY&~im4%;~Xin`wSW9_ekQl_PV(O2X<dzk`OL9R{muMrb2Keez%WGY_iHGt)N| zveiC$8Buk1Evk`MRl!a@_e+9!K#CP1BXdrOKRBfQ4?11K7-DkKCk^;^g^}Mw(nIg^ zFJO>mgQ%XOu1>b_*WBmvHC&=y0FZ-H5x#f*KV!c#-<WBt`QZO>^1PZ*z6ukj6Ku`> zHfX0hnb&V(_az{<vpoz!bmT4{Y`^-QlX~Y+nu#Yw#;UVO!CW;pO7R>{2=3(F&*t#| ztVaQ6kjq_G_T3r?C`aQLY_?(-Ez8r_oxbo@;fA2K>V)k%Y@~{|jGe!^?`D4t+CZRg zmg8XF2_*meJ0SHy&&lv^{%L%vlBOUc2nPtGNaQGuBe7Tx$Sym?Qc;1Cfc(HQ_$_V} zgAKC1U^gs*hnmtUG`JKEdn4A37G92l5&)eK)(-zw5b!(Hbek9rz4x`ihBpgCMH|Xz zU1@I5kfz;&TvOSgM*H0{r8xioF|F6gdS(x+W8T=5Hpc{4xMsTF0n}@+t)K!qWeucY zDLbXXurkQ+?BoMF2~spuhp}A?*l>GD{#06TBO8^^maf%!P_UjqCRxf#Rk0CYenoYs z(TLNP@Cy*;h^+q;2|n!(HjJu8#+#NCWn>&Z2R0j&csvF4WJp>nr=S+;#7sZ4-r_*T zQ{bzPE&+`qv=DG0s6Z}{<imNFF#f~kHgl~g7;0`C%z!<iJNn(rZmi2x*A*@7acFYE z&8KGhkps+N3g!KCv3hB0kTl>O$yrtIx%jb%r+YcNm*+fWt#z{1xh9KKP8Z!Cdc8$p zJs3|#r!<Pl?vGvfCH-H0TAF*(Of_(1{bcrbr0-VX@2FUKS&Xyc%#IOXg3PV_Tx$}G z<W}sE*!;zaA<9HR6v`pTmq5lF80!3QYIK1yPnpa9&@K@X((u=vjU!o0*45fJcDyVN zwc~jW>*FiZI1_)yQjTN8JR4j!^UK);?jZaTB}XngI&M}uRknj`4^L4+jij}<QVg5f zT%PeF5G*i%orCxvoE9Xy+uN*&fqL}uV4r)S;0v#+WGN(ethu6LAWo$|ez%A1SrB?3 zM@b_@08Qb%)8W(}^`R*rwt8Ji2J39bD&>mNBM<}>*VlIY+#fN_3hY>0BGdM}aA?7& zrUYsBz1H($NTZW|()Y4D1rY7vRk8LsO8Q#H@}d^!8Ol|j6FWf8m($$#o?J=;?TsT= zI2rIzC$iCOtcMd{EuJCwB=g=a-q`zhiTk*xtaZMHo@_Ep8S^k)?0h};9dE*Y>SJ0` zxbH#1X-jjg9l>Z_MI;?0x!L!`u(Q5|FUqXIilLYw1PL&bSLbe#;;S3qBs;CF1hq(B z(y=&jS-WibXQ|X=bMzg*Qtay^C|$qxqj2<+->d8eA`as|%oI(@?F?gYku)P)R3U$k znpunJ?Xr2nkMdK!cN-oO4_zdB2$)l#pUc~Lx{x(!s6o8X4${24rt45AC<3c=+%qsU zMLLcARwCcQbPcoPrA)r(H24l{XPzLH1Pn%^Wk4#!*9ckOkbIf*N_e}dHnQ4YqnY5z zROqA)w|e>#96nWy^mk{PM)|i#;9mXK8##_S3j~M=2+tCwJYYji6Ub%^dCx`G#xmm4 z-~@rK$Ol%Nj3pq-a!UA#%je}y87hM0h1ym+FV1%yNQ<1lOK|d;Hb#d(&-VRIHK3j+ z*WcH#6`?%|6M=aV^4d06t+=CJnSotiR9TmwrpSdM8<6S7xYWa}Vd*cD+^=bK#6TaT zojN6(9TppK4(ay4%p@zyo#fYLR>&!7wh5*<*uY7p=cp5u2cS}{DogY<S{qCtiZxH6 zpgd`0k@q1m5aW>p4Go9MQx<q1Jd1sfo7o#cr}dKB(20dj+XIVb&!fAg1^!+2a^}+6 z50^?tc8}8PG(`c<<j{l2NF~=Atlxep_nP2`Mw~F!ayk(51GD%7+)aO_OR6$XcYiEz zH+HEhDAHDFG&8SXXEumL3Hiq5cZ|-;;alu%2MI=!sqe^m7{1HMf*W!@viuO(Uo6CD zOMXT2@Hb_$>BU3->(ydS8tpyIPvIpC7bj2C^LO7?0|H0GI9TMn2+ku~xkt7UwPz!- zimT>2p`#BysIqk);$*Y9h~vVwKwxz&(evd(MN|-;3qJr@H!W(ppDQLgBucDf^Boc+ zef8@+A`iXq@uz3ulLn{dC>H82$7>2VX&MAdxiMSP%Ab9KJ@bDN_HE=p<hNEbfkISi zb(?4G<KFIRt=!HzbvbZZ_r=EGvJCgc#=<n?4)?n{JJ&v*aaJ6r_aPb%UzP=qu<sV3 z_Cd=fik?I&fs(NELZv6lxe!?;rX+ei>vXxpRcEWmbHNPCP1;R7I*UQn8$yF$q{WO1 z9U}NG1Sxjz)ax^2TPyIUeY__TQRW`5qc^sppgt#F0>}rU30f-GSV;dG?AyrJ*n8py zJ*%jDJED65<k5y)CO5<gN()cMpR!V&f=@!HC;DNK!$QiD%7ns9hQh3eA(T7vlXrw- zKC=mM>0TK#o4_?6^0VAd41-?b&oOc!Bc61&@Hi4!J*tkFeq5J7QORyEsL5YPp!kx8 z-0h-Fx6A#UC5vO>x+>KySAgR#nuH*b!K5I&9E_Do3VwND3bA1=%@h<O+js}L8PvbT zFj%qB8L;FN5CA}ix7zFd6ki5a##2rA;gc7*aogbe0k~k(>!a0fCi^?`nw^_Dw!I*t zh9eu(UEs4r76v(##ldPw#bRY7m;tz7srL05gSDOVIC7$Zxxr5K&tnu@%a-orVvJ20 zCoNM2E(mgg7CjdZHW2l|^g*2E!egY?$5kfOS`L2;SFBfacEI`(>E(epMo)2O>t8OP z{WrdM(<O{Kv)k?F@zyPQaUt2d3qaD8A_+nhfl(<Vk?wZZSH(;YLPbQWu);nncdOKW zomq&pId@2s9TNPphbH+1fB@7qnUc-Uw42?_ml*1egS%Ahu`Q;}Hf|!Bl_q>5Of}Q- z?K#GF9D?>*;m%sW>XDfY`bQ$qcQ2N*|0^Sa<W{O%&izZC6ERY-47=0x)PK&=?l*KD zxTCvWZO>h-3pss6{c;f1E=taZ@(vsKO<EX+bIm2;uMW;q&B>P8)jx>eYWMG|z~+UG z8Nv3gYarx^{#_f~1=#)HCf~i0kw^K%Oc4`6#?DJv08lqq7&~4DMIpW#SG+!JRP;Pd z)=k@iPw`z93<7a*dde?&<z<=DtF^K9S~zBFe@x{^$cR2gxi*Koa$%$?>ijEw*kOf8 z-+v)S{A62%um<TgTsA}b#E)I1c}xPG7=cv4iust(YyV-}wxZt!nGzymu9h9|bL=5X zV0cym?fBvCrEVBh<7W>BurX(iRf(tx!dyGI$6uQe{M{sTWXmAng6AXOl5oEqC4%z7 zkXnh~?1TtynBnwSv>(ekFQ@f0bXQ6K6o^i1MT8TIk9TNVOD;`SXh05@#($n5@Ta(? z{Oja@1q$i3NqAhzH|vouni4U{QieePO4;EYU_ZJr!`7CH_dv!kjhYGAP>E><`Z932 zUojX9c*N_m12~~?u~Z?zR4Kgsm^`^l&;+n~`8<>3>oknM=S(n7AmRC=ON$MQpSnPO z5-nULBC}Yf97=5~BYklqYvt3o+=$Bvs%WG!XQbl9C~bvov@iKNJLD?&BL$Zaf({2t z?g}p^1(*5LtHHu{aq~R-yGt~LQ+g8ftyEufK(QlEXgmMli9#=8f^fuFM*)k&GgLtQ zo*cvDM@Cp&+q;e{$!&R*Jce<*>UP)=cC9M<#V{d7KnyBctAjWJ$|Y4~GP@BRueX0< zjVd@xL&DR_(qHzj>zl3FbF3)URl?IgdF}EB$#AMO7wLQ-3MPC}SCoLw$ga{KI)dk^ zHnUQ<R)-jaNlLFu@gJGovZ93A6BjZZYa~`0_7A67Rh#6w1;Fy%Ne0~%nL<ZG4F`{V z-&yj>z)DpZX*yA`$MbSEr(4#(SDRBb|6rad^8rrTU!h%e>lbmjC*5`uCTuxj70*y% zSkT*jW^rx@kMAMs!>@@O1>#t^q9o%sH@Et2|25ZnA@)5vTtzhr(dSGJbH9fxoeJ6* zwf(vv*AbN!jmD?QdM~vYHL6K`><|Wt1y;=7VE=g3GRA@&1VOEw_k5J##>A3!JP4LO z?GP6B#%!{jcu5(=O&aco($*!p^2PZn%Cl?_i=s?Ua21gTqgR4Le%M|E?tQ}<RsA}A zV^+oz^_q3W@;guf^`MR0OUk#KHVO*6Xxv`TSPq!_bMVJ{rA2LBMu%JKhWig&K|*16 zDv-P)2pi>jnh5D@J5?dyb)?x272XW$)0y~F0x<0CT!mNDQJo%`!6^Z0trUuknR1)4 z&~-4|_d9aP@N1}|caw*f-9o};k(B$h$_0Mi7qrpO(mOuJj(JpJ71|AD+D%Zf5)=jR zxjZGl#n%)sHv9?raWBgS0VhL0cNO8VgNN+1u<`Smm%DVs`x}*xuM;HYzbsG{amZ=q zU}z|zDkjD+pNVUJeH!<22l}}W?b!P^k81hoNx$#@ic0_Et0j$VHthJ%p)O)HFz|9P zE*JrPiXF&`-@j#?Q;HFdv$!^?m<v5>I&S66psyz=y%^`j-dcFQZs*U=I40iBEB;8# z*Y9FmOSUX1pLJezM<?#PYmX>GgvxmjejfcPE808U17Hn_feblis$YbfW_Q?%`6 zOUnWmS)Jm~*%Frta=7TN0@|Ek5gwM7VbaLJJg_nitIBa<{3p#M?r~2KNtT+^?d0nL z<b&(TbJrFh+IFWwA1ANps<VaCFe(Rz3@Q8B%ixjW#Oi_A=n;<L5B@nkq?8ihLUhec zS!#Z9GJ0A_eyw#S*If0HGrDc0Uhvu1#DbIv|00V=DW*}<W)`0PIyaC!1Lk)~w}VB# zdcI?9STXCM-HliT<5wTu=al<+Si21Perd1=4?-@_o6`_AHHm4WTfdlSTt(Y9<)Fx3 ztK;2@dmSQo6rc%lFL2nGPDi&+3#hzIm+lOhHc=Wl+3-cX>B1<hijXavFMv=#_d1Un zxSx1z+a5|E-e%Z{997(!%8s0*Dg}5lQ$zO7M0U8fyqwQnzIY`L;NW1=<~^Ezr`2)W z)yMST*mpjlJ<c+uEI3D4`iQ7;=bGd$AG~bnf!?z}t0Pnd0ImEf)dKFdVcs{{HeFnV zw~#p$?5T;uN!<1X0BR-$aYB$dLXiMaB@5RR@>9tA4x(mf&*XR|=C{O?TX*hL4*!D; zQ~|&H)&8mqD|opFXzC62W7p`mLL4;!J&mg3yaO(czx@0#B;yaeg=JD$O%f3$V#KZI zp_%)2HTOB{l^=1xD(~>0?`gqaa`}<jtx%sqNCnK*8})o0zI{|K3`l`&@i;z@WlzV= ztL9wlQZ_|9d6s!16u3|dWrW&`iG9Hv_vT?|!ou2ZyO&5js>z=FXpqX`(0mMpnZNeL z$w555mKm{f#u0Wzm>_scB>RoLu3d94C4rWsxMq^Nw;TJ1@h{f>`D4MLW(H@t;m=MS zT$O8KuoyVfZ8^DGTE1^gx(zAuJQZ{h^H@}YH!v-pX%si6nQt7fk2ORU(%?#lMDr5m zG$Is67G09+Gjq)SVcHDeRt0SS4c9?mOZl^gOe<=<G6d|SXG#|<rei1I6-A2cQgUyQ zSK{09LRfs4Sc&Pk@Lzs9hm`Cg7z(rt+X+vp($-`?jY*0OWmsjCH`Rxf7>IPZo;Us* z_A+nB-Lp;d1Nxe|UA~qanm>nh11PczS`@)5kf+3Txog;dp2i;YlE+wl>RDyVN3}BT zpH4kVU+km1I|}FIrG`{|H<3Lfd)&3oB{0j7Xb3CX;O|yrg>>4!<`!+6LUE4?k{n?Z zJVjR|z=VfzktH*UUkcmUr%tBow+gtOfGvDuhz^SqDZJiUDww}4rxF17{`XGKy2I8w zyHyK`%UVo3lk|GSK$AE%uW}%&ms%Fheq6rgR^L&limo6yp(2^M=Se#GeawY^Zf@%S zN>U&sw2|z_;0yo)kN;Q0!002anZT)~Bs7gbsjxxMjhgoeAemtwgHd@m0-h5y1Y+D1 zio=_&ySd5nNpRr<myua=u(p81m}W)JpJc^9t{<F5ksR9#xCJgja~1=Kxs~g_ZVn1g zMgH>I3xSGSG^?ZT5E7x;W3zUE?BZzgS!h8Wka_(#Fvn+rL{VOH*ZMKlk3SQGMae>d z@~WY4hf*7>ulCQh(diM=_q4fN#D3=y^{7`#Jb@fyk|F~a=FfsCGdp&;Ar!2T%!sH% zQ%%ol{4$i-knDo>HFxJ^pq7|W7|K-V1a05bFt9&co+7@xL5<h%xXL5KhueyJ7CnO7 z!#~QBN_;>;9D#j_p2oJv)ca&<U3CDp8iDgVqw60yzRR9nORQ{6)`(UQUL+K)zCqo1 zwQXTn@`tA|@LLmrO73{pR{)^zEAYB>EEucn8~LaOVqG5|I6yz2y(Jp)J<OP}tzcJZ z=z=`FbU(oT=Qk(SHnwwo+1}Y4152gf3D?Ky?pJX>&j$5|ex0}B8@@?$C0}f9l&Q-y zUy*>KD5eQYwj~?J`)@Bhr}a4o=Xmeb!c4X}+Ab{&LYgbu8jEfD>xuCKrgH<1B4Bm3 z@&$pym^>Hm=~v9ixSywz4QxP)?HYjdvviHbj!Eqg`{pzVfRBy%(Vf>vL=@00hLk*_ zy_KFQ00R~sJ7aGrhotEGu1#e`g`!M-*DO{A#9EcU#a6rMs8xgHQk-#4T0T87A3|fW z>Pq!6d$3wcT9e#nATpeC3uDXv&_qX0Fp)V}e)Px4S{Pof?^}o4;--hWW1dVvQVj6Q zatjv%LISS8xzjf7Z7JJxZf9q(E9594vUkArVLTUGxI-`x2F{eCtXWWHny3J5bZ9;+ zuxaeYp*YX#b2oYn@2f+YVnbzGdP`?fA((+iPy@-0u{#z^hUj}nG_=(@_By=qu!$1o zKV2@T_s&oZ|E2%uvf8Wk2O^21_e-F_CYnpAa3^p<EtLbWpIR>Sd`?9i-$?I&y!x~A zR>g#(l-l#5QWyFIqRXy=1l#g2)ZU(p)1%<}+wABmj0#9TK#^lk!b$XMG^+_UQ}`Nj zR5Z!S6xxjyeGCl5#XW`#y1xLvUWtNwdz<kB*drjQ<dKdZ?Pec3pW60b?$w~;FxG9N z!ysV#0y}qXCZqr=1C8brf-ZC^X*%QBQj`Vbcp+83>u^w5F|G^dG>)MV-shu#iFjM4 z{qO3~gz}~X*rh3~nm?t)t|ml+Y0+2RwruZX+mv@c=k8f9ww19Vu3ZUwHQ1vaD;jMU zRSzX=zCEAsq~@Sfmh=6p1U_MdpWuq`H7Oyg+goX)5Xl1RQ0BtO=zM+!L;{f7vL-AN zWvhCkB0r0NN_)HfbjpbPws&yeUCDxX<35G+Z`a8d3yxskD=3dGW<KVMYM&gRo<1dU zQX+126HI+R{LC@jrA>f{KRd?S0~;}?JWhmm==Zc%+~c8T_|k8Wc6J&b9yoD6qlvTP ze;2dT)7A65U#jzseD;&H2|o`H$lu+Nn?S6N#?R|wS%#PZ?@Rj703~V~u5)xL{Nff* z^P{<2G$MgxL#Zqzg3Rvfvkt2zhdJ`m`G`+PBGC9jWUPud0^IsVJ$r)1$c%jKj9y(L z9@%mv?r9BPPTzw;1&tjasCFj7o#<-3FtkxpDD65i1MWP8FYJg)SGF4cCgm*&!iGz3 z&z`+eS@1uKp=H$Hu%xV_c1__l@WBqvM}kC53;fou-6hrNprG()et1lOL*KMFZkYfl zU}PJ+h`DH0BWLtR!r1Rmf3?mp=s?nG-hg*uB=(TX5X_w@v+EA?@rX;leUho7prvYh z^FimT9JP^U>I403ooD*uOCzm8$hVYSg7Md3ZYklb<IQor$(8I#2uuPnALV*Sh3k`G z@}xN26m0E#E+P$;WMSC{;cn97^V^=rc^^ihAw6_hp(v^yies!}_O#}!knWoAe1Hd+ z#(;#qqIi;HpTN%>Kw#-&q6l_qo%Q_7_=kNeM9nYK(7c+$W#fudpFkkVPzHnk``q%b zcgCz?A4LtiYG`g2r6{Un)_Ff&6eriC^J>|irg0x;A~>-z&I)+)=NP*P6qSia{Z7og z%I3|^Y{HYs7iEzz9J4)X=4LCZfY5Z)_T6n9`aT{Xyk>pb1Z=hnK*iWLZgkK-MCB$C zK$JLw(<8>EMb)VWE=ks$STu=ywhK<ny9-8kc>({_zAOfqqI2v5aGw9sPQcIi7b>3A zmM_igcWCc2+_I=S97FT2+yw@xFB?(oUwy3ia`opr>{!zh)u<uWZ0Kj!E<XA?$h#~{ z4uf#!#&fAcv6xOw4{wOpz~rv0`OA5-p|>P6gN0%snZ962-I4Beu9&B*lnfXx$Au|0 z3$tNp5cT<V{-$uY;s=Vf1F=%ohdry$>A&|xpGJg<H`pYkDyk(#CRlaSyq<6nl)(7% z5H6$23ooPSBi)U~+Oyx{!EuUTKF~Ae%q)^TMV^z9f$^nX3g;JNvp(hymo5%+oN#y~ zn90eO<PNsv#@x5%C`<3)YMsox8)KB3eV3qOJ{lzlC>x+aoaorNaKQ1zdG?E~=HzcE zL(|~7ZZy*Q-bWi!%#mT$Q^0=vmOo#{OOlV{^QJpgk1T5~cV<EZiAt-@mogyitF~*J zq|KsdcdboXyBJaNy5HsaQg5M>ijMd$O$q-i_h~p*Mdn+&EuV#Li<jLg^tI<(%Emii z>esE8B*=V<9?PSfPrrfpr*K_hG(#K~x)x3^e9cddYtHm$`0-SgMVU7>2HzK~KvZj6 zIXn;W2uh43D#GBzct_j8%}56T$oIpaCXgqv0$!I$aIp+iW#LALL&tmAC2296Mux~R z?eS2Cn2<Y7*mW8mB@}$MUWSob{&EI{se|F=BBkV5tpO5F@8ui;N5qxW?8y%uMfqCi z!kzcm%W{rokV;${6t`iB?$&CAKrho(4nqMHvkEW_7_7wR)yFkmCX)A~Kub_um5^ZY znK%+#Q4{?=jbc_hgh;!Il*V(eM}hWicoSD~s|JK6t#~R1#|!lQ8zyWWBms3GPnk#v ztwPaJLP_cck<+~Z-T=7Br6W;R-Xk9gO>^Vn`q$!hKO<gP#nXgH3^cS=@Z21K5rDYr zcuxAP@F|h(G`xi*q|krRDykD+1%yY&AV*QKF9_UoJ4U^ZMNcIY-aFoJKZ{YvfdPoZ z8bQzcM<5<W^L71}ScueyRX_mYtQ|Ln|HP)Ce%i_TDyaLkKOTD@b@#4}k^SekA5?wP zfVW*3f2{myANj(tCM*@dsm_y-kfj>bw;kGKvWo&33i)+eC1xmzq{e!mk8e*~On#@F zx?Nhhm^r5G{2<^)5})S(_5!rbWHRN?4%zM6lmX~qe~P5fHI(^BDwovOc5r3<IxjG6 z1kzIn9!Cat*m<=2vNn)^eXJ)YlnZeX+M^RODz*x^!KSxDtDBo&DmCK7ov$1eb0`3n z)-6ahWm!-}4sTHbkv6>P2&y9a)?B+3<r>>ZbbC13-Fs1RTZQEfKg4nXl(ChwWF|R> zq1CW}T~2ArvN6%6R-B5B2lJ4HoLVc1N8<#}^PvWJ7_d6e>9nWs65T?1<XSkrVxyHy zQ3=2F#uSA{JxU-FVfJ1I2J6Q_sD(`z<FESX*fdT)W=I*lIKZ5A`+_lw?+RJzN1r~P z=hR*6H$8>`Ia?1HPFiQ^!vq6#XQCK!&6e(*zNR2iKynaTuZ(RctnpG#qqGH?CK+ic z8UUqu{54x)j!QZM;<$#FKkbn67g3omRTdV~ArufZGk@=}qw9Pv;L<L8z1jSzKFQuv zHPv}7<!$YC=HrIrmI{r_I0?>>jL_*crCWFZ!F=*bog?c9n*=Dm41ZjsMU8$Up{jAw zU!tf0P6C<IYhm_h26af(0X*ZAf_{o4BtR4fR4P4YcVA}1ck0_(c=v~bBm&y%11uzC zaM4v8?PW<6yKely=;r+oCd3Tm?BX$M4HS~0NED{f*py>oHZ+Z-GFc^%1tszi5nG)W z<d1_V1%6wakKYSNHyvYan+AM*Kygfh-gIx(7qPFm<6cgoW@}s}mwM&j(~BYyA>*OY zr20|5Tevv;O0^eKkvYh)qChBMeAZeZIwf$h!LK%rE;4i^?)K%}dYu(>E19G`RRcWC zwrpw4M}oqw8`e#X_51=JqSo36T%%vj*y`-<Fkz`)SLbG&zuDo)023j2vVLqbg@n&e zZKX84X!*QFT`zr%Q1q^|{E?PEqKmo9{#h+V43V9DaZJX~qJupFncf87Y7jPJ>g3Go zYKL}6mwy-9G+VXk6Ol@n2f^zIr=SK)_6`+KUnunGFtqEy>)y)tAB!Q9a*L*yP02ZK zYN%RkH4b)`Z?RlWpG={6w3A%k9v?S3uS>f`JQrZoqaK4=d_DF=R^m8N>#a@bMyJ+z zRdZ}QU?CJSS%vORZrdmmqt`Y`B@Nji-)=$<+-g`p9$Ps=vVE9Nx<F02-wECSv%u(t zei8K^n)~qggn<j5M2|FatCP|L*#mYP|0w$g;=k3xG*u<J!TP1dLl98|gsf>Le?kZ^ z>G!z=QoWUNMz6r3jn2l*)O)ycE@jCb4%W}%>?)mS_5C3E1mTVj@97->1vUvjZ9Bl) zm|G6&l91Zt#S*T+Y*mVR^rZkdR9c@%Wu*$9#ADDE*yA#RS#K>@zlBw2#Tf2djmMDt zWi*Hrb6b6!C9u%IwD0xa+N*;-tJPEKcSJ@Y&Bqs7p{Cn3mww|Im!C2{{Hk-yPz+2P zUJAA3C$Bd{*C>6X?g@Jn-@+7UFE1a$!m0A7f(-;|xOsiP?zFKgD3Fu89v1{gcGPdZ zyrQ<@RehhBkAVX&gAL$<uuO?u`lyi$j>kNjOgGGI{g#S@`Xcq_a!a4Df5N4cj=wWL z0SJ>~w!5r@Du3AVGSJsoaR}-a+qA4iGtMfJXmpx*I;|WK_8<gY24B~Pk4wpF6qi;* z5y0)n-^I$42iah(3${AqmnMYM4~oH>3+9_1KIeX$u*UM$S~8m0Ru%lxIvG%Wmcjlm zo)<Y(2c7wKt`XhDCQy%sC+t1WlBE8LkHs6#T;*c<l&w|{H}M*<J(rU+Vd(oS>e`?L zEX>4L6OB&0zENk6zs(C6{}Jg$AnFL^G;FOZ>Llw&1+i&tR5W;O35fl~?IvimNpGi> z{-3jbKkGvb84%{w=c{m)6lxV&2oc^c7dFV+Yi`PEvlf2*njby`lb7`V)aN#Q30$>3 zG&k>2X1&$O<CuWrNo=AyF0Et4TUtc_4U+FzLWk>GksY_TbiP;oBw<=Q?-^l(z&xS9 ztgp2`1VJ;nRTAFROu_|x1P4U=w_!&uc=~Kd(1}RMcL0@AvdXq;YMiWgd3|J4r$~g` zA6gKVjqq@(CKx_NleD_#cX$xIvsu*-y!C9=*GdtH>;T2>z|MX90D8T;TF)V|>p<#_ zRJ4q03)RkFt?9h%1LRl>Pj@sT{PC7)Sx4=iHIF%Xx4{aFmOm(`6aW*vlAz>w%2;=q zOs9eB&&@9F=MS0<Jj&{I&6W&M?7Y8sYQ6XS^|BRr=OW5>+V@p*2}l*S1w}yX%90~S z+1&Y`^*x=Hh!t}9Y-Fu$6R5+}(tORLABq{Tj_l$spAbuQkyd(<;I5w6H1>V(;j5SV z7)5akuKwiHM}@w}7(wjc9Nir2Xg1Ht(C|BT0XbAf+#uDbHa4}WKxs7TSQPV%Wo<1j zI(-3rofaz3p<@f#SV#j&V3fj4Zydnc^iLPdxt?IpRe)E;5S0nHOPI+a3?Vx5hObh+ zE31uF-S5r><|aw{QD^Ha_c^2coqgAr{&RVFphk&M+8{jy4W%m7B_egwY#E0QQUIjH zHy!uN_NxcJ98Uy!NL$k-SqKSFM@xl+i637}(JjN9Fpek3ccUIR`k5PAE8Hf%6EmFJ z39th2bJKrP9ZR|FQOmDSGFogHESVw)+I%=q<$)NRkAty|@>0sk9ypgFI2&+A8YA>M zdGz*+scOUy80nKBPji-I0OQ(bvwTr?;?qf1R|KIfvv$~tvI%p8?8K30&G*IHO@2JP zvk*imHGTz&3`H*%nc)B^-7WuIUIw{-9qMg9dC4CX9DdzI?NY?}&0&WXniuJU+>h)f zhd$H>fwT6}sGI?2^V453#p!PMKgF-T%$^d;NSOgQCMt$CKsUOWCh{<{&#CFutvS}$ z_47d_?9JkB+c++q1(PvKsTqp<+T(h^z{_?lLs$i-{F4zrM*L?ORj=yV+k7v1veqj8 zV;fjr<1!>z`HmY@ym26-=hDrkh_82imBJv0Bn-L;S00xG9EidKs0`%5o4MA8PPdw} z>CN(0uCfXV9|4QO(0V4w?>Z&l1jSU5GPl*)=>cZOJj^=92T&|~ANdi(D%x*dtP_b+ z@AY_u@$YleS=lc;SL;het48kE24ypiR0j4G8o`X+hL*8bYpsUXgtqdf{9@FUDr$g` zlIu73a!ClxdlP!HtXV0AluvQ!y)YX5myLk5JWxf%{DugTP&GvDOIees2Khj96VivD zCHpf5Ii;Dn@#jF4y`#?V-|_J%ZE{gzb&=;RLb+8Z@9;hPS7MrDl@r^GNlaL5DvRJi zX{x}beH<}PsdbsRA*kQl`s6z2c<sfFt=@>XQXCBfI%>XdAdC6Uzr{>e9`Xa%Lk*C# zpA^mHce`>bpIhB^*S~V&Y?DUyQAZe~=Pl(N<a4_#7v+~$RuwQt<w)@<Nc#pKAd+z0 zAZ~Y`_GNCGHU)i>eDKH}17A^xXoPfwlc5lFG<s#TcAbSYD27g21b;UI$snlJK(5qM z9kc0b={kzibjpr`*rDId;d>F)o^l^Etx(jImpwi4nF^KeE1wmB+Cm;y90q13xvRdD z+nxj-+C13#I1&tijh6$&7h*sVlyInd%{F8sLh>rhOri#1gU^FA{S65#Mq@B3T&lXv zAVr3{4Q;1K_lKkUSzV;5^70~Z;f|?mv@0hSrlsD6J-*$^Jb`y2wUVwkFGU^_1Ozmw zEkug-$);2Dsui<*O<Z~%`Jun{g_$cMQ!$AkT5+S**W<3R<|rfA;ivtWx|TnmitNt2 zK+LC|-qlDAC$gSctOiqJ%`;$9l9&W*V3>nb#FV89!nC+j;C)&FHr?FPzFL%MTatN6 z8u)U3FQV_PiV-^vU=wooJL19Qx!&cSlv!g^&X8HT%-vP?&)>r;bNZRTY(s!3^lb3J z>4x%_o0~KYMmsb2hA(1oU>Pq*a4rC|emk0ax0+yN_6IbZ)go)Y(lQ(tQiRp}B==W@ zG<Hbn$)G~ZxXvVtM4PE21K8wXnWm!`_S+srU{_fzO&SWx$MNn@m}||l^qB0rO&kJk z)7(|aoI3ZGR@YpYNiJ*e^u@&AR*aE37)B(hnfaFR%b`HJys?Zpst=!DHa=XQ2=hM| zW;=ZNP_Yl*i5!~Q>Z$MG$FTpN{Sjk<qzvxuKESFJ-f4`ZJ@PZPqOm{ik!nK$w|^D5 zz!F&kGK+2~Io2+G>vbXi>3&fPoPhgVxo~qfA~FZ;E;1nQx8Thj>~Gmsq82blOIq<V zwj@Qin)*$R8mN(Zd6}hG1|F45TT*<5;YD?{svpB#hHycZf(US4H&YLxGSi?Qnu+w? zQ@M|N(^_{NY<9LfvoC^xD3$3T&$^k&!n}(`QdkCG#YH3m-N>yU#V2gcY(_9RkYrEb zsz*L3H;wisFjDu*%HB9T6&=^QDGYLHsKTfo*o4PfKI_h|?Z!uvGVnXfG6qOVKro%{ zbf3yHZ_b{c%M}!@a?tVl(oKhzSH9C28^1mB-xYOw%Qy>9c?#G(a(^}Qs|nVg7sd}g zKEUlIy||;v3@)M5C6-vBKUe+y-TD>t!820qTyV5^bFm{|J9(Ur0=2p#vUe!?jc;~* z8y@KLfa(VGzvIXYI$XD>d`8D~jt;I1h*CqW4>AJ;rpK?`<9-yQu@upo|A`l}1fk!X z92pp`x~9O1`ZdnI*lT#0_HEYld2L`dlnCR0ceoh5lzFVD<;|Icr&QfZ)|GWL$!sRf z^#}JOWtHJnX{uo{A+?ne$%at#R)e{!@D=dhsG3m23iO{mFUj<h^;>?<w<9cu`5f0c zT(OGceBcZzYmRG1f((23M)V!hiQ}h@%!?MKmEOrQOi>eUEF8QXTioHnx$I{VFpu^O zZEWBI7iR+nRD62wg16?I<8y9)J!v^c(-aa2N4eGY=N^u>_Q{t>NZSWyiK4*smGO>y zcLAr+zq@uRq9W)7#4)FbY`IGo$U3+6b#O6%jBF!Ku!qGyvV>C<+-mD`fIhtbbWSZ} zH22)h^);Uo`+32qSgEsF!(rGWR!4?~U%12c+j2_g9WL}RU?{6DIlJ)YX!6g^=kKfA z)~Uf7EQl-wKdA-HwA_AV%Zd~3!4!-|3k*-yDLP^5^Wh5!COBgSaZ>V2Uyjo5C4MRF zB<gl`aKBCXJuBK!^xl=UEp_8xX|Z}8{l$Ga=Kwzt)sf6w>n3oB<i)p=M%9Uq+B2`H zBlK)e{CoyF*;d+ApGjUe+H%&>>$wMPfG4c+?e(YEzEW3TR-@>?B@||8Mfp;qSmqW7 z0?H0?$Gm~1B44;fXo?FFKx%IcNM?X<obWQ1GFFV-4l>LpJhvdMSQNGxxM>3>2aAa} zA8{%|z|Zt|j)x^;Rf*!utFecW(>EU%ix0|arU{Sk%r@3cpR25Rv(6NmawvmD1X!GC zre)rP|9|>HIGRf%AdRf-u;&OYZw*Z3x7HTAp7{6I^pm>GEa&H6sWzbf)*hed#EsvY zc{$2j(NZGr&*gl7+ng;fq$mFRBS6bKQ&quEwxV3+W@+;x1Eo5tP%T%j3RrJSJr}|* zACxbgeT3akjtXoH*-bd_<<O&dwF>7`KvQ^v>^@33UCJfIMWd(&{z;vEHG@eYTmYN) z#oqnMh8M4E0)<>X!XDIzY_Iz2u?uU?gY=8QieCjvQK1$xk_Rn4VH+hID9pug%<77W zG?>9}ay|x1&y%Hz(_ae`*oUcS1&}dMcYq(-db(?#U%#}Qzw($xaa?*;!l{UzcmVzV zohaZOkDwJ9HeHdz>Da3>zGgX9L0K)*x78^Lk1`&De7Ea~iR4N@Ep+^bM14zhE|i1x zm*D*ADSF{2p!eD`VqIIkK%xMtQkiBR{%d(s87I%^l;?Y9(K3U85)Ae6sN+tF?do83 zO3~J`-WprKl=Up-I&ct5d*d+kyVw6xbh1UNc_|`5nV<g2bZ5ILGl^Oq)PX-@Y)Qtt zcYEY6ydt`tK~5`D{uIxzsJcod1Z@`veai(FYd5q_pCrk)ls!g0FpF2B%%Q9wCr;Gc zSk8W-C4O3K7my~Jt>~VmV#9JGt}t?&@Y?yk<`u-uUGGZXnm4Vp{FRvmrCJkP;y`l% zV&Jnynlp*V#_X@*{gn<^Ey<4MuZ3b*MGXEF1<nG!F)9rWSUxYj$BT7M(Ay6_-rL^S zfs)&gA${&&(HUti?}hP-928JdU21zob|dvpP+ODzty86zn4=d_)83r)&n%cms%d~k zO^`O$xjwU9wmx<nKhqyicm!W(G%M#q(QM-J<gi&?OlL#k8-$1uAk=Tt^hijP7iBLb zIHTTtG(4`i_c*QDxIac^w3MZD_(J`DKb~RzrNYeyOpRu7j*TQz<=FBqdrKqj&xRUm z%z|2@E<-Rl&_ecJmap#|-&Nla{1_1(VJM<~8wrVtk~xky;To$RJHq~yRo?TrBlTLM zy<2l(L>?_!jv|z5ghn-01Z#O&xeK9ISNsN03$=LM?ReAE&3%`jIycE!2+&wAU(HI* zf7jmsoL#87re0O4Pt$|Re+>tXX7g9HRF3fm@B+M19VuRh7ee6ClI~?fn^vvlUwPCP zECo5NWq9Z@4JWqa2ev;9#M=<+&PtAZ1vOnCwNuXk$AdPrL<a&3UOY1q{hkY+PcHDw z%Vx|#Q6sS?fg&!q8ns#2kR+I%zm$LfuvgE)*WLumdrb@l^>NAv;?;yBVM3Qn@d2D9 z*FO0Q9O+3AW&#t9x143~lt*7C#UntCS3#IK_P_7ZQ{MH6#8Vj7d%6H+*?9&LIVKWy z8N_lyTRwo9N1El`pJNa2fveyA9qfjx3K_-vU<T2u9pPrlEx+bgPtuxm<f%+Nl!G6s z+>oD{Auz`fvyPP|(Ndh3!o57d3$dTx4S6^&aBT{QY8CVsUss&K4O~*>JH)GF#Wgn^ zsln=8UBsSb^8?P7M@ueu99t{D#PS%4Cp4ZLXB@O%fwK1{--U5@Lci?yi)<19WRIii zxj9RUM13-hSCs3VsLvHE>9TlX#^+2CrtP%ABNI+aq?A1i;_j0tW51XBbD~sfN}egs z#E43Rm;T>)%)r-Y+64}?A69NUI8tY62R*g)-KL;(ln72yY`h8d&8Pm~=)TG%4wxTQ z&NP*l^z-#124R|U>P`Lck|(=)+Wq{V_nC~5kvp5WI(=>L_7BbZJo=Ci;K{sAmoB`W zKB^GDY4VdLSZpkZ$*!cVfIb?!@h+2?pYvn*Iec~y?W@$s=zI+PWtts*_>tT|H3aao z&v+%g{^|Kgyug}j1&F_{%}=Sm#{$nKJH99W^JhawOKAbC4~jAoBS5ypoj{j@knpcs z$uD;?835{n0}`FtG)+|ji=d#ehI<lclc+X-N^u354rFZ=6(BJKE|y83*0NO@_0l4Q z`g?OScTMD@qshi433e5P+~7AZ#T<q&Qh`;)M_$O~Weq6EO-zyoT?-Lf8@3wv+3^iC zyO#>c|4dJFW7f+3S;I1w89}d`Fj+!ALWpy(y9wCSG)YpP;GZ|^LCdPbnQ>ZU_PQrM z)Z+!39)j+RXbOZJi0FQ~S{Ixiayvs_0cls~AQp@rlM4PMT_Nf*kOdfe5cb1h`aC@F zf;^|u<)wFIbSI*D)*4~*8L}840u!jRG*y@)35bweV%y1tG_)P`A5xjeUvDJSZN4D8 zgF1~|4&J(vE*{U9(>N(_ryI}pS1%)VaTK@nU}~w7s0#RK%$z?i3+=p~q|aXE%&xIa z_ju?YC|ut1;)LhPwPC#TwgR5D$J(@%;D$m<w=;#!PG%YFWiyYmQW?VG#xbNotIS3z zt39}BP;R{QPM0I-_X1Ng8QC%7&Zjjax}zb<Bzq$qjN<zJdLkB=;zknh<0$dweu;wP z)$=2K8a&*^dv)Kt0w{mLco86e9=c-V6GmNCLWQ&M{O<HT$IjMOj2sfDfAB+#-m#}r zGiS(qQW^AnscmG}GRCTFlU{duUyShkb+r7zn*OZIJ<ILOLe;}K96g{m10gA;0NB+I zQ2EeQD&HpDQ!1eD6q+f49!|$`!@sn52+x`C>IDL`s;=ek^|X9|<g?&*@Qh==d2c(q z8~=LlbL42!udGmHqTHfZo;x<RAd*pTJ$D6kI0f$|I2L?~IVRiCdO4a#e=~7gKR7xX zy8|7Xci*LWuxQy8u&SXYp_Cau@lHakiKx9df2qqRBrGtAKT8WU^xVa4KixMHO>2oi zKHX*%tN2M72CDC*s(qBcKd*zx;2X=jqa$pIDkoj~wfsYi@8(;gEAZD;##U?lkFR4X zfn5C3mOI4&9NF8P4^32vvSL_?R7G4a8ggWdLbcNhXriRa`aKN7nyiX9gAYGd3$x2z zWhE96OWsOdO2dc8+4@v!KYsDOCEVide?Rp0PiN!^quc2DIvYcdcR8(>@HE|W<MV6( zfe}6gawPtU9KX~bz}f7)^!l^roRPMAbTtBT@rYBtFt?rD>So3IM4$4Gc6#A1roFdu zT$UeIYm><P*`<rjQ|tUVWECLFlT@20<221$#f1SFh*-7Q{(k)P^kCp|ktO<NZ5XM9 zfliy3<<H=Wm$3Vn12F7*c0k%UWJja{g;1g<1(IB&3|8=?J5@j{Q2G~M58t~GheK=e z`OO12P7mYo-oY)@Ffb)B0|G5q$SNSwN>AvJ4iZ5tT{SVAE4Gr1baG>MfR%S9GbPN= z-ly51(!RuLJX!5T@p%qWf)!<()yQMy{ZwJ)Xe0+B926^u9WMtw3=B0OWZTQ`SbO#i zX!L%z@-Hh*IxSlesi7bDa+Ue?`g`~}lDP@{lM1b6L4j4mY$<8W*0Q?%b$N`Jh&I>t z7mY*Qy(Az)qe7A+l-zrl1}FNNym(-#tBg}$1<QBu?da>j#m;oqzPxARIRX~0&in`B zqI{Ksxn}g4;VlUREE=DSjmsVZ?VKwYwOdZ>dVO!Fi>>t(zg@!iBJR;Jqo6>Ejebb> z5J)9!0nh#|Du>hGn`FsvXLW4hQ-iUUPL&p4L%kX-H%v3zKhe|jsmtH9P1Uj|M7G<+ z(ZCjwm}?7Q)v7A-=7^&t(!B@teZUu_@#~23iOdzu=nEfK3X|}DNx`V=$4$I_-!pKh z)%cCefF<yzl6m5Im&1hqTfwjFrQeKGxxSbA@A1ffdHYZ4mu0+xc>Joduiwh{=Z8=7 z#Tkm$S&|AVh!p_eCoF`e-$13aYCMZ;G9qZB(&Di@w9?~OBxTz`6-|NXB3R$^3`S<_ zOIQB_JMZPV-pt^%2A+@sUaSxZw+pWbndI^w=P(jZ-cgpz=)|BDcY#Y8vu1_N16I+W zdD@D$I_)L-8*stnj_ockcZ&-vg{E!h8fSITc26qzH*ss9(KzO+WI>5?!}BE8C4!_F z<-wwVT4GyWkQ1y^awcuZ)F7n3hW=otSE8s+GZ+c4)HgWzF1Ej?f@w4bH>jZ8=#UMX z9quxWiA_LMyy3(<m^wigzINYtQsSyRvN3*Ixl4|afu;meu_hdi(SG1&sd{a}+pE!D zAEfo53`LMz^<1;wSyqIFf|RqQ<Wv6qLDbHa;-QN1&mr?9cNfv$-Zrd%C>u>YKpIwO zAqaoH!`bc;(0S|#Bl)IczE*QxnJB{eWu#rWE&rxEvpw^e8}xj&pFS<5C8)}2(bb9_ zpe}vBNzuVFU6p~C<#6z%00Y*zxH0=x$~^HrACkU&Tp~WYSXh51<1lCF@r;qGz><|* zgcV0EGCR>2vo#c94CuPZxk}w6Hu+}h6gfLRAAw}yo1@>tb>H4proG|)-Hz9&g`~<x z&?r|xqEG<ch`Jh|N+B(UTn!_OBxV48lmD)6>3)BJ{TMnE&C;e4gKfUaRlo>TKIoW7 zcY-&+#+oHs1le0j5p?;A4@4IMZ6{@Yc2YgNFvA^4;^}f`Z8hn|U|+v<!-s+`*-qiu zq*guA%LE7&8G}b7_*m0_!14mYhi)TFOch=39`}sQ-yv!Xs<^<aet9+`By_B>t%iR* zu=0(;<}pqXY*;2q7Fw)#v)8AYETh%>_>sqL2D^Kg?yCQqc}nH5;{cOkH!&w2DbA@| z*M(rmyq#5nko0UTi}Bo*43D#ryLw4rpqK<c9n9BGy2sKH6772(JV}Uqh2r1ym>pJ| z>7GjL3MJdBbnk=^Bd)Z@N4PjaO-Cj4HuJE=X!z4m=tfhz0Ymi!B>B~;cnQACZF7|s zAu8cjLw%2c^w}>iJMP#?sK3UB-s?Sah_LZ`<7HbiIWCgM*z-Bs82m|Rc^%4B{F26| zpGha5OA&M)HpSaU?r6C`3P)Z4KoAkK(NIvoXckdZ^l!VK6_8|uOq)CIJz$UTEyCg_ zsmf>BiXa=ij$#lKC-Z9beu}N&cqn4zAU9g1`c*=%c8H+rHXth~23Z8#DFHzro(4(= z4^(*7wJR|beq%`|%Mupua3z=7l5Z?VkZ5Ozx-kh~R`xnHC|1ZrWuLap6@`Mq{^ZHl zo|ji~xWG(`!b1A1+gTLh=b$?7J7soyprFg{@qUOa(&6x(P#ZTFf9<-!)?Hgr(1tOU zOptheh#%YPVz6jOua0cdO?le`3^X-Im<%e=xLbTZTfAm7zu^l-fW&O=wpVEQj*C4J z7}#b=u<2e$N;mpiux8)8`9!?YlgdycpFkMD&!HzL)Af|u)tta*?U=iz<FyY<FcT9Q zcf8wIcZpvTAb`I2a7>-wC4>I_cDmt~WuoE2^0>A76f8$l;#EK+$6iPPt1Ll>8Qf<; zVI0)=hq1Zze|rHCjfIW%ZEW<3d-1(VAoXod!>&iB{hf9)tcf9@vBdI-mBl=y5}nS^ zkC6_ux|TfsvcSj3YJw_>i}!PclQ(MF4NAW<x~4Y0st7N@k0W>O@NTm~{6H-1A=BS0 zgPyG`=1U$l#z#>^2`4F`>?M;t@O#`YLWlOvx$u~9bjBP$IS<Lx&Baep%Jfuo{JxM7 z<>vJ@2M5WD*Cs!`pU-9lS8@e(J|Om11Cj{?gZJ+{6*j#c@^da_<}NB8;%WG$cH-Pe z@i8J$l8up=#sUzlP;f0oM^g$tpg-M?a4a*o#|g{UaX1;J#XL8HD4{xduSgTMIA#3q zbfFzPWi?>2@q?-Mm(hmjW#0x1MDtV+Nur^XRcWP5g)cYo@QCMFHdSbO>}y6|Dan_f zuTp-eL%&{yuzM3@8d~PIxwzb15QXrhR|O4zOjtbMq6o|Eto3F#Y5x#NJr*mm8nXI9 ze*Ed|%J(S}xqz%QX#~r6JFds8qs#bna|Nkb+Tir39EnDs`PfpOh#02Xv^v3n;^G%q z%bqSf`b`fH!;kys=&c<squF0Jay)xaW!+;OSW^WWM~d_!4=sr4P&DV+{|{|n*%epQ zbd5WK!3pjV2<|R}ySuwXaF-A~xVsZVaCdhN?gV!mf(-gju4mmp;r%k}teH7|PIp&V zb?w?!eUeC-JQ^AKU}NF)A+>=K2@b#}ALL*r4<=uYT{h@7S$6qSn-|eh?sjj#)hiPC zLwH#7D}q;m6k$>9>q9k*v7q2xa)~Bc(JTA<t+<cVqI^!1$>;)}Dh;Jm+_ctQ?YCO5 zoMp7!MrFR4s&AoPv@wjO)2i01Jv?$1^Ws2$Sus!z>A@y+r$@lc4iP1Cbe}<{KHOla zNI(hqI%;g8m%+!Fi>8v%7#-J{?{Lp&)+fK(jUr9Z5B8AfjKY`kvrAw5&X{9^_B2y# zy6hjxw$<9L*qTDW>sF%!qP)zJoq6}6|5j@SFkJwk0@yxZG&r}?zm3{Ycy-$!J8v0u z`qqV@`&DsKVp_}tNR?;$D$^#cN0-<`>KcSF$I3s}%Z!8dr-jil3t3>$rUH!#O-1s? zy7~78e+K0JVHl5SB3sE63g|V}3nic<Kua=1!h0N>VT;7nL2+b^xhkRc2w**Qt}D}) z%U}SJB^MUu;>*rxutQle^)0DdC3dPDx%!Wlc+T-XVM;M@@%)n<Hxp<8A>}+Xlhc}J zzm)=dk{3(h-{{@-<wy=m`@`dw#L;D*@^vXILuKz96&JARDZ_DQZ}6ZL#+6*$VdS(C z7)gRTU*8@rnqsq)@~OpZ@;T3g!Xa2!O)(Pc+%EH+;124Meh!P&(C<fw5Re|>@0#X~ zNYK@vb><JeOb89>Mfd=M4v%~Jpyr0~j02P0M7sGLWWYhsAOi1t(zinpH_Jtf%c?^S z003)SbiDx`flZ&XqF3MrQ)fa7-mGKb+Z6|`rlcjv>^cg^EdFJVSb43k#u(oFsErME zi=v}aZ|1An5)mm%x}ALZ%vPm*X#QNZ=UlFKnp5Ojp!PG}KT3~l=jemQ_FArRBS(O@ z_R~t9!$hPs<LlEx3q%V>7lQMbQ<v^brw5k`q%}DKo~-XjnYyUC80nK$_Ece(5<=XR zE@S3?2J9n^Ki#CgTGR7ejJqnF=`Au$hLM!RD0T_B5OYGo5t|pLQWPa*J(NZtc{t#@ z9^9+r5D%#Y)`z`+F<d(}pYu_o74@oE+*0mZZ?BHlyqx;ro%?o9VyR*}le$h%+)RnP zGXd$t=oWm~F!3BnEU|IQX4qN7vo!o&tgRy9b_w$Qrh<7HKw3=ID?<UpaKDMmhvAF^ z*@U|5@#Gj56_*h|m0G1DhIlzEzt1$B2(8b3N;RQEl3y~=lG$=seYuz4yF(S9$zYd{ zMq<+8Guiesypec#q(%n5oEH`(e0SQ;>AvERI$!wxOE4aMCqK}?H-%%>oCRWnvW~wj zD@*6)y&`o#qVqS+OvA_+K(x*~lC%h;xzmu>M{QKf^%vdi7wd2eY!!%6$eQAHR{+Aq z(jHj_Ej7TUk#6Ra+F^Aj>w<hS2`Nxw{|<#UcnBsy{USx7jL~4S8pEBW%vt0io8~6= zOlLw8n)rQ$Xa)eFpppEz5n7+$bNY$&tB^{OZ_eU=_3x&il|_?CAbo3rnX?hVlP3W_ z?wekPxELJ&j#?lerCFaz->;9g52UdPcM5H=g7u7ibNg?jHfH#x3sJK-qNNLyAoA-P z(wg^$p!oL(Zh^cn8Q-O+229<kcMqpsUlvh+`}OF(4=)X75z37ah2DOd_<5X0lgZ&4 z4-*U(wwdSp-djD{cpK-r|C)0E+cy!B!($pBh#_{*`BiA|=$v|h&qs={-=SO&DB3?+ zVO^lzb(KuRZb1&_Y}Cz$vp6y%poc-g8Zll?;l#wY>iQf7rGhi7lIW7O8=bDRby$gq zXgXs@%~UJAxkbO#VIB_Uz)hLtGj?0Q(bfjOH70pqMjsKW;jEd<vpCm<W1m-s7PHV+ z+<Hi@i{CJOZ8^%qxKJ<_JeR^^3ma9lVPLH~2m-pqF32Aqz(t@7p$4NOqL+F&X*1*y zKD&N+KfP(^8ZDa{QIV@CF53%{tH>nmw<dM=Ki{GigZXGpZtrD@TuaSOw9bzAx=Yd| z@}|`3Ybn@8O#&EC1lKXjW1Lr$g1(U=i^%%t4y^Ogwfu?}q^jK{q5w>?WSynsk_r<O z$-XX3e#q5IY&u+jetFuN&q;l2#01zMi;}QvtteheVTonLis2CC>~^MrzmY1IWTE(p ztmlDldi&c_%Zmhh&bCEXy-kcwXmStori^E5O&JnOU-6PrQt~a+9@fE@>O|RlmXt3n z$~4<ev_@Obei+Dd9U8JWb><7Ni`Y{$J%4@@!)*v&MhJl&l~ZaOvYXZlalNPudp1?} z>7?*dZH<0^7$#uO+9}f2DIBtFC^5s=p4_cx7_l0cf&)AFuw9rA6es3KFybM^pnf;J zoBhJ=7e+TdAtJQcX{qTymBe~GcXpGZCO4(D2azugAE4LF{onV10y}!B+Y~OZBrngd zFj^AVZ}Hm#-DbVj#7Ll@2D$0F64Mi0^8L%zNN<jN!~&({zE(k}R;T#&y&ny@x5KNm zrnU<4ko?}yLvHKjQy$96)-umO-d=Mki{c<eQpS8iDv+l5nmnJ^JBR&TZ~dXKwJh6J z%f?Tz);~NoaYlb}ykg~T%fLB{_h3P(?;U^p=lhVktPuR20T&|WN-fsSobZ4fO^l#< z6cV88i)^i<!;%Kus5N2h@Y2n$^M>YYG0z>wU^pR=-IBXCa=(G)b79tXIjMc1rCYNr zoC%rQp^^z&ED{VF?vzPi7I}$oH5zIpwTu+8MbN;dGE*!K5lliOUh-M|4+uNdVw?>x zZ7Mp%#CC}3j6?qUA{u(#caGha*G>9scyH(get6~4s!%m>%qBiNj#nTFuE$^c?qPe@ zYbq&0H_}QLgDd?-qlMegbIVVzVy_IM+U-8V1dI5!ZWXAJD~k9@!Hi&KT1a%g#}H+S zco|yyI&TwT{>CuUdU{&=O=XLe*%mKDe!tqC6=Y}ZxtL@lCz$!5q=mkw`PykpU7rNb z#7@OR7V#?3=J{)2iN4uF^*p;gY(JeaDX1fk0E10rq~eh7D`@%J-8+}7pGDT@=P%eq zEsTL><@5P-MUE~bP9%LJFsxcN&74<N`D&)So3G1FAM`iyo>hwYLHIXXtHWY&9WZWY ziI=sf+t~p7^~_E7>NFV5oaTAh^JOI{uJio~RH-#MZ=B=ocozBma5VqEHoM_JsIJqv zf!P;z6!L|su+;KZ>i#a8UNS+%gVK{aE(27M2#<U$YKBpQH{4Hf?vb<@18o-b#ij@f zmd%itrPU&^)@jA4yZ1wS+|La%)S_S~1H7%4hBb32!r<W8$Ir$0<vpDD#Vjl@gQ?%t z+;ec@%U?_`E84|v4soY{k;#@WF#NNkc1eN=em2&iU4XRs21ig~q_@@m>er49jHTBX z+f5RQeVuB<k4D|ITOI4c)>{M;e;Qytd`;3j>}2;X=W--Nq1=B9Lb$0bEA(n^7Q?OL zy_kYtnA6Hxy1r6?#9U30JUL?-Q_<JDy5t5$%(Ajb^6?&t3!nfsCct~RJ8PSpcIFZ? zqMv>2$asXTP+_1k$(Pd1na(iQ1Ym-zsDxb&kZcMtfp~@Rz(YIbGacF{>DrpZmiX(v z=goliB)WU~)2Kn@DXlRWATWpd<V++@-P^8C)Zlie+Y(<pNl*?maa4`%eA-7o7&8%G zCjohp!`RX6Ib+*6xh&oHlLHkN_x@2_^B7!Cx1A*Xh5l;_=^qA<J&;YSOhi^1xsV#1 zpB0qyImXuQ&`oLT6Jt6&{!{}j8!n6;FW+2(Bh{O}UxUz|nxc+_#VFBsLUM{6mq-uK z=a{#%Z+5v%A%2{=7bh5-GyRcJ(?}nTBI)xHHvX24SuGGHNTMr~j=tS~v+7IYI-CO^ zF2(He7~-&;f=8)>F0Y&Aqw@pG+N!k3myhcdRG>mdX3^fR3!k=yS35mruOS<;EQ*ez z?McRuS@&%eqC&}b{Y_|#VHiC7CdnAF6NTZ#KiUGggl7{5DVj>jbg#DG$CLQv$z<qD z<Gu|;EP*TmYi~1~6K`+FmMsQo$j@bLNp(v*8BC*KOp`H_=(pa!3)?Zw!Om&!izR0A zr?~s*12>yv6b|6P-b4$7aJ3H#E%PLxUy$rjCpuTN#qJvI)y19{Jowtg;#9Z9lPMWI zTVSFZ^*JQ!2WG|!%4nmRp_oaj34^(<Z)hYwp>kVR!3ni_WN<7rA(H7mIc9LMNoYvM zGCCq9YgsDF@oS4VnE;gSfl>*tsDnyeBsow6P0>Qn)o!aW78d2RC20kIqy^I+;YWse zYZoTN3+OOE3tAdVidzu8hF~(N!xwrbz<hoGy2^t<eQgyNPwj{Yp&yE9x<tDvrMik> zoovqof7pfvOTZ5o=BaRESHw^JPj<#VoGFUgGERqWEVocHv~8t@6r;7B(N-K3Th4f+ zTIn$6k*;oBfPw%kb?ExiP|)+mT5$x*sNG2WH%|+iuq8beD9mt|^2YB-a0}N`f~0E$ z7#9^me=l#m4D=qy7-K3UU?w?9mEmS|bhulpzQ3RU#$oMJE<lpW3I0HLcwhBanOzBX z3!5#nVL(x-wec=dg8HajZuuvRcM(kXy|x%B`$I>2^0>hwV~W~QFh^r$NeaJX7!)-j zJ3_CU`vIuuZZ)x;(B79kL5rM^;_9puf-w4FWObpV^0-ZV;e}jxXT{PPL`|hAslBah z!kQ+;@<C5&h=_ww=Ec{gEEoUO5rgFE?|3z^kzyLZz*xm9sCeYEeucIp%Yy>9+_o<z z4;Pbze`r*~@*h=8hOHR#i*O=j?y=eo5hL1t7?tjbM2eCSCzxPu6EgJb18RGjtz&-| zI=#mPI=J7W@NW@1mIF!AB&-PA@}~aclWnfwitgT377e9VJiO?hUZmp+-(J`a?@LDh zkfv7%j?90$F1p<}xyQCaaxkM(M3^FUoTfL%c|ff>_j@NXf98l;ZD^aW%gyW_!p1;@ zo0ChT4#xMt_wlzKIIh{yd3l?NE-XE{{Rn~)6bVoSS7y)QkBEc1?CjN!6zU|7mf+cy z31QXU>gV=pC;qBCBJY0e7*U-m`4V2WrYV<;!Vho2z(t4*#SVlSJ^!{&*Yz#d)+@kj zT$M6Z+{UPmx<D5trpf3qsY~-=H*%oygO;$Y?NEX4oV96j#Sjf`)1U*<NoLv}=a|4? z#fXfY$@XQ?L0x5|S@}|W{@<aCnkeTl!s?Eo`=bmM8SY5xfb}O#Q4@V#?A&x(Y@`c8 zY`PjDAv#_Z6Qsf+FC&6n+rAklkGHtK-x0JH4nfC3Pp4SZsZ1mjPv`N5q>n$$WLuBC zk5%)EE*&xZKJFH{Z;A)))vGwjNIw9$0+xZTK9Y}1cWx<-e0iU}UK7_u_WAaR7BvDd zF3+(aG_iSZ<0B(A%${ZyOI6h78$L+e>)k7?XS`4`U69T2O{YTbzBUio?#@K{sWNck z|MNYQ_mjh(C<mckaQKdqg6qDrX`038)~g&6`7DBJrPJl1Ts#Zns-y*3o5)SKqAg!J zdmKzcNvF*EmIRXoFXfENQJ`hIh=1{61?8;1`a0zL-hq%X`xavcvb6?}PX$OFYR!X{ z199u~>M-F4nUDSU*n2!5W3$t~?BXBNUv<u6WnTv@r!o`sCx-)-W4ZnoYo}=AW}BhX zwd91*unuG&S5<;pwl?0^Gy$uRyq<|Zb>ULkZ_cDOBAx3q-fiu2@M`56e_S)_ZSmNt zq34uE6oc-(zgs_LjEfL<?-lj4bk^@H<kkH+VpiHZ%0c?o_XEeiB8^7lw6owptJyhI zXO(vmgYLlU)$S%i%UnbFLHmQkbgJ;w6eFbs3_FLX1JFb^tGUrUmaXbNrJRX`83C3p z@uDd&Hu65sZMiUq0rYjclUPZVMH2VHNqjiv*Sm2|wfj>p_#I<$=tg$D$93Tspza&$ z_+?iOJ^ld48G9Ee5)xW{0qE5pJW@lS=NiwX6dY+VYc1?9L5#@C&C7sFiX6!VF3@0D zF>YgqM1UA2BpX`IzBi5NIVDPp%R`(8h%0d_efbbimb1eaMp290(4P>4=&y6%JW7j0 zA#se8*b*Mc!fO#?4IF*jUH3L~=GIR+(hWSsJ;-kx_r?B0W58<%g`IjIZ%_PV!DlDd z53@3bzbiJxy3Lzchr{CGyl4wn5@>=an_XiB_&ZeQrFl#f;Q5*maEJ+fi51JDNg@|s zurVp?nQrKsVI-hjcl&{im>tUDoMzblRC%)Fz?bmTntOz&{KDpb(E|p#pOp#pTYo(L zbH-C<Q0+9+yjy6Vc-kM639Z1p)V@;G(1sSXE!PRgAwK=5#Fz$*?G3NyqwvhoNWlp& z1C5t9z<U&84voa_QC5TC^AZSyuy7F1S#MC}A-bC=?HgEd1Q&Mc`CLaikK!r*=wr`w z?YcHb73ZJwLoIg94-@Hk<yZc?zO5$@vA1to=M|uC{)-xH=~p<w75PTyN5{za@B`n^ zQIp7jx`>qfMS@!I(_7Fdb>O!?V)Y?9(_;+ZZTD<j)<SpTC_50C@vaN|Ds3jqRjk?& z|47M-4xXKRu3-_nxhgiNJywNZ$8^<c^D*5cNypK(m=<1zln5xt5P;<{*OrAlZx}$I z2{rNEjzsExp>$osJOQ-M_GOJIoIENe$k^|!+&FRvE$`40*~b=>&hsh@(w|M|Q8cp> zmyvaq`k$@6d~Cx10s}|<<9UI^Tq%X`LFtbz&8;5ad6d734skgdggS^`iF%}rxe}nr zM?Zy;gt1|=<tb}BOi=J_C|!IuE%?^pqi}Wpx+@s&GqZ@Bwnj&0!wQek?e-#QO*&bf zLW`F084Yo4#!%Y1t(8-w-UrhL?GxaxP|G~CtNR2^@iqR4=>5#7nua;dE15PQRdlI$ zb?z?a(aV`ohsy>6a@}cUDP@t~#g1UD3QTtlTRgO$6P9;ru4$I4;`i$S%KQmQpZ#I{ zpY!qyebCsuZ@%aL4wrbV|Hkn!2#NvcRm!vI69k;+oi-HH$eyNPi0AzbP9R2I^|o#b zl;f^R;UD;K%*P3^`fy|Vw|x~4zIrWnejlV6Ryk7TQO!HT+`?Oa;1L0>y`2yWTmJ<~ z7W!z7J`+AWqEXyOeDp8r8z{|G4`-$av^}Mam0oD6=H1q3!Hf`8@!@5tcpv69nk&<q z$m%67A_Fs?M|3euNO8ijPc%XI=0)@T2d(8{Q^`cDsZ9;3y#2vwuqI>#J`*3Gz}g?G ziP1_JksKV8ZGyh4Q?Q31VOuocyI=YH`jzo8C_A&23Q?OyZo9I*PBy_#ATqgp3>C!- zK^AP$u2)iWo=4VFX2qIuyKhYD!fIlUU=7~UdCO-!aXfoU|1fDFXfo%i=Ve9S->{MC z1j^-;Fq|rO<OuE*{EAF0VRQe@R+=mBb4X!+S97P`Y~bI^wKpsJNlfLc7-25%EZ!p+ ziru9mTtXx;T}{1+%j#@@^jP+C=6BuN`o`^UR;)<*Q3a-23sd$b*UCf12Pe|?<kxY- z@6_1xIf{WA6y&I`I8`TJJ+D9Keu^cJR2CM7z@Rr8an<3EB=Nh(#{T}h!gbpG{pMb^ z$8tfWZsg(@7NE}ML3C$-9+rZt00K8O426*JFrmJIt6q(L*q-O5&!=4hm}sic^Th!; z9PcKcxm!`mzXLVT4&f4OBV=q`4vMaGS(Kf1HBUFGOBdS2{yFSuAg6(KW(GLCp4X{0 zQ|FaoYEnulHb7+TV<mn`zQ4qKG_+qz;qOx)ErrRg#1k}R1$P5tKbQ|oV436yc`G|^ zAwV!fOgNH*xhc<YWk-BDJfuwWo9@Ihw$FlR@it7RfXuoW`$*pF15dLFY(gsoo#thX z?^i4tU%5sKCS$O9DcGe(`w4oXHZChXjy$iTA87RwSmN2Mxa;uq!>L_+QKLb_T;E!Z z)pR%(o^tZ%s%FglHDT`;C%#@32bPQ((VLHTdb>B#u|)Ee3gv1w@oZ+|Anu14R|;M0 ziW;EViUC@x89E^p(ITInoBVEUJpE6E&XjUp-T(xh$RCRoc$ksYH3mGd>@!Qnh--*S zP@pXAe$riJV%;#Q@M!NL_!}=HoK8E-4MUc?hb~~Rr~}pvb?EBVq}oawhqAk1Ev5?1 zwO9!8gA5DbgA*tXbEmoK8&Udj5NPzVN91Uj14J8)-0QR(1q%?JLy<vSA)J;2EKPEV zxY)iNHmMqKM0Z<&j$LWk@i8(t(Ixsy4w@;&2MbGQm5A|!eaCwqM1@j&)ppARjC3h` z!(4h^#>d;ist-61ouh*(Uop_&4RP<&?j@I-g06J9Np6DPWL|rCd;mM7vPs{_*4;G5 zz9XXCBQm!*>*8;9Z3bPg9#VBQ49f1pb7Ps)iKw3X8@PXhGuusb5_&VKh3#0$n!OfP z&}K$)HXz-q|74``8wMnoE6w>qL+%cg7K6(38CB#}XYc;=s`q#lwgf$bRcj<<v;V#~ zuC`^*JN|YAA7B-zW6empbtHT6O-B!w4F^*lrEmdyez*wKYUvZ?a^vFt;{<uv-`*(w zYFv3{R1VsX5P!gYwD0j`9U%uZ#45*Gl6+)?>|uy5XJNnWU`KHR5xPXFIsPV2wZ(k$ ziZC6POC)JLF`P`<@Q`LtFh8uew+`utAJQTgk_A$@2|D~RE{r}(U4etvU;HH&q-rDL zs!T{7#iR7|C9ZG!%Qz5^z`17po~^}v;i?L>6HY9<ud=zwUSh<~0s~^o>jSG?1HleD zu$1gJO?36boEmYpwLnJ1y*r8fpENtCh4X}O9-S9+yNmd(H^11|26g(26$uqf)8tND z(+W>Y^PXZWPN?@Fk^Qi8$xx<ULMZ;_e#}$}9V%2gPceqpf^8#Slcf*vjEM1cR5lEp zLGNk85qy`EA>x+dJ_{rC`Grqk)A5ahO!3zGwx|LR%qzgW8buJ5y!~vX`VxQC;OAMQ zKWxBp=RrrguLI%9At9BHYEbH}Iy)I?wH*}oN554_a~?>Pv2gc*PsdrxYBkP<`M_<I zCao5;sip_(h#L~mFlR(etn(;DSYdnJ>ss8#L8`H1KDu#S?H|yph(5gSE53f+{`IzZ zE0(aBGwE^%TDhx$-EymR_>&V|7_eveJiMs-`5?m6GX8QpPh{Uqvq6tGQ+j-tFpmOr zhLt)T6fJ;7fPz#g=}>`b5;(@hgouw0v=1M+yB)xV&0<%nxJ~)zW%Hq>pGP=V`(M)v zdd-sAei5xNF^jJ;7BFB9de|JgNrAHEw}81y<OKkVw8Epzf@xmy=Gz0n8UyxeaQT65 z+d7|M?D)sNmQr86Mna7e^sAkxs<|V`KOeoY3fdZ$cD@tBH(HHWXI7Yq9+qZ(`J6gP zVY#4oFMjt6o23{F5F*Z4$CWtLUw;u_PU_4!Djd!1u$9NlX&+JRS$5AmLlwoD%gJub zFDu9=^?73ALnsi1VpVXaTa0)vw1|*iLC8>gJ5SxNR#cl=I1^yI3nwbL{F;`(Z!Y9Z zxj!%M$Df(|7?Odoo4n12nOLZHeOdX{%h*sM*Max^e$k&pLP)|wg+V1K3>UVqDEh*8 zVx!!?)gmI^7iKS=yDU7wp2<&6RM`Z&?XXFn(w_cy&TsQDhd732qq*#Q@q6)eLoa!D zo^#A-qzqRftVd7l7c{bRN_y!-i0fX{!8+aJ)=Cla@y+FHB{C0tZ9$%_1;YnSUAQzk z=(`~+u_(NhbBDc+@@Hvv29J*r<G<ju8>TkzhbQ5_3bU?Du}26T>Fk~$Xm=NLhd~#! zv0dE|+S_BZPU}g=9dMtr!VR|A<*5>7Vdd0llo&E!DLYz9;)DT&=mC~hxxDAj^~PhA zl<nfVtlrp|@lWlPjlw_#Yi{n2d5}2$&y)a-b!Qf$<Z2qtg*PSp+YC>UMtNJMS1V!G zVM=g9{OF+GucapPKX7Rmiq2#1bUf|T_=NXcCv}%>K|=DC4g-+d?h|YhoO$t<g#6)2 zXl_-JXPs!ums1(vyu;TAp&ZUlGO7reb}X3mn}9%_C2Ksb^FVwG#qujnAUXA*c@$GM z@xZV?Pf@V`12a0V?GJB_{Ad_nsX({GdC3!tZ4tSOhDeBj=8}aemAEC(B<*^}XSy_E zXO^8glvKj<lv;L0E0bT{da;aWRj_yyw>=s^O{>_W<MJ1-4AxZX&|s96eWNi{oO*W4 z-Mi4^@fN(WefOojpa{*$xu4t^_1~U5nU*KDB6)R@I%B2#0-9VJCFJ1^IX^@D9~a;? zd*N=IsYb9)|BR-aV_p7P=}if}$;#`<)NPqzeA(tO|I-Mz>2wslK&1jEJBJ_^HiEc5 z8htbPdtlMX*0=?B^D_NM*gOU!8Ys>*;u#KpO84;lHL|FMSCy}{fuef0-XKKmzCqcg zYvqdD+<$%j%G9SkzAHxV`sVFRv^hC;{87olVySAX4!@{c>4i9(U%yGBp3gig{kS*2 zN~pX{6|9x((#@%+M`A!FE)@ne0!7M_&`GEVhAM+Cu}uUIbSFMzlIM6keMTuQCehFH z@S+5!@O$P+XEp@hV=4G*WSaSAKFZ|m6b_FC;T5OwTOaS5lunD4x~PRP143jt1ipV( zI|Ux_xNnNZ7T)(5QYu)WsM`NI2(uEYbo=<#is+aM)|)5q+NS>)&`8qck1Cy|HbK7Y zhsMg@>LO*Kq9&Ic53yUP#2QhwijG)TP#wcum>Sk~{<T*sx!=)gKNSFC>q?=%71)PX zsU-tcI7iRa6-Huv$<YycNWbtBDr19~>RE{0*QJTu+z)*#pnzyU*guaYtI2h7{?~4! ze{b-QTOIw#KMhA1V8Zkt^d4GBXmOk`GBBpwnNV^-v-2W3QJ|m@T`}YU2}-n7Bw&M- z1r&9MbNRNqhflo<o^9#7Z9tPkr=|EA7g!%EEKP7P?z1l5J$%D2O(*TDlp&gnD27rc zjMu-ar^Jm`c|#>_Zo!Yzo^4WPAqD)52Q#dBD@PBDItFXLC1RM+2JJt@p&*3?epp!{ z$unx%EOAZenbxljJRh4sEO^EF911$x3g}R&=pTLNEWFEl`AoT@f7kOiwU*A!m#nqP z6u%850HQ{h1?eT?mZtdH+T;;HCS=(V^Y!)?=8CSH<_mQl#<^UNx#lmoqXk_$vTz?0 zYwlgMUEjRF{ZJL<K4Kn{?@&r=q*NZ&dLknJF8Nf4t8$9o;RlC7&Wys{&VN7K?`3O; z-=Zv#W%7#JD46KoQNzrC+RJ1^G9kZ9;*-@K#v-<RMD((HzY}-UR%`r2s?Hi2i`nH+ zhybw{^HPv4>$DgoOGv`5y}}R^ygrg$)59v91v4d=%a;MrSu?!za&`<gGuKlWMF;GH z)^;p5YaxK{&+@ah)t<m`#hIZXG!?Dw+~4JSY(d~(I2$XVKmnZsHJfn?7zuJ5CbSTE zbVLam-C^vCQueIJ3-ct&uY;4wOp8Z(5ze96+BRtfr3<#j5QyWb0G_7ZlZ|u(JK+Hv zqvsph$`FJx*=bc89(wAKv@G?!(eNFA7-sKmV>)DzlBv}{B-gk;?}%QYV%iN-Q~@D$ zt(C5DbzB-MIGCvqLXxL8;*NtF4o0$A8xmHu7yn{US}#h_Z(=?eaT19XIVgW6jt)#1 zy@n7plulJYl`gQ&A|pc<ElMG{`>qa-W`1)U0H`PHX$kx>&8~joQ93O;SsR|Wk^Xwy z;PCzAr9l#?&{t4#K=sL2pmOVQ=&wZj5V7XDK|HF?A02iUast;h^|R~{d$?-S*7hk# z<{84O_uEJ>*Dq(i=~!N9c9ytrM@TTHGQ3PdGj{C-MVqYu#(_*!E;rjr2d|2WJ0nzf zFK{=Xj1BS*KVj)kf04tO*VYX8xGq1wAyC`9t6gIU7q|=+{Ta4Qw1#uLzgz@=uKf#} zj>|%CVb))qgH8d1=Z!bx!gRsXDjS3|bIrsg{p>P@*|$>2`b@U0;f|>fF;un?L)B(0 zJM=bs5Hfgh`RjlZEAsgz=5b|n&u3SFL{in7oidhl1F=Wz^`4?Whn?T6fi;tzYs*!( z5Oa2n{<Uh_l~RYxMA#+wZYw6CkT3ik!jVKW0bo4+VJzm7&=LtrRVz>YPaT2|f}D-i zJIx9bRV~n)1c(MFuTQM=KxD05{7Vj+n$t1H3?8({-MB4rk6uBhFP9`{gFcttipaFB zJ{4^qis$Z@1^g&r>rH(YhS#Oa=<q+LZCwga8rwouU6^nt=BAT1n<|eX6EV3xY7of| zm6DQD@*+x6qa$k|kT4El1Fx>1^;7QZ>e*g`Mntd%&k!Dix_-J(B^BT(31ZoKG&gw_ z8ObH38)rWb+a&k2Gk_nvni3~N4Ik{0F|nEG$E~-yU49EbtMO>&PJyf~yqx8QJ)Ir* z09fW@)@gdUyUI&@wbOL;03`BId@LfsvnB4^tl|Mr&F7pPA%@Zg<XK*o3b&1`ZJxIy zWmRg~v^FJgX`c^L_2P%<mFEI1EWh%skJ1&`79N#zmCO}|XaCSWSOwhz_f7_&d1;Vn zkGwd6lrLjf=g(VrL-R{29%)35pmNfxAP@Ow^M!IT_}v{fGi&tOENcZy7=<xrxm?NM zi!4wsd?g$b09u@!(i@K!m8cJnyNdY=!^q3@ddq&|R$dAy69EI6qU?WD)Bq$G<vK9# zay##7i70gDKV-n@L=RC6E2ynk>9CS*^&MrVS7qSj|E<)OqGQoG<~QOqK;iXX?V#{! z^rO^93W@`e6}T04)y4k!=lj9@f{w=}f#QA-5AB1Ky%W)%(4PMf*)m=zfGOfVrqgSd z2oDt?ga!)<8qLl+oC34SPM!d@z7{(1z`~!^MgD0L;wit*T~FB;eZ!5p&nSVIAJ317 zpFEf+1c;x9!2@1eS8Lee-M>jtu=FtM(EFk!bqdHs>&OlHei?n4{G;F>-3TB6v2%fo zC@>Q_b~>23$O0UVg=cgt_6~LUZ5!|7p3x&V(p0zDPBFOlRdS`VNtIg<>0Ng-I$ciT zfFIIEk;43~B4J?7F_Um0djXZGS^P{e4Tk}{{PU#bSI<|MT8Cbx?c)EvSDZOOqOs*P z=xFPP#R;{_K9W^gAHh5ltx2mvv;qG|x)v!&wuAHKuccnv-~iI3Urp4%AQ=VwQ>}O| zE5I8<QK75hAd&axOtjlQBP=D&jKKFqL&(akjbx+ctc8;|tBPWxlrIhb#C9iX)yjJU z%XupfO#ADm$>3r2`=4x)am4oN^ymBhdMn?SgR|J_NT#&e3h<CR>83!g4bS;}uevu- z^Ydla@|*u+nrFqL>>wrOASD$1J@x+BcxQo7vTg)as+u(|1a9YN**}qh-e}?icRhcc ziT@^Zmit=-l8b1|b6`3+m@6w4`x=*epI*&q)XW;pT%%T=*O-Leq&5Vl{smX@^-szD zt9%tAP8Na`g9qL}xxbYxe}G_G`9a;_%#mzAJ2;oh>eSkM{RP|QkHxf9Ksc=XGe`|* zAHy4u%6_vS9Ac@EHJtpPby6>m^7k>nZJ43Xmhw-M{R9+SsGu)gkpHZ#Pff|ppaX=i z*2n7FtA9kt9|AY>lOSlwK`cDwzfNpB674*DQO93Ej*1vf4C7$wHe9!}8<r|n;-(z8 zuYeEhoqfSF>?CisGIf~;+59Mh4TjviQ1LHLebjVf&a{VIZe$YaZlIFu3tq60$I;JZ zzg0(s43b=hIvwvOx{6$`5)cz0#;BS{p<zZ6rdkf~<Mb^M=+o3Yze!n|qISAb`me1H z!M<EuuXX|OO^PoApwBKa;NNeh|HZ?x6uqib)Z`dErst7OAdmm=IuJ6U|94qB%>ohn z|Gq6<082vb^}j!qrttUt0~r6iR6F`FLh|3er!W5+{l8{%jsKwx|20!_8v|M3|DPLQ zpr)k!(oR$ed8=Hxz7wb5f4#DzD4O4d``6utx&Qi<RogNCa4A!X+~Z*C0?EU4=Q*No zb(ZGOgMtuo?m^#<?s}eQ$R?ZPYN5mB4=qu@($5A%EW;ar&AX5aqznA)EQO_z(}3PP z&<i>3zQW%=`h_cd6)QoE`Zduu!L}f2U+I+&zW=Vkvrr7zk@?F&Y*0Wmm}ll=>;5q6 zWPWEwkPc)N7km`{8_T0_zsrKWTX-<-Pax0Q*<HW|fs=ZFhXK;4E68MD<hMu7>m3Ij ziJ{rFDGl0qqsw2DnW3O5!S7L}cT>0O&%MYQNbevOU53(WiT@^wpQY0q7<hVFRN}!| zneXd%fL>|NQftTtB?hI(F~hvZlPB5fKbzM*Oq$BlneetV3^ow?BBt?tuK%2l0jFM+ z_0j@_LyDCl1&{lDd}zebUxQ|cLWk(+BbI-0uv1`EFDjIdod=`tA6=+=g!%G62Q}6J z2@52qei)cvg`~H4z&{;EV=#EoCP-;k(+<131aN?Pr&OqL%*p!f=@0@tu8w&Pd-|Co z!VJRu)Rxaz8v4mW(w)JjT_g!KYz<!RLa3-ACgjrd3xwOLJb_P{lQXyg2lnV+s!`v$ zT$qf=fw`4<da!?TSs0C)Ru<<#YhXxR1_zTo*DcT39-1lSTV#oJc2{o$W?~4Y>d_o_ zA$F$w+Wc)VMFKB*c5Fr?|Ex9io{q&A*b=|}m~}oC<h&tNB-SuNf=f2_6iLAr<_O!C zRrlPM;zVaL;ACUKK~O4Bv)i5rFCG+SEyDa&{%TN)m9OiO5I86piJhh3Weqw5e6J-} z;g<AI$UTZ8XLYSpMnpm=%&jk#4g;17Dz~>cP470Mi>-W?2N@NY@vK$>zbIm&(mldk zK*1)yK`qasP`qiq?pz-CAJ_q*71Cn!OGc`sWeP!lZ%>}L$2M6g<i<zbdmUw?uo>Nv zISH&fDp3k1Hp4+Mh$x5Q2MuKrwj-=?qCf%*<XrpQ{EHVe#UPd=Kzi<qczkawk3E!% z!%oIvOypnb+zg*G$LL-H7&cT?$!51LJ-RCFI_1QCi+K;Lj&;HH4lCPhG)HWJK-;Ri zQxd6|NB6_O{^}yg!W9P7lL=guM4xmYaN=ml<zoC>-xYi5UW7cv?id%-#J6WGOiX08 z&^znTo4MKZ=+YKqs)6@wyAU+wuM_%joB1G93DCu>6E)yyj35Q(7ipuj0ZFwns{|Y+ zj09)eBa5R<{!03HYMC4$-Zv-+ytY{zaQD;3)yEFg0_nj~`Eo1o;^&tEF<?<JUfN`n zeCU-Pc0^kHa|igUCGM%l+3BN%HdA!90S;XFhlLG%o-$=Jm&GtR|C5(Lo7#AYC}kav zz!mj7jPozW=q;Th-kXw%TnnR*s6@#l_pM*iDQ2SsUyf5tQ(rlx?Uw$8<eu=fly&?x zc)&~>ohz#E@{DTpa5QZ${A7PzXn4ZYY$Y(_zX>f$A<9>h`szLadW@C~;i@wCx3O*y zSa2kTHn0i~JCG+z{#-2VAs)6|70rCyV1u|Ge=$wEoerNiPSN$6)Sp7t<>l9WqoRj2 z^?jm0^=k-x-e!J)k9xgjm^~prtnbCo$7hz$lPe|?k8{I^EuNN$ALm^9nN&?s@G23e z`s?9MVnaslwGrf>LRZZWJ84>+l}50F-m5j(5x*wVzMY&Lb{5I%K2iG4OR6$tz3_%0 zO^?uWhEx8@UY~PzQuNfO!Og^<fB41N>3c9)Si{wL7x_F_W-(7;tT@CbF8`H{+0bOW zv8=V#EMtgRlIbyjhHf!7JJX<^ywlca6wMsu`TK%}`(bhQqg;-Gmzeizf0XHp)NqTl zoZW}-Xv@s#H5obQ3z#1kUh$1&^7xL2v2GNWidagxB<?%3BJihCt>cbC2`i%}A;Vdq zpnPY+?qx1^I0XBMP_k}8qdBj#8AQm^N*4d8E%^iU06Kz=0?)@?irmdC3aDZ)H_szv z8)F?K>>RZghOINkxzg}eA5L04t$PBsJ#Ojg${ZXLlq1YdJ6Z2*h+?kVsh>~mC)Dqg zq}j>3%A0p;l^w@cRKlWSp^{?p<!+oQp8F;o9&~nfl7o&e*Oom&h5XsAIpwI(k%l1W z86M$h_dOot^%f`IWA|O%K`f!Bnb#{s67s9|1!MjmP=)l_JGf}wm-?Vpgly`0ZE3j0 zvpmRW-GnBK^Va1o^8G8K-E;EwfyDRanz;9yieU)razpskzDxl;IzN5>mTL|{K`mE% z7sk?<9-BIbowt)n91^1BZKk%VTu*u&W!?@>4o93$I&5rC$6F?EowwDWPicb+n9Ee8 zNXrSSm~royoofOPJDxw)-7$rfDhq9OI=4JBP0+p5D7~FjNSUC=1lueIPwk1$u;O_+ zDZYbbxpUC)6}^ZvoKM|mxO*Bk#hqTZ&g4oXg?*J@e+zY1p3+}VC=VO16J^IgVr*($ zhVfT1COsOr7vIlT|2n0@P8CDI#0(*E{xuvfCPN%eQAKiUJjdfBcVrQFuzdAXOT7d< zX#Dk+>s=aS89lMsM7D3UlZjrkOb8tZ9EUar|N8f$YxQbFd)s@I%rgWzh7xU-r}_Kq zlKoTn^RD+3!P-%t?c1vP>OGa>V1UIhQ6%c~bJV@}YYk9L4b!%#tKfl*wfjige%zF< z6>5#4UXQ1&F>!|AY=B+QbMNv`bY(c!ReOBepzH8%OIWe)DOeBiqH`H__2E~M#}Dmh zrn8&<9nGaZZps1Ak@5r4Y|S1K9P_uu`2pRJTa2$5hQbn}%#;@WsE7w5MmBGE*i+~c znZ?2|aG=!BcLLmW`7hI;QVyXwRfzhJ{ZIYBl;93gd<}8S_ltYqPkf4#VyX=%PCAv0 z=@022yzZZdP*-c>E*zwGvHE#~y9o%@sj#6T=MQWLWu)ElICr!rJbb<F2BiVsD~-vv z4dRCuckHVi0RU0TDXT65VD2!K3u(}k%!P3aT+&is?63_RxjAbd3N+U&EaG$EQQnP+ zx5RsYy)7A1LJb@TMLk4iOstFWfEL8qT%(56&C$RqGKJ3=DVJMmTkYIVU5U`Bc+_!1 z=6j|O^Txzo=9Ndaa87F6arbt^N76+>-ao-R*#itzGWq_kZ!9xDNPX6G1dQQ0Neel= z!`@OD#H5d-LQOq^6TK+~$1`LV@AD0qc#+A&Hb*I#VW?k~><&iT{n3e8(_BJz2376z z7zBUalCwkkTpB1sxflt;=QRwvK9Jt_DC%|+u)ZB+o+o2)wJe&4Q1Hr0Nn+}7D(>9Q z@N_!|bb5NLoj-XgQ+r+3?y@KC_Y1c;bt9vHFZXu$;{KGxF81<ZWZ->9bF)Sflcz2P z&pT%kAwx~Ynvv0K7A9tz;k_(Ol%WHluMLM#75}QoFI65x3h!aL$_isYx-7i6D0BBf z#BE%dY%BG3ZF35U1}F0qx)7&mCxJc3#I5VG<Ka8Y8ILyeha)t2$f+PtbF>(TQ_?SG zfS?n92%sF`qOF!H{+s$1)k=Cmu$gC5^}rIgDjN*`S{;X^+x%l!dnTJx;i13q9KV~N z`;?NEzA8zDCMl)j1Sq;bvd`0_YPhCcW5mX#Kx3Y`c(rEtcELx;W0}}4D|O3g8D;e2 z?)60C`*x6oO4JdGBV-%KRQ_+DP11Ae)Bp&yNiQe=lx1NlV%XEULN>N2zpEx=Q{(as z8n&?3i!RO6y;awix@s#U)`vc>>sm#;BoJ9~+0wdP$?1M8YQ@}l`-t3Ys6Y4FHb_kB z<B}13n3u7xfrv<kHY>ylWJ~a;)wiZuTw;4zR;DhL<tVyC4K)t(0RNPvAnzD{#C}-5 zT%oP2*=p!{-^qo<;!%7sS_MWdpaKuc2Z!O4J6$Dx&(=WgJAdr6la+;6Z1}_F^H%WP zcSY?U(vX86^=OS(r6?YUC8t^p)gUxqp9#R*opUT1gcMD>eU`v-W_RV#c=G;M$Dc78 zeijk>9w2Y|`T`}o30B_Fe&29Iu7BF|+uU5QU#><GFc*WkYlk;LVxpqi-3S(z?wcYL z)1mM*HAI%<_~%H>#`KGTr)KDI!Mf#_!E|+tsV^47p7LOH{BU#RUwxv2Cw5rS{g`=J zQ3pY_aJBe}h@O@>z!=mqNBiPs!<C*|AY|#iiWtj~SLf}3C7do?dJVxcJ?%6hx7?9I zf6QJE?LYan;{b>q9<Sbes05^RcgKJFfU;6u^9V~LM(18dgmH9ky_A>UACY59+I{n0 z2YBvT$ZDgo?NKCdQoJyG<17aN6mVw|XIHJ%-Bx1-O@DW4z6%AYj9N#W2A)M4bJ=v5 zFTY>?_B}PFCZeBX+17d6@~ET0>ci)zCVP7F{$tl+`_HsPDk`I~329eubuvK-)UI<l zag&15r!fdELXI%%k&84B&i$5gh2J7u3tS4Oq0q$OAp5sbK49qte(T~_yo<X`CJ^{& zj*WTfM?)WkwbONL`K=dI6aqR0f$bN==9x%4?d3tIxUNS&a%Tn!b=2#_)U1zvO(z1( zRg)~MM<3cD`EYONri0Z_2cHk|hhO>sIS)tweV#rlUe=-#-tMKZLudfq?mKAt)rxcw zYn1pz9ckRO@VdWP+6Qw`H>EgHeAUDBmMl$J-DaN?$7vx;c2MkRuGsCTAI&+H5l;2C z$8!(KvKuLBlJ>MFPL(i5Rk(gTL*-$fFjlq?pQ`C<2kjq8ec*wfCS4hVFx4Z{Xh|ZG z0KnrVP0;N!y1~YDCRpffuf{=+Oo*V*a_ZOfK`u4B&nb>UM}mmUK=0jzp0j~SY)?SF zt4%(xI0spUT~88{@pclokcmU=U%Ki~#!b}jOlc};UokdR-X8GhVqaSy^rhL!Eg*Cw zZx?AjHzYJy9p8t87e*dU-~5C=80f|V2mMvu;6j*GD)l^nO1}u>&5=p*At({oRB%)! z!cmj9eD|5<w{@O!bY-T6zhK?mj7`2d%#|O(`(>%GuA&wAFrxK89E_VNpUTUzuyNuz z&!7%bV4+RKR$-3gBVz&NwT%V9uWS@~Yy{XyI>({HFLR8j&icmZvGJSj&DVcJvP3&& z2G~plK3CITUOHC5`)>rT51n8b8%+u#A^`seq~i#```;EC5=yS<n?p5FOJ=e5SkxF0 z3%9XM^ml)cn43lp@a_4^a0kgqEj4v{N5V(oAyb0JR6X6_At5xNHN8nPJ&94wYef>} z+Np7trN6b+@R|6w8gHn|6i(YyYy>I7kVo<@$bX5-{*oX$kro3_K{&U`?GCEp2Qc3| z;DGBwGm#O&!Qv*5g{LpBXjlDtqkV_4eh7k+w*kq#AP~t|!b~s6{Q%DlF&vCcPN(-M zycGSXE`L*A4NfSB4*4zjByCSr+VZ^`%X*)M0UBZY4G8t&H<|lAx4eYK05oDg?G_VD zhvh4RVd0QTY-L(loVdkU8o*fbw&Z2*H4rxV)p>i)>*htJq^Smy=LT&Ko!2f8<wGn( z%lbEjNN$-0_14kK2r5N9l(qG4_~j2A%%35ubD*eaV4_h2r4*y92((Na?k64ga5zI< ziF|<!vB4OUf~UI)bExSV5<%H#CYk3j;-Ty^M>r_tqW*8NkXApvsxp?rW9sV^DSc8^ zAZ3n&C3sAkx*s469>qGaB+zO8&feR<Wp>(eXfruDsl4ByZ{5ZQ2OM5mu0q-hv}I$N zPw1(~N~8%5USBS6cu<U3MePipJJpPVfhy#>Fpy4Si!?!NqUz<-BZ2>$l}e$2be|pP zvP<2_yYWXzlVywL(>w^e!=V?c_9U`~7VVFWdY;gJ=8t13mRx6OceAz>+le28OBuD^ zLrDb$9~+%9<QLr0LMbqjo0M$5tf)L$S&VV^5ozYy3DfT387GAA5Y9bgNuF~dc#Jg> z2?oSdOfJ-vkFzI4m}JT%jf$hLh4OFTDR_QyQ*;vfF0hiq;vhrismod0n)CrL*mH&- z`=jksub%V3j+)l-DQJ2WmE<RCXG9Y$o3@@0mHvRBs|kX6Xw&tojM<zPH4uBi5a=Vw zzw~ziULNSvOU`5hed{0;Xd4mRq`}XjSDu&esXl9@<Nafr67EJZp2f-LgFE`BzoYzk z0xMEX3WS&TyJ?SVN>H>hpl*_JKXL3AT}dUn`dg47s7{&Hj{;9-c^Kr4QkCEVqG!AY zN&Y@`t$hsgZYPXz7t5!S8UDL+`8R~bmH>{qqFP^tzw8h&+b!(Yt&+iamqyoP0E9ye zWvh~O&%NoD@SvrH{+1y9z+2Ea_jA0}a!{m+B_@`73j-ewY6NSVi`M%VbR17=S#qZ+ zZF39o8VkK13t+x6;pw*eF^g}uXj1FJ;w*eF1z~vaMRjEcD`@BlWf->cig!-};0jI# zS3?gF%A2e|Tp#7zdw$Y6Cp{A2p{tYmIIX|7d>&K&*4dcH&O4SKgmoY$fD4;q9?Eqc zNS)br9tHu<82|I>lOslF8!>-O`Q4bo8GS4^UgvN*N_?BX{$6{eCoV>laDIm``K{MI z%{EHr@Z{b4YHP2!{Yv0HXxegm(#D+-Il%(1p0yvZ-(=sUAbf3wA!9iGkG}o9g|=q{ zQT^t3@GD4udA;djncFGQ>Bau+@3NC^kgY<)cSS5B0d6Ja1p@2>(MTFrJRZ|QNnm95 zWfq?s^YciomDF6U6H*vw&CvkHFTNuI>+YkR-;+LP&qCWZpPYaxzbGID+zfb#vuX5i zd~t->Fk>kxNfmF1u)3bOKDU%dRXeAs0He6ysMWqw-hihOR{~rm6K!VZ8s|x=QDVXz zr8yOuZbOT^+{x)&I|?JQFEL#mg|3OzbHI|MbZozK2KOqJlY{B%n)+gmsFZ){GV$~H zf#exazH9{j0Fl~U@~@iZo~c3NcgB-Wb>h3|k-ON24$4d86(l{*gCR5j;{tFEL@H$n z|N4p6G%Tl#YO%dfU_W=>u!1w>feRRMPvsH5b>_`Z_3qFs+Y8q;zO;B4j|TY@>Kzqj z-!@_e?HgnVv=8|GT+9<1ayu3ubjOWf;kf0J&~`?)6Ub@`=PsRc2Q~_Q2WT6p3bnv( zW`9}OH^FN*)C5epZpu8bqbF~HJ;+sg1a^V_(K|C*{XHHrEbWlA_&ayWK@Vb{CFTn` z2NXX|!JkF+KhM@o?SpsppiM%_5n*~gihdSF7f0^e<-wFz&d!s>aQx1*H)^3?4m`&5 z82T*52`Bab{lhn`W*p58=6TW=O`M_1pTB7fYH1QzILz>y(k{ZQj^#gPt<85QZ;;Yc zQpWMV&h@J>jL&|FJ&UF{Qp6p>lDzxr6y@7Do4R7z7FpW9>T{Yqlxy5hB`~vGxjKPe z*-TVlGVgA#PsD`svuON>;U=SOgp<sta_{ah5l8W!^sDW-3(PH_nky;-%c+jF%M_Q= z6fgpAn$Kkexg^!?KK;%h>0omk6ksEl(DZB1<(IUx4=LaLVs?eC?Zwrzp+(^HlhSz_ z#}L>k=8TCm#+6b!FC-#d6sGw;h9?jFz@s01iSc9eH-(z>v^vSextf2^rOT)C(QY?@ zs>bGNmiE4fAy1&!k4SH>AzVrI!KLpEcF)vZ1JQXw+CcmvMjrf?__Rtd<@wV0(aPV? z?6^Z0J~PtWb!DG70X+PP;mX>$6eutme?~)~Typ6t;RD;|E}}<z{*;Zrdw6O27u_DO z3D)dCCnJu;mo^n>Plq|yQKfg#IJ6lB6ilcqCcZF9YA8B%3}}dVxtXkZ>Pu9wG#Asu z3olj<8PiwfMHhXjDz=mpYQgwWA7Z1Iv#*WoYN&Z@zeL7#z0J26>2>X9xLi55U|@KX z4g$UYjEpq*%2Kk|q2`Bw`J;Uyq$o15UGbLvMSYt#Q_E{`b<5;u_eLp?$4d)M$Wj6$ z<xff@#nA3!0&WT92G<)^Wh@?{xN6xz*TF>`*T26~57dNa$m)qD;<>!-q6Nlk;Lse8 zJ??!|wll0=9>WS(b_Vci%sR@~h$U~~8ZQaK?U5<_Hqf?LT5hK->zYINCtCXM?-s>= zeTwZx^kY*c!iK+d6Ch%m7@%zXcotJn=~?uui;L{LDflj`%{%sxE#*}>@2)Q%x7%8M znY*v8!T${=<M_X^_4X+36y098Y13-0+nl=ld2{lt7HzbQ;9kn9EihkEnK(PmA!ptZ zApa&xwi!)%ASTHfPm!4HsAt~6_5av=@2IApD1I<jR0ITRO4A^sbft+j3q`5YYlzZ& z=q(g0y-05Y(tD5&p{X?KO$Z&NLkIyv0wjd&<NN#V`TetJ&;GT0&hEa$IWIFzxpUvW zckcbnym_T}Tyu3y3~9gBi=k>gpmv!V9O+{(Uhgn&Afu&!Ti-WO@Vm`Gb=7nv9n{BL zoVH}V<TW{+_@n=deKGan{Ak&E2;)dc>k={O82ty9<acn_qaadZ&$eXLsi|(c_l8m< z`*-5&cv@OJeh9sH-$O+$X$`YV&H5>7*#5x~?xp<NNL)!y5%8yyyjhYv_vDEvbgw9U z8JTmd{NL`tbM<?jQ@57*m8vJo&Wjst+Is$tFV}F=$>Rqj92Vj#6yd_jEB0emwYiYq zO4V3$R5&Jd!_T!Y5S~4<?kxK<v5H-4WC^L?P&}5oVvt*1eZBlX0*nA_`0r+Kh_V@% z<LKKovDfV8XYPw?Tz?-H!+r(CjiP`jG4OW7kD3Na3+CG`i#Q{W5Ckj5$z*0)@k6@1 z9z=(u8xz{#^zP8VP_*<J|Dc9g^~&+z{QRz+zF>MgvS&Rpgfe|f<!u$PbAK+kqz+y8 z`I|e3n@*kDkh*{7g7~=0fW!!6U#!8QC)3gR%ry^sKs8R<C==t8<s{v)ktoG6M{Wt# z?0Z9ZSTXmLJ<#95o7EZ=DT4VN4-?Po-fO-)U^mKjSkn9yGrh?Qmf%hFG<r0)i!K2p zHXmT#H5HGIGak>XXRff(Vl#L4iW#aPkD8#RS{<&L-!SQnZ<w^6a_@Z6==PYMiucI% z9=^s}0xm%`yz8V83-9W=dg=|}_;BH}Jxr>+o{O$Gn^K!wDL&c31uK62gdglJ8~SZy zX{%afPAmJmS@yqu`w3ro=B2}H9JxKO?+m$`>giQ^bn1Cl6nw*swnAq-7Y59_JquU2 zL>Y7tg+7vepSEqcA4-`87^SX5HN%|iC<meOHV?*e<Pn0&Od{cjd!>n5ccVD84@Z+f zjR@8LmOCJ<>m>mf_VCz|W4k(oa8O=T3*(@qS0w^dg2mn1(<%u24=00!SBJ9?OA1sv zb`qbvw`}d3G~+{bE9M$YM{o=Eo@K}1VD`}k#o@~hm;yMDWqt6h39s*ctKvOxje}Et zJ-tr1iq9)YJaFgOlF){wiIZt!Ul7_&O>#x$4XiYUk-gUl^xicJ5D|l4n8wy^BavMm z0&GYu(B(VcBY7z5v)q$SXPB+4;&r{sSzFOB<~B?xo4<#X*R7`z8?yCR0t18h+Gy)C zZe}fcU@5gSBxe)N-8W*LwdcQ#i_TEGWv52nXC@>4F{<@!@-jQIF5MrXO{EewYJ^fw zh>%y~2)+J``EN}hArS>ig>&wTj0WZw$@dFpcBby;ugg7ARM1M0`vrcwZ9}=j*jMS? zmNMByWszwR;MGbQC(~;ssLl~0W~9w#Ipu+G)kX`$yu7-b+St^boA`H20p6o&kqD={ zbOPnDZ>Mc)ful5c>1kTqq(dO~-!~~8nLV=taNN3{^We3Nt=-$swll8}+k-<xwbOXH z{C(q+JSw2mI;r+h1y@CLBV^D`?fp3R3gV&wHkccnwE4g|WC`tY5K*pyu~KE%At)Cd zCCEdV4W-6hAl{)F41N3J^*HTUTH;|R7EvVf^w(0`QfuGStq<P}FQlb|4-|(yDTXs+ zs*m@8_x6cFu63d-J{{L8<cG<cR86^ppGO|)vk&|k>u)1tW_j|{FxMY9s~<Rt^;GE3 ztsHmq<nqk!#UpoDot1%8ZL2gD1iA-sc8XGIrrMgOu0oz6+|lMm6LIO)`Ov=doIe)T zE%FG57E3rp`h=i$SlImnY+j_&5f;6^70!!>1*&p>Ta7y9Q`A(RWEdfbPx&6@?EmeO z$SOGUK)L3RGom~pih9jcP|`P(wC-_wz^WVrG3`puOLap8x&(;{FG5Pcm2WJTB9_vv z=w4Q@xNmOR)cW1WU7Z`Xw<^^t)930gyZ29x!CPb?ZRI#fE=yLlTUU=B7JY>4NavrI zL2*SEnaPembS}%aIIy27Ev0lXZ6}VkH_FY82P_RWZCGbTSSm-FqS7%V3DO_uf;=?8 z-!o8+R9PZizg=vbv>d(`b_n-?cfG5Gbn1y|;a`oat&hx{up0<;GwDT4BDlluS8eQ7 za?tCs=*2|;UEmc72+;cOhmh;UhL=PLYc@B&(~Mpoa3?=k#@Xq6XByaD#d8E4psA}l z`IG&#z{u<)oGA=}9`FqCjgCc!FYC=ACXXJ46<!S33NqRl>e){*PFm{oY-vPDk{nAp z>ip8=IAS;QK9&fTSQ9dbTWp}S8?7C*ANlt(n6w0x6~+Y%c?G&*^X_vL$}%F0GwK0C zv<*0&iFPQQ0XO*8Mp5H%z~~?Q0DX7kjfvLUT=MWGRlUULg>f;LI~#Ql7P7E<GG=`@ zZ@{9_12>^Nsrvk`(~Ji9{OmonJf*#a5`D&V3iWj7yx`8;x(pTM#^SE5g>~URdi4(c zTBz{Q!OG-_j>SdOzW)6O6JN2Kkvo4OIy7x#x2h7WGgKz`R(!XmVf$TZQJbo*Fb_|| zttpN1@fJy{+Jy&?mPqMk&Ce*MRBpZQf-!61k~8>K#h^sfpy55sh*u(u>O^X#1FF=_ zLT0viS%`*`?!7e+HeZ8;W_@uKehrysqY{msnu--3z|$gyK_cfK2nq^XK&U{h6*(_Q zQDTa9->$$t$Ez|Z%_57c^3kd_EeO%2X79fzN2plUyS(NRh;W3;&o_muw(f;R5e2t= z<!TSB6|`2&lW}7f*)Iq59Sj9#@V4Ow@#nK1^Fvz-`=z<Z)j!Es(2<usR!|ulPnM<; zKC<>7G<TYNs<$Lyb=mi0cInB-yPbceLtYvM-*#Q$_qM*&62`F^{Ukj)9iI#iMhx<g zCNC_h2QC%slWe+QV&0*K$Eo_FjN%VgmaF$%icYI|z&ujZ*a2K2yOAX{_xNAW?xt7$ zn1>SzZ(pLBe7?<3^^K9tTv1AbUsoiC<{w94zdMIUJr8(k{9a#f(&h0XCOESO{zzv| zw60O^VBWR5&qFpX&f-txcLGW8bVFffM4|@zO~N~a7GO_lUy$*Q4R?ej2U^bmmzP5L zd1JhP+j2+2+|5{O$YAixxx1>($XV!GOUs)W?4~f=0Pljmfd^K-b@!07w_G3+qtW5T zCcWKaQzkz%K4SH>j?>4km<V?#Cvg+s1{FrNCL_5A(IGG3Y~`WcW&^<V>7B@`*-yY< zmqPP}?;ATAxEJP@_RfNxIt5!-mV>@OtSbK{=H%^`MA&swdY3d2^gG7gX`-pSpF6p% z?gyJ|$8bk3pH%Q;?xH!PGJ|~Nh%4xBAK5oxV}qxAxWmoTIs5dPFW9^GONM&)?<h>P zvYtS<k<5UhvHf)s#(~7IxUt-9%|R1`-w3OSg79pWx@@(f@inFr>Ox=m+K}24k9ir7 z5jo$^*5OqT6NW0&8TvM56}K8H=RJWmr7IV|K0Na*Av88>e?G6AMsfRuFNb0XX`67% z@dl;K7`>D=qxQ^A5)tcpz(m<vczU|teYY>Z<@=qJc;0=rzFG*1>5-Ut+94--B=HlA zqd0t@YY<JO^R`wL=oYoT(JZ92+r@_XRhXt`anY0Qfad4H9_TTZ?Bcmadnv@Etn9JO zr>N(*09Vv&ZbPrE-~l3ZPL!*08t#AcM4TYtYHFavs7K$OXe%kNrzOVIIbL28iV4La zOTcMhPQ6MF)Q;~+f<ZfiKZy;Ug9=g>==M~M-M9>g@;8kHW`n^wr2&%8O_f?UjGf~; z5KD)W$!NB@_XT|69nqQ|Slv!_SIP9y0=^Lr-FwDdD6hPA5kmR9gq%>!D-RXFZl-R} z#6Y;c^ayD|o9pw&Sgx!K>7F>s6YiH1UO+%z7|P|3=0@C-8^3*g95%dea}=4m<*E1$ z5Md&-qRu^ru#I-ELs4Q*2?Ly^IQ&5BPtwhF!4}FpS5uVNJ*0G695_aC!6#`M2@aU| zxxl{ui;q1E111I=j8<^il3D^E>MAn1DsE4f*A)jPGAib*&v`N#m@3k%zixJ~p~ujI zB6Vtj5ZCawk;NaEtW^Rs3Zo>sbUBA@qP6Q!Jgn9(9E}`Y5hxcZVk;>vF_$u*b}JZg z-tq8&42`bBIIqmQyD!)=II~Oc7dOVDFM}lSOal{`K-6}xXT#8KXma;WtbO1w-J!ND zH2K#%t+L%OKfE^MBsVA2x+~Qb!AYfZdf7sJBN=Ss7ki@u?44JT=YG<J5}tuT2&ul< zIh)_|vbW{j9=R2p7`G!fL!mlVGTN+f*40M1zFD1giB;GXiC#Gma?t}s5V*b?IfwrH zbM^~_qLI3Bc7II$|I&Me+yv7zYgdSOSt&3~mzQrCt;$N@_bbwNh%0Pe0D)c=QW6<@ zZk+KU7iPE@Q+5Sl#;-Sz%}$V^&q5ud*h5DW9{-I!04iT@1#jIv$@%j7Q-jx^53J$= za&vz^1Nj$TMf*N9IS)bcTwA(*m&%;+`Zv(kvwyhm*(Uo-{k3_A4Ebq$muktLC+Kr$ zki_-&b(~$Agi_*p5QsI%afbHGivl}><|l>ckwR;99eS^Cod>={e&@!Wi$H<v5e0T1 zp8Z_jQ{XCS9zL&h380hvbZP~O7+PnW=>>X;gvw<JANSfj@_z~oXAg}<Q?4?RKO{W^ zf!@<cn5pD9{f}DQ2|Lwcr)Z#p!Aq@#BP%_Rt6Dd5qk-|!Q)-r$uYy2}e4Q1=4OPFz zU(j~EI)GhZlmiZ{1S#qDQVSv)MEHOqa{x*JbjUn?!L>a0!Xq~_<!Prf6A1LP=v8YK z_@?<^Ak&d7|0!kmG6?jRk;dP(bCnsv36T7$>x`a}PCOHW!|jgJ11ri!xpJU&gc-$w z4;gIk5qrV4_DBY3y`i9C_IJ@(H}ZVSmD%|&<Wr#3vPl*-pQ2V*$^DcfTe^!t&ij6V z$XyY!2SDd3i;u78hO&ohbrmuINxh|y*HE{CKEs9tfPz}Tte6lF-R>Y%PHu+Nn*()~ z4EFI%EXJ&mm1jT-^haRjnW>=h`Evj@9Ee&M!LJt{G5E6r$;hih?IL_B%A5e@2R&3Y zhdXsS@|w^mn4cF~E6eFXAad-5b0CY~#Fbaw4ph>IlBwO&+ds;ETBgg)7e=L|#@)}? zGh_fmN&@umJ-Qsfow%^z<7?vJhPp4wUPqg;^~9sZVvb4)|4A|c9vvcb1@seO{zZ_f zK{*YJe)GKFtOgY9h*<Ecn@S@LdHW!swE(^M0J?vAT7Ee_nOnJ0q}C?2d&~eU9+I1z zBv%t%+#M6i)rZ2(a_(DO+mA|kL7<xeGe9mJ;<ZkWQS!sByRRyg+F+8kGRL1Hntp@9 z=HBJfZxLqZriXmzK_Yu+LqcgflqQCL`0&%c1s+HY|3vW|OjMSOuD!}B&twlVU*So< z@<npT-WQ{A$jeeLq2*SMZ{#u36z+w}3a+A3@jZnYG_A+BW4M_2(|78E*Zo-f`^~B! z*x3C*)x2I08h=8vnH$N1L0xf4GrxmcSNGXytlbTvIJK$zLZ0!qVsdQ@hE4NJbiI<~ zhLMZ=ps1>wJeQh0UA$E*Wn>l0i8;;dWQL@R3+^7wU4rs2lUP6^Nk^Q&ZMDy|8?+d_ zhA%OTwf&U!xlHtKW9gx-2sJa)=|D}UbMCn;NxSi5A~ke=5-HCPU978B@~W-dDRCg} zWGKq}iZqE8yKI<~Bm-wpy7Kd~Z5<1bwGW4cJSp1-g5xoXCsvAEu%8pv2CWBLGj%Q$ zal#<oV1S)>LpkblyL=Pl>C}fLVVNg*xHl`As_tri9p#sN4Nu9tnGlT{^5M-4E}6#| z^_{?rUo&$anCkyaGw1XL{lqV)GaG)#%lv3M9pVy$HBUNpur93p+>&;)pd|9f={dZ_ zam#Dae(0H91`dWg#_hcEe_n)RtH{DKosXZ2#j_2y_&vh+ih8!Zd@CUMG;90mp{3iE z?b_#P8*C;?DIh!>mKIZ2P!imNS!a<9CbCe07N6b)R2jhyU8J<4LhJWyGhn3N1@YOv zPq+%s4LONtLRCo0%3R&i#^d?svMmxXr8E4sgEMZ0z$n#<{f*4D_1-o}o)7hwr?Di* z6c(aRlhY6V8+SKzGUX97Yo$$3LvC~=80wi!x9({@;iRKht;#Zav<jSW)gmu^it<B8 z2^z?~uDuE~)5p)nQye?<wo?6ad_`yR(^?L@D1gA}@6ZRJML+Fzp;Tf^QdfmP`(~Kl zPO1(j$ghN~shA-y3Lp4=QrJftWS{gqPywUVd*O%j8jbu!eC-45Cd9_TQM4TlUUO)> zRuS-%A3Si^>G^8`_W@(whB{Lg`m27Y=87AiDB}|XV6ammGFtVma|s{w;PpV#&bLd? zfiT57BJ{)!)_3Z2t;M%DDJYo*^NCnHbm??4=#&_xK~LZKw{dT2r7<4v)N8^xyT85C zSUJ5W5I{aK(0DS^A~$~{R{DtfGANR!6_|8Usi%vd#^lp{k7Md;2$?GLQh_-bHs0ye z=6i=TF=~O1Zq41{<X?qzve{aVL9c&7!mvV~J5Ey7b&#jP&Sjh1FV{Ow?o}P&cucfg zehAwpge$mz|Gd4HP|Kfou(Ney0vCMjoe^F?R}5{C=5^)H5q=}99Ap9YDEB1haL2<v zYaTXB!JTvJh;cK`ZeWQ!>JF3F!VcQ<ZvckrJpjNooc=Z8zu#sQOX-OvpyZtAe86%3 z3lMjU8H3ZiH9?*|^?f<@492ANCD)NT{I67LITb!9sjw%gcbUu&w|p75t~kxkAop}P zh80W2w+St^BI*@tuwk8Pn^>#BxQ5i>>hIp8P3RA!?uSZ}2%msYvrhkTJW2sqOq^4I z?pe9djGE!dn*K#f75FImuKW5(9g0xb7SLLV%XdPMTyY;M9%B|$VDk)3WW4aKUB>Hr zS$<B!Hw&t?EFkbu3(qzQ9&$6jIWrPYRFWKD=N>!<>blhxb`BINo>h4e-9tIzf*}F~ zZ1nxbMqJiQsrn~}+S#@8Yfpd2W4m5{f825pJ}?)5>e#s=-z4ooe4uyh44VRcte>7Y ziSb;flT6f})49+TV(rpOt^MY=he28k_&>i4-%ERdpT;z&t>4^?ltpg}4;{@^eSmgO zH#FrI4!mh3@7S5hVWVnMwSQ)snnopf3vwNN(_WQn6MT072P87+Y~c8t&R4N9F(<x! z`?X7Q?!~2FrtD?h!d7d-SHf&|R(xn{<ev<4@Zk=xn&0i)+mw|xE!!Qe-XmR&vXNVF znf_Y<U(-mcA5!L0poMtcOW9C&C;s^&e=KJF&T2C*$0l6I=^W@i+ja5_z_v&29j7mo z<*X<OU`L#XUpbv;h4k$(`F(5j7ciK(4#RcX%sa?Uo<tGN$O<Mybuz~&8%ojLK)S4W zv3rs3Pm8yYyLQI7%e+p83I<7}rwnYBzuCNNS|O-CFG%-{=UjIx{SA<a#xN%!RnC8h zgnOX7o1K%U{uz*kqY#~nL@PnT#|1X3xXxiJWobIkE^2<&wWAz$-+XUInYvP&G`X<m zy{=77=RRM@f>;}=tZ()UK7So=PSx(6V1_gx9E>ATM$cnaD}PrwISRGNuY1kOdDP&? zp`L}A{N^I=ESFeL`1CP#HwXAz7sv<f&|(EIPFu7cFJlk2%u^=?6TrA2xzDU$ZqF}l zKqb7hYtWFh8uOBOnl9s&mdN4S>xIlQ9M3AXKdEw-Tv5*vQ+q5G5?#mnbeeQEV+|2O z*!q5dDS+rc?`iUTME_SE&S}9XS6$oX<;2&v%byy9Tw$ber%ERY$-1EW$`<ddm=UH+ z-ZV}YfJY%k`~q}bIw~a@aIpFs`bTG!MXO*zVzp)A>V3_%OauLf)dh89hF7@j6<TD& zo`t0Dry29eq9b8v)x_QNnVWlo#a=5O-NO#$gOJB~UT?U&4&v{Du#EP8J(F>7A_z45 z{>(u#g(+GcQ%>-+n$b0Fdv@!CN;r>@IqL`m9qLb+jgmI<Udn?OW7__*tLlA0SoD-M zT}?B=?(p<}snqsoaVMfB7I0r2<dQ|}QMMwP<87h=JB0>jb2E*l4@VvO8-2S*rG8P6 z9h9Cf9m=tS2V?3Vs=~60OChJSMS|p$gA4A{(3zQ5z#90J-kkXR`@2m~|Evx3ssPQt zw;t%HPfkruO{6D>NM-{f{RGA}@};vgGMNel5^+;kR}aq#4IS0=cy|JQYk5TCIdjo0 znceREKXVT#;|E0+#Pe!Kd!G5Z7av)4MOGrtyOaY?;>d0+4d63!2AOtsD7C#gBk;_Z z{J(bZXFCU4PV_x$$2Uo{obJ5DRaaoq5*(#X3<6n@lpU*dFY?CRkcxhwubo{*J{%g< zmhE$lorm%<l=@o?J&mEik~H|OdKY20=VUh2(sa@n1<QQkyHu=r>1Ciw_XqU~u5bH) zl3ADv4ecgi$9oz^b$*><76*J+K|ddW0pSlWT<)(7P%dCB9QyDwNiXnlwAY`$x_Yx( z(}B(z_SitCGS9`|vT*zD^VUalCD%aD<o`3!kFFK4Fxy0gGA1+M=73>OwhhPPevkgj z^y2=L*?16^Y5@TP38iqrqTf`gabNDWI>wXeGFnG@GT&})e^JG5*qNc9G{J(L$f<+0 zqUNkdQ;PLM5HEP;NVUDU6Zv?MHg@|Y?$}$nOhCjBCi<q0GP%hQ=@R7Ezb(BWGs!c$ zza!5H%-;RLP)oAC3)O_*e9hun=jJH*vuiTdU+_};kV`rsU-ce;lWo`UrcxF7ZGIB= z_ZH8WhmxZs9%WvX>@frI($$3dO3Te{e{M1}@qilh1XArzsVvZlefDnkV=KgH*&*uL zyW@|?Q_AxT$+He(<UOp#mBOQ=&4RV)!BEy;OD4_`A5{6Cpkfy3Pxx=*oH!n6^r1i- z{NL#4&()i7@6)P(ZVHhnRr!6)iT4cx{xAWSUK-$F$}=#Hw0GZcD>{Jzg2%VXdf&qk zX&hHe+us|TWph<68rH8)lokH>y#RvE-tS!Ny0>lZo{llVdskm#fwiAld56;$>Ij<Z z%~zP#!+H;yqv78lJ0$8cLPkD^7gVV`#AK(sUdE)P&y&ksR&(_(frJ0u%C~%*mO=lh zO6i7Je?beZHORDJs4>Vl^_^MK47B<flSJZbsgU8c^TWQ_Dp$De?F@U^=CH|q>~`Ey zDD4fK5}=cPEG?2Cn4%6}n47*MTDsE)4}T{yfua0p)yXSD8FSYWUS!pN;-a+T)x5(y zw{0I?;GxE5T3AMSh@mhQzTOcsd&A{nmMnI&buFbQ(O>v4jC!~%w8qFKn4DM7_;u{P zkSsV*+vHI1G^Clp<=hEGeb<1v<^B!PH>Zm^`{(InKThf4G{v?_WkxE6VZQmB;e|fF zdRE1E!P;M;>}Y?x2`r`Np~!kzh`E#BrOx9jyRp_!HtWJ@yYk1fiF88c0ypTx=ILx$ zgG0Oj(3+A#LIt<T(Dl`u;?hH^n;&O%Xqq1X7$lOrrT5omD#1wV<IYY_oN^O^!&WG! zo`j$IqN^9ixBy@Ht^HEr+O>~UYX#nP?DlDMqSyPMnS|=UJ>9YzxtIw3#^H;@Xb$An zT95ozUUH!BmkiRKsrI_k-3F5#`({p`dql|JYd3gfXnF_M8c?>gUmmt%t+KgKVrB;P zibWwUu%>jUKS4PYa7XDMHp{#H!((_gy=RS6Wply-pN0vrg=%-Y5=Ir=)$d5pSa^A) zf`jQF{E7LCnXj#I*WWz&y|jIl=#SgP#eNL0%9F@^wt4V#ustf-%2+^$^-Dgg<Z4kq z0TYCXB5yw|YQUcQkY+?o?U3M0O!fD-Y!eb#eCsk`J6|(n9fPZ=m0tVWo%~?!6s3;h zX3>BqEq#1m?*cHNu|GZbRp%wE{Nt<%p7H^+H*tpPvbi#Cbv6|tE+?c3(Se7|^qY>r z=V;(Ljm4s1hDYl3dyF2T9er%TdTZ3}U*}`Q)U#p@zh+teNu+hpJJS&U|1?DA;ZC`z z;`JVAgZUTN#W5ne$=+Kz^4X7r=r5`^EVr$b?Rr-~VJYpu=7%1Av_7#Jy5bjDumiUv zK)&iVp0xHlJ{m|GqdFYtoN9C@zC28(#f|q1jU{YvuNa6`T>vLbhtyLC^s=6NnWio} zF#VkG#-G8RmWfwp=tq7P94kkM<u*tiM+pq^xGC#aA~iEYpm#_@q2D5XLC%`?$TZiW z^LkiTF1_w`kO<=Ig>!dAEm;Pk+E0SQQaA?GLmcWbr!FRoBZ>J&!FMk)@#_<uLUE?* zS(@!49oyR*H(pPNPfb~c^ERQpqP9Ptd~~Gm5f-9NVXOO#K@<z9OK~rNgp;gT-5R;$ z#F`_gMsCr#wbO?ks64SQyoCWt*Ie^R>0t^@*$a@Yviw<_>wRLfC5PrKYJhESpG<!2 zYoAy6ZL5DB<if2DASi%u-}^^Mpry3rZ>Xu<!b5FC<CV=PV**njw#_IsjPI|%Q~+OK ze2^l?V`98D<Z)k66s;*nH`!F>@^8*c$)>%vJ!4ZPFBbYuIC+R&HIK8p^3?xclWocv zTUr#paQXGH<I_>c%LUiVUpfZ5wyy7|t|>%{v86Le$3oMDIe2eNFHt|$n1F`Kv-a$1 z)?Wad-;=FeR$JD)_e-mc6||_N0KWsWd>?Hw<0WaMDa=1eXQ^s47B)R*t$%#rPyY}q zQn*8&_LlS<%#zIgdE=3^Fz^PDU_GLlAHQ-99GAf**1v{qA5e+~q~kfo@r^&}_a@DZ z7scPb8<VJL1cs<9(~f7vjy9EEwndAol&-By?x=ERDE9Z+3}$LL$3EUyjUd&$xzx+j zUlKqW8!)llY-VV8ES`b78>{TJaXXR6nwtyl01M5nw|YG4nq$2l_G)kLC40^NH?7~x z2;?-UkWYt0gOdGg)0mJnqJD)n&dca2EZoDiXeOvXlD3{3Y#zCIRHy~;eah=-cylvC zrFtNGOgx>=+f31xtrO5^X9Ym$|A4v@mY!M6eF*T371EKJ`ViN<syE`woxey>=A_w~ zq1a)Rxx7-$FUnlH#IuU~c)_W>X6rOOL;5i2sNmLX(V(~A{X1VyQP~qX5K~?v=C?C} zSrll3T9u0P^8Giy-bQmIn%a<3{lRFZ6a;Z@1|1)<G6W@0VghaxBbO94WmqR0DU)^O zS!r>4Z#>YwOnUk4n6{0FMo4g5AZc(Q1G02AX}-mL^D9;Pqi`14s%Go;Q^5)>^VN%e zSVQESx@~mr#AIWeHEsrc{3Ve(X;8kw3fkPac_lh1%{W203A;}Sn3+FapS%~|SF$x< zaqHB}FVYyW@{-{E{Culj?Pqi-&v*)k8~LNbr~h)iwD0~(<So&zt?_ejUDl?Y<fq3g z@7TG$07>RmwiMag+iNo_DkKEv0B`%8=Y9#u5eOAGW$OLCGlf$?M8>-N3z002&x}hH zy$V2Yl?ni;wrKoVX4KbhfxuiX3Rco59GDmg72V$1W(C}Gp)>yyr5m_fMa9>R1W?3# z7V`iQ{D0=qMc(`rwzkp}ZvnXklv@l|qV4$n``$kA`{(x!mTPB~Q@;Ry(Ess0bOb<f z(MSsjH2_iWMLF=9Isjq-T>sZS$p2M7@^j9E|Hyc+)?vkZ1`!*mYB9>i0L>xj{|BN8 zpqZdM)n6~T>MaFjED7rY$z1+pVmP(n+1QnCW}UgOxgY!ZOm$YCMLw3RKC8>XqT4V5 zz`mI)nmlsZXE|&DAQ}$j1SDBSojJsyE`FhW7Y@J~zLqeLL?XM8N<ikao{f!7Rzztf zg8u#;z=MA8s;LF6;m0R?czQM>*sfgk$dClM{_~Ad()h>-{k1PdOZl8TN*u|Xtl73F z=$q%4xwPlz=0?O=dB8ms0299B_m7cM-{$e<L#Z-t$3|!%Uw;2*G7S&S{v#l2=*rE8 zuIKfC2ncso{Os_VJ@g%VfnLn_^~o6DwDFS;!8`%?adF@wHT*kBy)7q2rBd>2l9ChJ zch6K77&<}|HS80=Sfub=ZV{lVV@!TA-+@U@<CDj=&LVCqbB_+u?CY!)qKx9`Va8%S zEeND`Z!-;l*_KJ|Sm3Sh8Ccxo^nUODuTyF&AM^b<4`EjtKA@C~6OF5Gvq#HIDl;Ba z?so}5!KXx<wru5J9&GFbMBstUij3lE69WZ5n9~0{TQ~SjtT}z%R`<)cGQrLzR0~*_ zc{_VctTF%FjTgbgB2N;cF1%Er?f66U=-L@eoEO<2e12xi0z9th1W77d8M7jhav4B? z?rl1!Zc;?2=*~cJ1I0~Q_r)vszXFft&yAP2IAIUuFBrtIhnD6aq$|it(Ws=|m@EVi zqIwq0TBGMO9P)D<7&GI+=1S`RJKf$dK($-87eCx!6$JW8lgoX7zn+Q!1*H0YL$>9M zw~PSbpQass`gm8F^(COp@6|-g-YB2%s0FlPLv@PbiS0We^Ln=@J%SB@;Ilp2+kqC} z{|$KrjKs5ZodZaTto^M-0Czx%kY|wzK#TPmIQjqI|G&qFkr2{FlxKya?3p-#UgkaM z9hV9C%KqwXr49hF)G2uqWtJ&Vsc896W0}ru+T!#rm9B{Os%*xc^b0z{PvbbxDn=%L zxM`HN$Ra4?a(g@R0?9E$?LR{-`F$lNxr%4lbM9Qu_xbhIm%dMazBsG!Q=B2jtH-r$ z=(h)5{APdzJ!MMdde%^(-E@(`nT&!&kpF9)j-2~F?eRrFnapQ*o?ZEGG0OG%Sk1FL zi+esfbUK$?p43vFqC5lS7Q_+v&g#BRIQDN{u}ns!OcyE~Q2n>k1gLhHQsb%C56%0! zqzX2lx3|tp{)~F9mEcnrA@uszB@%(!6O#Pl+1V)Heharjejt9ipM2-snPU9B`1SEd zXV7<9z(#;T_oMIDQmJ4)Uj0vE1f`|`^BB-gje(vO28ozmXWz+D^}|#1{Lk7VF0DGG zGH#mx+BzDSAH!pBmt5m`{$F1oRc;*~93d@H_2y@_-;a#&4nGVZZHt@^qCd-cd;SbO zJ0k*Rb7O>2>T6*3S+<A<4bL!s;KS!<XDL?H<tWcEUvAdBm(P;-gID}f@*A!}cg|9l zNs}iZYR*<Sfkxtm&?5j+2wb=BW?sctmQ#W5xSJl;z(;7MiPc(b(z`RrWMbZGbG_g} zxgUV2a+xi`ttJ+O>S?&Jt;<j+QR3T@dO}3ztb?f{=;x6l?Zx-5tKr9smvOs`s|np) zSDUrnFzEPyqgk}=Zwgk55*4RkG7;Mq+jjzP%H0b5S2QLBo*Oig{@tfa=6BBfdJV*i zJ)6LPZiuSa@5Qy$4*EM=pkSw}d4sp-$7YauvXz33Z)4o>0L9V31pl*keDX%1pZm(T zOXg#%;Ptyf+v`26nM_8g@5X@bJ41$8+Y%HL{qmFSqkv$dalW?4r>C&JWq4^@YTb0( zuG>N0BCuX7C|}{$R#(+Q+o|@1==wK_3S`5pe9Z@-x4_!yxwj|UT*y8WyoPb^0WmXr zm#Nw>w^35UEuv{XkDKSA4*WLKSWl&%1CZqB00Yz(|8mruN!w!@H%vE}b>e%ZjK@95 zslLRe*}sCiPAuH=GPX`nZL(}*k_<`<wn|H74hsah>K01Fnggd14z4qbZdD1rVDg@C zChui-E}mIyvP;kkevB~Xyth`Qr-Qd)&YKTx=nKRy|1q)IMiM#*TqOrGT5-s1$A(oB z#($-ht=*)_A|~?tdQ^9_UOOxfw+5d`NQ0v-=!9#U-o<WM$A$1^wA^#BNX5-T*3#F% zMHKA!TYO-NP#6pea<|BPWDM@cO(Zw#wbWs@rEC$>=BS#6cj~DUMsVv7d%Mp(T{YPy z+e~bLA)3`#xkV#=f^F0j>uhvTssq4wluh($E=)0tH)^BiJgDnStJE8<;909nAQ8^b z6$)mP?17~`cCmsiCfntLgo&r_csGiwDA9DH2~O|`s_nAF>djA0n>)+iHVTZPRPg)d z7gHCuL+5T6i{Dh8o|$XG#J1pLJri5;hg)OX<3Pw+8no1rWmP9F9i1k<Rc^z1&^B3$ z%Y!eyIx3Ye8;ZNE^4hU&w&_%EsNW1~)h@UYV-;lM4!#%;g>U|f{rrrcUdxR}@t%ZE zyg_n8x}7vE5T86jD>{x$_)YMZ@T_py@wIKr%BVK;N6m}rjXJjbVxh<^hgWJ~|H{rO z-JA2;6@%9c17c${^Z_gKOe^8J1xL{6h4)tl9b0w-u{Tosy`T%XiB0@H5wpmcrqcMA zZ?XF(^4o#`JiuBt;=!5ov;2bhNM%C0En^xP;>O<?hpIAA5+U=BssarzG6x4CEiWKd zUBq3BbZ%bDl~s~!oj<Fwcky~`flT%6VaK1sf9`~Dn(U>$o5gMJY-o7M&*#YKr&uS| zYag!e%)2{N^1(Zq87(i~ew*51*Kl7_3N!`)(2HjXPu0z=G*~Bh29&h+>6D`_xfV~4 zU5@u-o*qQr=66+JVD^32cC+m?%e1J%Xer=bY@nw3-&=>}5NpVRa^I<j_Qqqkn(>H8 znBb$#hdN80@}zqwWHTwzFlc}+V($fZ-+X$Zc(aEU63gN>$pW8Xm6~4G>#9%k(bv?A z%>06}`7P|f_2*CT;p#*HJJBxy{CLh?GpnoTLt_+XaH?)HNW74HVd^hgMMisVV<?es zQ`&p6pnKOBCruL;qV&e3<&V$1SexTKG0g^sV||GB6*2Q}f6x6#*kwLVe$ZQ)#nWei z0oZ1t6fbu&_9c$TTr}eYRuA9u;ixV4EY@VOsGQ~Kl@ejWN_jt2xKaVZop5!8Ur`}% z?QNa}C#b+rGH^sxU7-WF&5lloeVz{|Bso?}K=$<4RA*k}27LZp_nY?_HDMy_r>^^+ zEbnv^lyG@CVj%&QL~vNycUT~EP}+^ieSMwrv1vTh<`50`Fvlq6QFbym>tK$J<~1%d zL~hG??k0>dp<ut}Q))T6N@{R<dNSS;cHO_vlma!lj;_cbL`Ff-+T|i{H7r~QLK>7r zX<y`eCtnMnGCe)^*#%02c-C~(p7QJNlPPIgnDZ8~T{Ix;jklQFlliAxs8(Ex{6w?V zkfD1X=@%X5l%~zV^FIN=d;p2W3Yu*%9Qo)Uv>90Q$l+R~?3YdvKj3!>@zuP=uz%qt zdGMdj#I1dwz7%*}CzGpuN&Ry1W+q=QU#xpV?=50~CRNmW2K8vNR)P#~rnXeKJ4{B$ zV@7&fvEz8dGUuPOR-!8`CtuB|dD4(mc){7J<?+Y;mK01Z%(-@np5F%PP1bf=;dl{m z*V}@NWRlvPVdA|7lUWI9P)JWI)#%eJyneu9Y@@8LJf^6ZnP-8J^G~-v;BPB@1RP)^ z19Rg-+3D1#2K4pmWdhpg^Bjmq)%^bnky<{*sz-T2B4(@!Z0h&s%`BxtZhAA6;6AnF zsE&`8wR^v==`KK(hECQdGdLV>ZxRQ{uk!^Z=^2n2)nJ{=h#Z4-u7vVR8l=sP;(o?- zUhk}^9!BRh6FlW7q<`9>=zX*br>a&Z%pVN}k`q@mwb-aSz3O+i=G|wjPqY6n53u&6 z`oB$rON;~-&Wj#3WH^sr)niV+z8cduXnRI#)EQ8}oLp6;t;r-9#~bUd$$r%U`{VZ5 z^^DO7OdU`7Lmi^C(VkyC*34b_9u1SZS5s76@G9*)wOBJ@6tfr4c3tXg2NEGkx8R<5 zAVeetYBnl4>^WmH)Wok6BK^LOP#%QU?B}wT;vF<vn6}>8V5YZ&Mf(TDoK_s#8;EoJ zXWbOr_#3P5=8I3N@xFMUZEoxusNwm46W(oq&f~hVe_M1n=Qvca;hA{sUnjjvCL2d( z5Ez0&gX084wt}xQ?En*ZjQpmFzj(H~flbowpnwWodV0Ano0&y`<q!jAo{bXWEe0s! z;$_d}AdYh&rGDRv`}F5+42~s*cI;y8MJ4e=AJXDVJ1ySB@5<~hCaNIrGNjvR`kuu8 z<mWNaL)GQoNkQv%l&;(!9Sf-qvYMrayk3p^myFfax$m&kxxqYL4%?}$s5{oU=N%`Z zQ#QSEVN7a-=g<s51Lw_ni=4jRyqCk9#a&pbYB_wrV+7du^q%eJat9D=T1ml&y19x< z`KeUJkx!*rgBIDHH^n!BS4e<YqiGZ=q<@O^TGs2`21qUr^+(yJlPXfVkgbyYb904i zeGK%Tf!bC?jcArS?|X?A{RY9%R!MbV|3(U#gzu51zKwpdASJ&-EVPhz`owY@X~t`d zT^!f3=EdKbcNd!be5)nvad;sU$R!tux_(|vnw=U|Z&lp=?MreD-`{G{xp~g64>&xo zt7?H{WvSCrTRcCl3kw2vMfhQ_iw6$Pb>?QXXdH&$a#|nQ3fei<x7g}`n^|9XPkcC| zNK`)DU}%{6QtphVuC=ndkEXfJY#Nl11~xxlp-;eertAdR1wXY4-dj58s5YCio-lk^ zDz7jIhnEd4#fW)MFAlg~&?yaPARTE%86_8G;^wuQ7CNw7OL@n14)PN5F~?KxgfLxU zS8-qfB4)$FjG+5Mkc?Dr6AR4S9lH8p6=pb}c>Rv0B?MyLinmO}$>7la8;l<TYai2M zCI3j3pWX%=`==o9;RqKW1AVGj`rdB0Z2Yxw(gX?OX_XnGGHyB}J8Rp5TC9oKp1YmK zkyR~ZRF9vHY4XzDz`5t`eZ#jC^Lo%WJ~=v=fijVu;;YzDgl%O>&?s3UUv35+==FHN zT{MjSaZo1j(_4!!eT6x4{xOGmntv@LZ>AZ5IW2QTP5O!ry4%`BbJPLrW<O{MzZvy4 zP87oS_S&zz#-`cHe-(2${03BG3xk$q9w`e~ew!xd-Au@<8U;T1u>D!3zILq}5@WD| zt6r&ALJwRnq^x{!OFZL<1^`UIJwL{ol^YmjAYfh#tuphMx56c{)%msFaFZ1l{K24u zlr%x7kz^EoZNupMw9Lqzt9}Ed2^(2;-w-x%2%8@Z){9Wrpt(|j<f@qQE$#9z9eVK+ z?)+SHr+2@(ro(T$-Oy5e27@{zaJ1jEDcS37&TJKI7tGQ|tmk*eQo(7b|9$$0O(wNT z_&%@*K}yR#^3F3UE*=sjM80t5+iy0sF1HKqoF*`nY9SY*(Q#<s%E$W$yv_pFg)(aa zrmJ?BK#^1j)R*3eIj+eps9F^ZO^{Vy`<r>r$fj=y73t1zLZSHundZ-Qk`~s3c#26@ z&S9`+mMFzjF=1FyX;2?UrQ|BBb@FDtEjh`)cWbqEK{XTI{QIyg!0$CB4h+`TUJF=$ z&h7j7;9ruaopt`<)RB`<9d6DkR+r2h(q2*SJ4|srUAz?X3{p9JIPja1e=XUeKF_dB zR}kN(jW|d*H@mN?P4bM1y0v#kEX&yz;f?~iy&0p#?g3A#a@pkBh`N=PqKH1(Aw4@! zlcv*MwVKH4<!?Ag=M&*iy-S@&Z4Y8jcE8tdehbKfsHxvyFN!FH>ko~w96Fru6?{B@ z6xX_LyMp&?TFY_&#i*mB@^Nd)jhrf#*((O&vl-lLtumzS1iGu{y4RUB2j*A(#r2;b z3U8e;u+~2c)Uv1SF8?VI^@23bIIBiOQ;qWE<+^bS#nUhx@SMUCvfgd$Em3<r$!n5y zTWnO$ifTOF6>5Db=45m!Y|JKl>98CTfY~c6`Lu@Q_EL{vW41DmCHN%nZR@V1c`-xU zHMAvk!b2qjl5OzeY4?#9+rE|Wb>`7~v${=wpKhT!%kUh&9+v;9=mQpEGmrStP&Bf$ zv(o}NaS-G}^F=e5nb7a_Kq|AobwxdB<{scJ+5InM_sdyespc^VH)(1OoZS?^@Ojmr zV!<z`p{5b5?@LBs0<lU1@&S4amWJOudC@XvwxH*(8&qJLmCEx^2tM{}Wlu3Nget>? z0VGlhC<G|w=QfHkGvd|OB|4#MqkWy##4Y`hPL=iG60raIY;R1;&l-8tO$eu2lb=^U zHj3nV|2z;&{f92!O!cl>$7W5~N1MRLP2hvlHcb{6e(I}_b7d_XJ<2PlKL@&bwwL<- zRn77=E8T%D`LH{nJ1<87%LH8icW$q?$g!Jy44s;mwSYuDstHgx+AE<EBqtkC1(eR; z`g9FaJ{*f5$3+B$FH!54`PWL+d4x8JBH<8{|CiC3D8E;*@1G>}h3VnhVr{f2)jne4 zt<w);ZtCeMA%1G&#|WrwT%FoUl!rxNE1uHvCaD~TjAmjz`f*T_DQsBR;0M8oA48_9 z3(A&`Gh^=B6$h-3Z&@*or^iQ#Ihr+9fj4yge}yLw!8F)&gT4Kdk_FAGPhUOQe3;G6 zHi1ZkG9L&A*1mhNN@{U#FU65EIrDu6GSAg#Is5iydTI^`6CSp{ovNzE9`Ff+!OCWt zSRKAh%|QM_8QOLuxmV;5kAR)s9hCv<m9TN?>|fN;QS;-aK3m7@r7Tn=L9UA16M7ZC zybWIMZR8n(x0zXWgHL4Xato$0NQ&Ki{|_?J!ZbU<`Nt=TV1_o@T%kEEs^h2yy-dXF z+Tw7U<qpP;&Ljd}GZE5_;&5t_e?*%#yZe_0k4`C?&!ibj#uzni7vpJi$+xpHXi2Kl z()EZ}V{5e$5fP;&l%tH8+b>?j3a8f%5PseDUhbHlY43?YM7`1p@zBuEgfhtSU+LCb z?BC3?02`;>3>*E9>@+a|>IJ=4c&;ng#!D!BgX!yIHh>ZI+K`Q2FFlPM-y!#lq4kaf z1Sqf6aTk`EsoYdto=g}Mf=xABk55k@S2)X?Hoc2-^syiBYoU0J{BR!N;=01iStw20 zZX)-F{GqZS#`Dtqe%)rrf$2&_yX{kLxI$7nw8_2_R_4I4%3oSk^neX*e0@nt+SbA* zhy2tsIw?5#%~XlFrXVLaxmNOsS1HGjiHU83>!Ar@KbU)b$RKiw|8reMP+^0g=9kb{ z4|JQi#xdA3d*4<cJ-6$oZQ1Yka5E?YA?9L3(!NfG(EN`UD|JtoAtDV;pV?fswb7eA za}07_Qnw7g9x9M2Jo+OL!zEcZ>tfknvLe5qCz!S78SraKfe+?u({8<>lWss%7<H1z z>`&9Ao9H8;hkwnp{@~CP&M9s0+}m+`)$=v)2Dl{4PnI~+m;Z^V2Lw0C1OTF<48PN5 z3<!8@xH#I(OuTKf@TfPbJPI#vXVf-ou=jU*tJ4bDTW?sozlss8T<!A@Yvr#UGB`>q zcZkD`KVNNaljl0!%nh_!CC&#}zfhLAhpZ}_o2_rsM6%po45Pu8L6`+wwpP~-E-gc3 zUMNo(2n$gYb5_|HDn{D;Mxvix(32)l<TyCMCz1paW>B_ptm;(nmLAiuJPQwTs%*+- zon}A_A`%x)3xQ8JaN+rJP&tNaQ;q}K@rwAdpG~5>fiy1tAzwZv?Qd=dUAl~Tja(kn zX2ZbqkHX_iM2A}s4kA<!?L-co2v3g|RB@z4;vvFrsIQdoV}P0WcYC_S+}s5=?tseh ztz;CU><ZEUR~4X|up-Q6zka1X1R0=NtMiFH>u$@84%hL9tErCv7)tIM7a7N=XjF%I zeUEHX*`~{LSoV<m6h_#Qe^A0dM4^zppd0#M@8?M5fX&~yn4e#8Yb6xM(QN;?s`glG zsY8L+;!=HU;ENJ3iGHrEsGx0KqgoRt2Z9dMAug3~YG9&QUvA--x^+UwYlZ6Qua6Sz z%rCV*;#L6PV7z$g_^$lzhPgwM@|ZfMK<h?cR*XCYk+A7LDEq8lc(8HZZ%vde#A*8^ z#fcCU;_)}3pdg9;@Hqcl;4HCJHEv)Y-%U>JTw57U`vKXm5PB`n;$zcXHE_$SHkw34 z%%O#iznhvMJj5HNB}}-qqtklRwnuxU<|5-hQ5{ZdU&vM2`B!k7_86A;C|!}AmcB=* zyhzS;Tiw)}-n~k?U|e^)x{vPC=bGKs_8N34`Y1Lnvg&cVLXrHoYe>^r<_??)7%lSn z>Z*FDGA+|l4JSfgRF)G#`FVbLY|NcRz?jrKdwP3&d!Ai$&AvKRYZC*x#|3y;U^W^# z?_KuiUco>GhctBIsI$^wA_DI@<Vueo%*6-f=9U*O`2V(?G8!*L`z=)ZkFC{j@$#1h zIaEKSY^Xy}$|Q|8YjI5378?iN!T#U`jaJue4y<a$Tcd%`nF=Q5Se(lL`*-^q>35fR zUAn-`f{f9EVbJLYKe}^lIi1(rxLgsI<?QGr6Xdb(H%9O_4nWcmx8N#_)TA|o<vANA z$kyW;5muyhUbKk`TC?@^b5B+ER>8U{&{P>QrxIT<^kf!y=b}Dq!Q4n|e1&1NjVn<+ zVly#bR4Sv`JR40Xy>@rJt#<0*Gn9PYxZ^psH#io~;Owt7)aEQMySLWUH8CL^-BLAQ zbBY&M%9Txa!CoFg-4Z!^x4?W7bI8FT_XeZ1Lr`0IdY;5wX?SYFi)LdR;+nH#inj~+ zFNM}@%YdbBzxacF^|MFmk=)p6O$g_4adK^I^<2&IfbjkF>xa1;<mYK?AzlXu{LE<d zx+erhDhXEu$HVZFG0t>HVI%m7cvzd>_{Lc3gRXMy3NYmL9y#v|(t=4SjhHR^`A5-A zMr9S>w#Q5srktx<-Q0*U=N0U93pwYKcLhEJvoka_Mf$o`I&C)~&wLakpXHqo%8k;i zD67BR%Hx+7xP(!Hcn3)fD#BWgYNqTf4JEPuB!8%jy4-56ZT%jG_rCld_g}8Mxz@1? zBf(?r-^0yZNzJy_smJ3x8^<UVylp&vVIQk8dD?&A^NjLA$_uxq?p=l6*=hAePhe*1 z;eLL8a5#3h(S3a|qxI~PbGx&q_>8uu)hlp+*`E)E=iuLG9Ptb|IC0R|(b9t+u@|A& zq4jBAnFo~hOawZh93Pb#v>U095mP@Ewb(iqq4@%Tw``IgAXi6>W=fsqsyv15mSZoT zws*F{=jl#Gy17vkM}(WLTp9RGp+@*}P~f5Xm2*tN4k*eeVTHU@YZw2BFD3G`DIxll z3F`%UuT+7G4C`SqjqxP(!ynnJqH=+4BLR3lWlf1_CAd)RJWWoi=tk46HK$(`p9U{( zZ+HQW<3m7%VZV+OIUU_e!j6uNRT!#Oe50Lt)D|D^?4Pr#w)ltC;QM!-{A4@E?+xvB zQlFvT%4zrUMvp<+v2;XPJe`+u!TdpcnDwlZW~+H+vzwd8WBz70KaVQkwzin`{g^sG z<$<cxL{x-8rUD$gPzyg%#0`B2z;}BYTanYDk-^yhNclWsNqE@~;^=CsjYgaKqp(oJ zmY0vMYEChjBz%SX8lu%g5nL6xF3sCkM9JC0^Qt`$QpE{#HA3bxCS<0hUXbahG1BB| zzaej5lJq>IA;1sSk;;jl=c(1~tiwTXqZQ*X7_o$Z`T1?zsO+7FL_|A%vv&74r@o}R z;5vAbjFjGXh>Dv|eEa}l<><P>$ppIx{nY8x)BTN&nbe}qrE2Wzgn~-s{`G1T!mkJm zh|^*7s_Du!I}M=#I6UZZ>Fca{dSmfBw;}Orsg9BaIOoN-Yut5kY@w*0fGPQ1myFB; zge&bfEsGk;_p;z@n8L>El)cwNQie&!kZ<q!A4;1}eKpnAk8{fGfhiYxe@Y#{wcjK~ z_NA(q&V_Dm{txv`IV{z!ekjE~wLVU(tjm%9@2G#V>6Y`wT#6Q*$>YZF_nL~k=2w3h zpscY{ruxHrkeM~xB0CWlMrHR)OJd9S*Op{EwRIq9m&sCx{D!|xSB7C(DFw@!varX! z{IvHCD5_37|F|=wEOy9E%#$eFpb-VjZ3<~MxF_R2R}Vjo?}>51q&4kAYHHvsPdx*% zNtK8T$y&ksZ$^gy8U_(LOkNu8y=%+wB#T?|gs|s#^Iw}4mtv=l$K!jKS9$78XEMXX zM^~nMw-CFlg=0`1*I6y?9mvit#OV)=7l*K5d5;fzg$0jak4ZY<nqHpIDk#n=10N65 zXE|A~DZQn9xb!~2e0R8^TYZ=4pc2v7)LV)fzjVu5+4HrVQNo!zeGW1=_R^n>QG=Mh z)o85{TnGQPrrij6362?(rF-!{wGPdn_UozvrJrKNWsah{ENFih$ub-NV>#Z`B`>?Q zxenNkfixe={#17jj?a8B#8b9`1`aA==+B!OYxcaDa!fZ_!$IAADoWf-?eP}fZcF|j zys!O!kMuTbF4H*O*k7dS&Xm5fkp;H?kcB-hgTJJ|6CR$w?Nct>*rjFq&t1?kCBU3L zTBtkaRZ!cy%nq)J`3-zG`^G|5o&6)Hz-lB?pxsd-LQ1?Her{SI;8m@jfdrERhh~4z zt^bX^w+@P{iS|V&$(N7>3&A}QED+p12^N9}cY?dSh7e%z;0_6{gEP1UcXtc!gERQt zJ>)x8_tm-goqAR0zjt;~wTs<7y?b}BUcGw#mJs$tMQXd{(VWTq-|BU&xd7v1C@16W ze+zb5VsI!|?~jhZ3;HN-9WY*f7Lpb4MmP+zDp8;apZ4M>&jA?9H3R09A&DiQRulzg zZ7lA2=t>)BU_mq{c^wbS@DB?iI!ZP!E*gA?fr<6=oVJSkO9gr4pkKZdAR$4ol8K5; zrl2%#CTx4*5O|klzIR(aTK)~K0?c!j%OEr%UxRPFX3S4SB)}i+HpgvZXu^|}L_$cG zpWo91zX>3(<#MQItY5N*!^|h;EnE!eej`%v7EvgEs<K;g*Nr7M-LV)5y++fB`6fr@ zDEB)|ZtTb}uyBs;`FRXEuXTxFjE2LS>)7p`bH7-sLV^PJShLeVY2*FgEylBDiD5(1 zP03$8aL*##zemkHu3npM3pDTV=$MJplGm?LB`dJ5z#W=wP8;aR?g&p3K`FcrflF)V zW-iOC`(F;YoS%|2IoJ?52rpIr?9IFuI}wOYO-+rAj9jiq?b-AEI|uRNWxhsMZr0Xr z#Y=BIu+6O0ds^>C&6|Z^Qf=ZHI?n8;5w<{bxFiQ;F?5B>b9$e9IIAbhaiOaE)U1Yk z2M!t5cqCOXQB=Q1FD@>YvF5Z9{Gx$ycT(IhvhF0R1H_qk80`<a&b`)K+c;&ctkApq zdJ6}rjXHWLduD10msH8~E4evmBI4tfy$`hGqf;#7a%N|9xS9p0*~;15b?owH7v()} z^m<y^4JhJL11#z0Q7hmv>s+Kvf@%xH<MNsc?pytaDE585R@xHhF)Fjkvq^in9uI|C zMO>KYkGiXj;^Xiu?`w}wPReWRsEjDf>kus)?53emv`Ei|9XsX}1KW+1>VBjj7B_W6 z#31wJdkcPj)P~h#N7|R=w9B1c^ESdt8qTFTzlw|VtLt(GQj#Kx_G`a8kUI_T!I@er zW%{>@U%!j#FdhZAn52<}Az(w2Jr&-;`m_-@O@hBm@|e1oGD<@$%Bu{f?`(J2+giQT zhSFv-df5Lsn>suMADFlcYI2F>bci>WSpgmYX!B<G@^M=rDiKB;=|M1yyh2@|ZspUe zf!;u+v7=r{)TpVwGlVg-IHRGV?pJYP@v`gX^6dDEU@hRm7VGgTJeRD1Ct>8Ei73CY zSyz;*{#5V%SgRmPA@OXIIYylFIet8Dt~edJZ<}{kMvqEU*Fo1CfTM?-#N*`7V#ce^ z&ehcc70Co(rLj0vA-ue85(VnWwVGdX!=3%(76lJ<9@Ogjbeh(9XY{fYnx4>L-*})F z*`slKwR^eCs9$c$<bI8M(6OxBpk290sb6mGvp%4ja~?E89D!HN{a816QR8gZxlmF% zj@G&9c5ru&yLrXO$>Sp6chlj<HuuR?a=YP7!%6W}wcI|&FuY?#wOUh#l4cjXq~S!Y zW#W(bLh>vv%^onpXA|V9dka_lC8Wi0AhjraLIumyR@?wfmsv)+9A45Tq(I5Oq1#Zq zxUw+T`g$JbgEqUSgOx0E+2vXI{F;G923VIv11B~Ip1-x79Y!>B1^GEd?U<GU(V~Ri z^{}mh@q!d{$vVq==(PBCp3Ml3g{Mb_S-p!z#I`rrLs_n*qoDaj+t&xnP`OYtm;|e9 z^YEc?rMs?jWY5lGx8l^<q)Qr$|Lkky(^CtTkR@4ah45sP5Q6kZzlZQ3tVr5LYfHf- zn=%}k>OU0*Ta7+bup@lN8{%0@&yRo<%zKLYolL(uy6Gyj#7fr&i&cGcvl%u6sbY1# zX_&)3^Ew(F98if)D3aSdy^8_nv`)KmsIgV<5QE{@4yKQ1W!cWa%>W*J8!0><Hal;c zrYDr%qHM@>T({E+3JNxrU}0%bDJr%hS}|Q&t{6fTF^#6GNfUR8_vdv*tO6Y>A`Z+< z6VAk_Fee`o@6VLVcit?MaGYRiT05DESg8lVYXf&g@m_T9FONI&Oc@K0D{CMBWFAk- zB~Jj`B;20&_CmfLM}?xyQiCwxsj9|j&{1amV@HlN>Hm;@N=4y^wtZGkesN*`x$R73 z&YmDOrpr^$rz{)rSEoODKf{%jF228e<$Ma|94BEC9##HK*0FAjS+=#r5^a@@EQx&N zo7g+;u%>73RLkh$WG$H;qcYVGZ%{(h>pytE^}cVhjgua%pgmz!9ravd`{;Fu=a~2+ z%L7q=r+Z_SQrA+Zd&S%f@Lv<nvVToA|J(0ksRw{D;Geb;ugm#b)PYpLq$I<IAMv`T zQgS$5>u?_j3ZnyJe~XQJ4-n|>bHJvv<u|M2Wo7|;8?RXX^RbteiyK2jO}d;7K-#fh zPdF;*YX|ZhzCJd{yR@&bt130MsN<wI8lagA|K0D~hp$<A3Bw10LeNeCyVu7e%3_ti z%M_8KSXitkxG<^Fmr7>1k-ucGgXH!YT`Ipjxry;VXI~JjgaIbeI;JqPxPL4r6^HPw zstQK!HnvdO{Cz4cqUqU`5OdG#o$MveT=we^7Rql2`ZNd9@JQpA5MhGL`Tfg%@pC9t z6oT-}jFiP$IU$>-b@_Ocjf3e(6Xmwb6`f*w-bvpLyq*P@pDpwn2kY$VTEM&Tq^>4^ zXZ)7#6F{Q`fyNAkUVuvV5-Kctb0#B}JP`Bi=Q@GlJiOhc7@+%`dJZBsDkF=x!pxtD zYfgJMNhvAW#xE2mYWdClJupe8HEVVz6AL1U0!53|%|>Tt-%zg9EY5F}Sen7`un2b= zg!C4ESfAUByXDq)B-9%$-Nm02Lz$Woupsfi=z4fOtem<sonKS<MK!KV3EzQQ+H50Y z-Rcdch^lZw0r4RHtcxB3Vzo1YP42~e__Y%q^py~4t%!%=uyJzjqmz)YEvR^A(|kEl zhS_YHj;5tA-Tj7sHpyB>y<#G+Vt&NK;-jd0a~V|4atPPDT;uJejM$KslxdmI7=!t2 z4o7iCg{r7UvpWN$MVp$1MG1IvX4XPIo{UEV2pH4DJZ&f=DLLf9M%!c7^M)SyuguuG z!b-lL-=nR1#m(x=U=og027Tj_=VJLy&!%PKo4A<MhTm3`u$;nw-UBq%UMCM~`KGTE zt$JF~@Y^?h<0=Gyr(m(Z$^&s?Mybz1Oe<WWR6#F0@FrtnXOFYw(+P~yU_e@t*$HkQ zRnEk)zS=qX%c^Z)Dt>csz^bHfTBy`Y*#er>G(a?-F><Y*N0TRB(ZO-~PFbw8^01f+ z3|8QjC7X2T_eh-@Qob2-h(<{n{9Z|c6&C=E*(<=@TCbI0XP-c&SS5tMz(_{672-4r z2eL+vYt+1!cFVl*`C&%6f`cU~J@hHxVtY^8Qm(r)J1coisCIsocapan)>D}Xr?+Hy zFRd<i{d}y^C@klDBJ>HU<xvarQ$H-%=%^|?L1vZ(e<b|S!;?XU>gFXy#jR}Z?VSn0 ztPjRNkA7Cdl2ng3uJb<n5K{L=BsgKpDBH=~;g{PhMs}|aA4KSIur6z#eaPfop}N4! zm;J_fv(G_l3CLIu{=ZYI7@2>q&j&e#?JqPoF6_q-rC5xQ+K*zlHNILev)X=iE@xhH zj)CT=E9g@Fg~w~o0QNX#vWTPjq%JPa2sg4FUHwh!6u;uPb8yh7?<$_^C)L^Lx>sAP zeWZ<#8cEN{$@rZisr))15FX9l=Vu@_mhscyjUMq+6WSMfcY~z~QKRM0UGO*5+p%De zgEU?CCI(XlJ@$Z*XywN12aNN4ghUgSGG?8-&(vXk53t!QR0&C2JpkpKCxiD}w|eKk zMBz#0+qqJzqTbm%tvkVfp@*QuSIE7g4ijs{NsyD%sfqvIb}yj}0$~zU5EXs4B1BgN z;4OenpqdC^P!HUvk<lvt?{^jn_mW8{+JFnKs3_kciJ+a6kj?I_9c%$RjKcam4$M`r zOdTc+peBGJAj$S^=b`}1QmWdE$+ZIyrY@4(Bi>F@>jyw#zdy$HwbOo^K_jio{%q`O zsgL9N@mG<zZbSR!=sq{eLnw?%hWpd0lUE~&S2{9PYdA2vZt0jpa<axP>BvRRM>_$u z4ilXLas1)!U1>|&KdkX20)40@n1d$V3LV~Y3Dm<cWrAMH>&e(o+f~lpCP({rM#!aD zI8haSQb2imj46;#_UjSAxFUqhmh4TV%B{M>$uYifz0<o@6;2TtLsS>s?F2lkFB*6- z;FQAeY}e-7MYM#TNB^^U=5CRN9BJOD$5d3-n3A6M<7^%xAl0?$vp2Bd@u!8jel&S^ z2Sv2~bUOUeb#>T8s#%ZwWf=FOtJ`j|gA-#Cn5RoxeQ2Lw{#;+fL)&3C{MIH0m(iwS zeCGvYUKDFCr_X_J`GPu>i3{T+7>FQ$-m*~{hg$Wh_WEuk%*%h9ZG#E7`VpILA@|A5 zEeZw+9fx)~)U=iKX@RbQiMfZU=m#$DH$MyhR2$<1TLy4#OtzezFHXk67mKm6WCloH zvgTx|iPmZIF>0n)g2B`#7Qu~Gnrt1W<!Ud*+RJ<O%d)70!PL|v%fla(^fhARGJ@4U zzpMm!H=a_<%2F8r`vG<9h4QsEO;IW%<)|`MQ7TUqDkJTy^@;<;B*maFt6~n`_yY;> zSfYR;wDrY+%=5h7m*bOre~rNtG?PSB1Z&|YZb8uzCEi;nS^QC^in@zZ)W4p8$D7tV z4eB0oJ)gIQHKiLR)ekjZyH%bAdb-EW*17y8Ru<LJ3|_I%qpIkh4*8ncc9m08F0Llf z&upsbTF%UVmQC}?)fCj9m6z#*np%#3ZoDm2LFeGc1e-QOpQMhZglyP?3kovMs$DPx zlYS+iJ4R(#ThCU*6!l0Z0QKP6j1Ay|0D9LiITzy$%Y^UM!`Jbvd>mI6GBY%|aAER- zt>QAwZ$I%L-6_=PXxIlk8;YmARb&@YeF^zG>AKSBdvjZz&w-aw+n;+(EY|=h_V`U7 zVqE#p<Q6mQW`2>45QN8EC$>pIjKtvm;~YU!&Yg`MD~5+IKNWGPKM-c;=1O3eyPxeO zav&EX2vw@&wvn$7rX6Y1`xEGM^5_(m`9jq}s?2oNLExBS>-Fioq<XiLoR<Nv#}rfd zhP-N?m0!jT<|=X;+0!RIwxij}(aj2+=E8F~xc(UwarnY|PrslL362xrJ#3^UHt4?+ zlUQ3l6yN(@zh6s6Lt6QQ$oQ7a$^Mep3r(~|$L^w6$j5Innl71mwjK|nynKvY=YM{2 zSrws{F~!i`8w6wKn?BUu+I_?+le|BOtPk2ysiE5>#=sSOmikCSTzn%-5&85hFS~#M zi45z+ZS&Xsw#Z08w8eh;Qb{@W7J_UQWE`7gCL$t&5h!wmUtUU28Jw|IM|RlW-Q9h7 zSPI#N<u>TnJVyGT1DvMnD{dMr5bK+6H(lP7?=3(49kt<PF8IdG_XgSt%!?PacN8p! zM5l~<IZp(*t!NxWjD>z6qeg*1BOyX2{{6lWIw+~i^LHOL%3R6b`g<64XJ=7d-?yHC z`v4jMAMkxi{~|z>X2-?rr^_c>m5f*S91vtIW!r+v&9IgH#1G&96R?fh?1a2Pe7nYC zLXQH_YhZZp{F5g`SQrS7m30cZ@7xd!cZn=DjbH!ZfB9`olY7stSs8n;*}iL%5?#Jo zV*W6M;?9p{gcCZ?i|k)Z(Nid&(l_IWxihphu0*!8QvYOzenXagrIX>0$2$&q@v@^` zuf^gK92K4LenvniO!gQtTOWS@Q-X?%c$^UlM~%(HFTI~sn0k%W`L||__*>8XU_|Pp zgXJfH=M-`}a0kv)Mm+a7fDgPFIu^t5ePDzv06Cb?s<)bv*X1`2`AhpfC_!_(H()<( znHPC=n*Ai^q;YY`MTjM-?!Gpr8b>0_SE56nH5hYbp<W^2G6Vu8@Utg{h2+@kVW7EE z!$Xk6`qke?y!NdlMyNb~g%7e0OEb)Y%Q}gKXLtX8{x{;kH1Q(~K(0TH1<1Pr0RgxY zbRFuMV#e<$235xPbF#mezA+A>I~ZBW;*!zhx(M(Gc-T|IosaJK&vYqi%fJ5FWD@YZ z?nW7xrMchw9(QwZ_@_J@cM|-7qx49JaG>oe%Kcj%+KKl)SJhftZ85#_A1I5SR56df zLblV->5_t4D=6LKsb@=s7XHn=yA;4|JUwrssHH;1SNGEIf3By(n>si$GIDfeXG0YF z;&CqQHTjE&*x1;BLmO}%%CiuK1qB5i9UV3GFll`edG`#CZ(QLP&JQ@?10QDuqr)e8 z{D|!r#RaeSRN}Myz6OSmn4Wg>!6Sbtu-IznL3Sg#36T3a=&dx&%HUy!9*dvNGcLq^ zU9ne&!zbQY?m~y};rDj`EW{Z}m^g_4a5D%@vMEPRe1;?yruz=Ns)=H_z+QXrevtf% zY_s&K=%7o$*UgW9$o~!xpAR~=N{OKV{%}9~YFQsT1t+dkN>JtiTM&RY_ntm=q_DxT zc`9_z$kf^aISaT-yB}Lp5>cX*|2?mfeIEIzu7%`ZSBJpa$};PnCak>|_gpE&&!!0# z!(HnyzbZ32zVq$>>t#TR&9wV38)I!ve%5o}^b5`LOBk>*N7|)cWv-GpIiPnE>$r~I z$f@77D!{%-cP2@sUodahT)$7s|6-ywO5%bAdZ7q(tJHMi;-q^lz1H|NDi=Zji%qIF zUA#-9>+Vi-N*jyN>GrOo%w#lPz@jewitK08(Pl&hcYr&MkdAt7e`$yLjb!!JH>b{? za|$wSPzxpCumI}MX~nAD^<rUyrYg!!+}fEUpwYUU0=1{BUAGUF*Gx92F1Fg30V@1! z3sAE-vb(qC<n3;BGewx=<3?lMCoLrWIuw}t6a5EXy?}`A8%w<6U2b^1iyP4?SnsIg z!H&W=+j%}IJ)h7cg?F2P<jMVjy;4SOB_+hN(4rqcoOj}~R4d@1*gK{<9wy!#zb0g_ z8}dO+G_C-#yD0^Cse2;xBzfX+7~-Q;ZKD8{#t8SWUotvyJagSRg6WVWHZji9Cq)HS zLJXyW11Ji>)bNU_EasQgZ!yn8Q&W3=1Jb9ZnFqw^6wuwaF|XaR+vL1W-6u>|tcz?K zPTMa!dQvqVJqo2J2}GJXvN~93fV1vv78LqkF9bCK=y161e4?8jpB5U|<7L17>_E&t z>HNxvM~D<3Yu$wZ<(*yF=3_wvpR-oc5HPqsH}S5L-P#B~t(%gtWh-3&76upQUJp8% zQqIJ^^+xCP7K*y81rs2at1iMZUklW?bBFdMoG9J4+svh^UY*rb_Qla_7VXH9M|kx3 z9K_A}kSzdF_tvxrqow2wYwj*jj+UJ2+&g1$>tA}~{s_x7aHoR2^}1PcNuz=77*OWW zC8M&^G-;iebTK5QoZVg}D5%%}uJO^lL5;l1(6t~#Y%(M@N*vhr1{*z@3#cg<$h^gT zjc}C`6%94-@ZlCFKFrt(zyKBzT`-$f5%O%zVzf%s_T26a4DPP7o=T8r(kI!S1ne}w zOr<X7m$s6{!#~xA?S?XJmfJw*x^xhZ8D9Xq_J#VZj(TkT=H$F-eMrCRRDk=09{gfg z*E~He(B0ZNCd*aK@c6u*M)|{F)?iI7>t+dpMZ>P|WH=w{Xx@C)eHqB{OfhhGe0zj@ z&hcP1&3lS`r)$vLWSR%>>al%^2ntB8{es~Ys6`?;X;XdQ7235u;Hs*TDwtNHIqMbf zQYWzdp(nX7F}oF`Ymum#YPE1ZIP>%yyCKHT#h`HPfFWb9aD1Z&y_c0b{*|H0bjt2H zxz~q3IJf$_WS&Z*P~AgRU!7GK?XWLTHq!!u0Rbb|@!>C`wCdj6R3NACl#`J(SCVME z5j>3gcCZf1%_$#T137gKCq|bG)V4A6bWr_}xaE`{*08Ovnivhuq91xiNfS*OnTCsn zHE_9mq7h<}9Cu;V8DCA{pk%h4aG{+HYnm@xKb~$z=xq~6NIQ5d=wjr%%sVX%(~!@@ zT)eN&T*XQ@dyICHBgAptUvqO$3f}Ej=La73bjEWLvx_xfEVSn~{?ZxdKEhP>Y{EJU zNVimRh|bD3Scx$qc{s2*`~ZZ94V-~n;#-ZC+f%PnAOYTPt@Lawry<m`n<k`Jh3dhk zkdVdQt0HrqiwK-(N!FmKHGbY|B}q0nCT;LdQMcai#X-XxT3(1b)U3R?Yt{Q|Vha)i z<CUDULF{?1=D)zUWti-^rI#Df%mIVn%zALu(94D@eRrqyPCNqg%~>wXaqVoJUJ3K_ zH$-?2?v@}*YL;%ltQ|m5n*Q`cvBMTUFqv5I+^3@AqeMm5jxNe`MmC4|{WAY}VUU9K z%JjAAKrKU^PMx^Xx;v`wT9j<lzqF5-4Qw33gFZH&1k=VDYM?tf+x^smr;gPF2^1P! z(_8W_XMS3r3T$mTFJO2(ee#@;lV&brg~+fK0e=lc%|1l2H(jgg+IV*>*MuCKSN$fx zw$B@Q(ofhLN@objl@=FSm4!C##tfh0U`1A$b%yz_uX`hj!u6ggpuZDL#)X`M!CKqS zVYlzfpfbimDQK&D$rU1N)`2R_JDp_lCVWyon%piGl8`6qGau`{)|i<t1@gBQ9~h52 ziBP=??__50-=LwBy&W5p*dMTq6=q%c*Er?%+u*qE9KT6j-rg>Oj1%WND=S-CWL!$Y zywHKsz(RV&-6V#}<#y*nMMYufHP)uRCedbcJ%}&!de9SSCS~T}kh|mdY5x-<bh(nJ zigRy=ZLTd1*n|pPzBA-)S3sA75R42!!U)-Y@;jcpnLKd!f+n_)g(`w!$yJwI*Kx;h zQ7kQ^D8;$8U_*29NA24rt_`Ct5D4=Dpi%`rBsiXQofJIhFLAUuSHQwj&k?HlebK*` z`sl3A*`(10bp*$tyQfTjHKnP0{xdr9*eXJ`X$M`GbO|^Yd@NC;+wcO~LVbx;8onbQ z3+>yfs;69Wylv8PS)%do;ByM0Ga&4|I7nHg3|k5dF-cq@Dzsy=d6!Go5fUoveLF__ z3{gpAe_0z9qd!ll;dk9yvpWl3qZRWO6Q!Z<*%~mr`ZMiVA5ze?`e%m#E75x>*`Li3 zklX?;-+|t}ik{f24f%OD;}n0*F!#hylFBS>^NC+0)4^h_+qT1%S;m_5!sxq_Yp(VI zSZC!Am$_ss*{MI4Dkj~M=id8u(}MeN=y(+^-aC2ZG6d7$^&{680kFPafqO0-gArJn zM}&oxl;z_W(!+{!nr#Q^gq?r%vM!bpMHD`Mt@UokL(uSM8+gb&zlGIogvlv6-v-}M zjqFSn-ofk4FIJV%pHI7fY_eNBovo_o;3d_`dhMds9o4s18y9%Csjiwx<fPS)Y&YN) z1o^W9U-dW;siD;i5GbUC=B&JLP_B}JXmG6Au-vRNrt?IS?05Jw#M4toJ2`vupF=Y? z-o`RUbk?h?in}`RZ(QIv+245Edu85TM@T>n+i$5BeyoiX^H(ZbLFy^-g`L)14|2It zwbH#gg|;N!T_m*5rmFIjse|8YnuOY-w$ja%PjITz%zHpX;j(+TEoY;c<D5x_%k@FR z-|Hf7;u!5}btAZIa=%DamBS^xlZrQ(V&gFOU92EK;F0MOzaD`E{z0m^UQ}lh$7@eE zjOh10$%N-TBpwU1U~)eops@I`RLk@|%&ZglY<6vfZ51M{MTYRB6RwcmX!$slG_EU{ zIq6eW<*JgwRV~9&eK{qxQ#W1G9K*4i*u!!<#1!F!2b0WPmeHJlyGCO%U&2;dUfpx* z0bO$2I{{zUQFYv!_q8wfU9BlC=k3DJR+}T{+Xc`_L4PQan!R^Vh2ozEgO?MXgwpXD z#&4(<BaSobB2xh#CkRCN^zh^|3%hI|yx5oeEkJ5>F{8_PnR|*7Wf(`$m!^9>6(Oz5 z@;tyKofQ$GOp(^@HJ(7YQW^yxkMeO>dREubQ1p@Yd%Z)~Ln1yvT?%+og1+ms0XVOI zlx06WL@>Z)v{9EBI`n7dU0dN0^X+DFXNu8FW=e7OX#H+=qV<X>!?2R8%h4qvQg}2+ zTav2_p^;4RQSO|K&&B1aPk+*!T&0mzObUk+Vr)asr)lkAvR}vibC}JI<F-w@Qo5`{ z93k~$o?%WtV)Zgx*@qB1LU8;z06$Ir%Fb?HZZrn>2zXbd{>{vol#sW0Y8`3=tL^5< zB`MI`hgqkmr&~gzqElz%<A7iboR+|oYK5x^$WhU%HXi&sJ^6S8Dgyd?IW{qoO&1KN zdq0oj4=g_r89hWXJ^aS1qVE9XMbhq4b5dP`L*hiudMf}o9Lj%52bv-y;!f(kyW^i) zI1t}yR)z}~SZmQ6ydD(x+^=uWJN}Jl;7DBa5TuU}NPS-e{ZFQ;b3GPIR_o3qrmFtD zwBSlMJ~;-wb16HJXD>3m&SnNK!Dy26?G(n10`mO%r+|3FxB(n;T}=~I+Hd;+=bei4 z>jMrC=x%3TD6vron}|-*)fjoZVEwm5fMX~j4+jETXm099ZtAVvSpR@tY5c32{4Z3J z|CP$}|6BQ6o+_jKjVSXjk;O?r0U(x#@%R4?PjdYVAZr7J9of_~JsW#|=s=E;oZiox z;^i|bDcV{(V>DISJ-xbjQt;_OW+sJThHJ4a22=qfv8-ji>e`<f3^fIJz4{K$>K)5b z^GX%^gLHHy+sBnN@@2H#6Ge0zN}6UyenbCg>uH{shZ+u{>8OhN@@CEpO_<vmHGki+ z?e=?bf|XAU43sGnSMuXLZ)|+CDla1<<5vAq^(^h1pEfn_c}~ENy@OLv+H1mCZLeJ2 ziX>;1voMu-8VTK4u&1V5R{q*YnKpdf>S`Y*K|_2jyK%C&Og8j+rlwZKf+-4`Dxa<4 z;-%ldR@+Ni>X(!ON#v(g=DM0n@~vNBZI@3~ujAoUN#$Wc#hmaW>mC5T$Qg72j@+!B zh6@fC-DJsNFo!Z(L66Y5b19E9RsG&aH>2~To$6!z3U$$b;5_?8`78wNBqt_j=BrKx z|C81>aUQ@_md!Uidll^<6<opI|5-!9N@#H-nQ@?}$tux`nyc%JM>LM%xC-~QGal!} zI0xzAK<N~R5e|;w_g&jEgR2VgJP@~!>@AYsFsGH895`l@J9m27`>khCn8Tf_oWwls zY6|8tqXi=ddW`LONGPMU>`UKP)$z2ce0HuWr>M;;qT=^#kY=Lmu`Vcc;oY04n(@Gh z$8`jp#dwL#iTWRR)GqVgcyXApW?BQfgf1jMbu*e#9~oX7nKc)V;qODI!Y47?GUwCL z(X-B*j!sqhJcIYk*8IJsvq;#2j&`z!)(>rYeTxD}+0^VcYOvK8{u@!Hf188eH)k-| zJ!s3p$;&428L;;9%jxM)%NNALvg<!Augh6Q!BmAB;q1K;wx8GWpsU*GKFWF8QGsgN zQx%9gkq{D6L0$W^-MB<RX~CF*i6wB~)*k3=8?v*x$ks2THC3#4`O0%EM;FH@4NIn@ zlQngAUr5SwNhw-cDQY%@sZ2JFJ$1dZfO11&9UYa++v(6G7Wf=yY#eEknW=?EK{3RP zJ=ZwvO6XuJ4!riCO(Lp2A`)y#!#CdM00Hqqm@fg&!4kpNTVN5s#zp$Jd*+AcX9}a( zK-7N9+H4e55?t!)@*O6u9znHB=jz6r=JmBWlI?O_8?J#;|5Z+uXp2u)#9V8ze2#e1 zlKEArNQjRntWjTD#RUyVl5z3myIBSBzC+URm4DwCH99tydHv!Z2ETkwjnp*=cmg(4 zIAT~>)Zpu%f3^0YvVi|>K?8OEE6wMBTlxP0$NvKy{||8dKfv+-FM;E)Zlm`o@fVvP z940jt_xk7lXwJ*)G{phT26f<Q2VDK_6POX+LdG(~PoAWhT{7cYefoccd?($ReQtto z<3yrktoWS`tJ8T#SYA;YJaTz^X>>ih*TrPbQ_9ddcQ-QA_XwkGO%U40b#r7RL|$nc zRb>FLQ%PU7FlKB*rbhsD#>OenG}AqJy9DGBch~7q8~RW9n+b1rYvWxK`3D=W-=8QV zhVCdG`nnEu%&W3D(Ux`wJe?7;g_WK+kVa~o%{~-wIyQO49Gu@ulW~-RGQ-L9u4{3= z5rr$!WB(vEe`tnA-`%q7uHk#QgK|@s&$&q7+q3k)TM@fKB~^l0dsHpOuQHE!qyHx$ zR`l~^wyx0ioek*<2?4v2Ib{LbxQ}Pg=Njg*3Npt-cMltjK97URAaIT`Mi7DG=eLxN z@4NnL#aWh8#sm&b1w5>IT$YB|1{O9KZ?!}Xf8K@aKF1j$Dc|X76z-Es@$@`v8}2f} zyau!nfcCzh$HN0fv~safCLG72U6OBL`GvBw*sj$9*`(+bru16FUgmxv*o3j$Re`S! zlamEy`1#cp#(;wWb^p;dJ`Z``E)Wi{LRA<!uW$Jb!c4-o>o|*y1?%|*^;_Fha?6`8 zYgY2yl&_9)>?(E3p_qLw$o?l(q|L=sQ)U=%{68^QU*<zsM{NfbJ~0K#7zM!N;q+CZ zYDqZc9vcC(stOCtCc(uO)<8Tq){71K-zm~O|3!tsjDS`g`9&+Op0AL-dG!JiOn^)O z-wZkb>HZC#^X&=TubZ3j9=H&2{B0zN6fJ?DRueRA|4@IA{J9#5=ZUuMFlq`W6*7Dq zs71tTRt+D}q9NnF0n#XdYw$t-rZy29<R+mHe#wVoo^<~6>K@rhAXN~r?IFp#ASNVC z>}x_c4l-UkJhXjRgQ_l#izTK2z?6b~Y-}03?jJsUD=5Iss6f{Fdi~REc6j(l+w#+H zZ;H&HL@ypT`io0QESL?;h<YM`0}B8ZF~k0B+}f1d`Y()s-TYwuTW=)m{W<BqoyV3~ zdc}v0maqC)FB5kd*Q`c0HKxvNOVD&a5kc;uwy)|~wid2SBy_(XezB!<AU~=X8s?|w zVNm$-msTsNdH(n(<DSa+wT674*3-`8J`)4`wBA0AfAW{yY!Sa@cEu8K(JtwkCx{UZ z2wa3T!Mx79!w{A)Jkg5Cum75Ww+=~JRtXgM?uCa8pZG4yiV{A4?Sg^qJqyF{VvN|I zi7jU5LC?~jq@2HOrOSMUEP3^5Y%truf#SO;D=Wd_Y|HHj5%K$}XlV#Zc|=|Jg06M< zLx+#A>DuiF|G8e|n_A{kU3zp5r<TjpnU#;49sUT;ze7$IAQflKq_HQJEZjf<2u!`H zn}X#h5Ht^91hO*x3_SZH+!OHHA7(6nVkt21e?XSt@$`Gs_*eU;scRsYw74Q*lLr8} z4?<|LpU6KV3!J~Dw#T&uV%`FmgNb;B-3Sf_fE<^sc^@Gkl^zbz#rqHXV(sXk19@fu z;Csqzuu~@bkxAoyXtb6awtnO>3b-(F{x9^|7YYx0)`^jUti<AC>;)anvwtP|w>7CX zQD@9Ite-Kc?vS+9@^BN~dIaV7FfU8eKOaBc<9{e`>_;6;DoscA0H`${_R;IUo#t;2 z>b5%}H0W3B@0LD=iDwoRRCo(gBFpq9x9cEgNn_imLZLr4a$v;E#1GM*h-hh%Q2ZtF z=fN569UX*dzh3KrM7m$v+1WXa|HY+Ao`LO`Gk#KB)OKTok!^kT2Lhq}@x<5ae{q;_ ztA~7aZyUgV5qF;z3D6|G;N9}C0sr8*N8SNTfhf{lz=Ip;hQhOOrw0-r4*tgAbe$HP zWp<qXfNbJx;Kwyv{2C_5+56c5;T75X4#Hi**Y_(PgcoQABj55)`Wq__v&yVZ8w@D& zkUta){f|?Cmp<}=-VvB)_+LP=OzaFj1i%^Jp$!W+3mjz|9Q*cU6+sL9RXWzZ_r^;q z`S-mb@3o`i2&y5RcPx@Yjsm^rw^DaU7FMn@^QdpUCA!?JeD>va+~;fD&t*Arn!Sm_ zdV3{8{kH>RI}b~2>twlj;%&nF<j{(C_EfPAh*xx7sZI&SOlM7%;%JGECyAhM8{zDU zvN4ShL2qU2APm7TZ>o~{$h9AW`YR-gc2F5`u?TB-1zfyVVqU^W>BU5w?X6bkzyCTM zsTt?l-gQZ{m=x#gTBi-*7m}&kTv<)`pc36p{)}$$zQpTC!>wcZu5PtObA3)zUC$3k za$IpA@n{+TGYFZ9c4ncC!+f*@7tV>$AfWiPu-B$z_XSW=rRZ|cAmpEO?3HD({YkZj zyeQ2ngG(%Vks_4pO<z16v1;L^<W+1klS$1gR^jb3TkQz>ahsTWeQyoxxajhu)&0#+ zA<MPs;$zyI&`h}E6560?`L<|VTxbpc=e5FvR2Nw8y7~?-fJHpVKTW{==c^i2Kq1oG zOP;Fd73EdScP6%4$F-5u8mf2#boZ^*v57q(Z?|1ZVa$53Ub>V@M`N{$0FXYJM$2g` zMHh3|ykYsMJ^#w6UwMkp&2#RxeLj%AZ(@RSWwgYzWaj$xi^3#?1uti5ifS<{yZ6(` zb*I>hcC&>oWH89XJ|En@PmJ9a80wR1_3;H(<g)i>58h_jGf6dLPv=a#;-#KlA(Jeh z8OQ{D*C4pu-T!C?EU0C{`L=nbj>}xDsr_JFMM(dxmW`TOf>u=04`TwD-6BE!ccV^| zDPq|C@E!mlW*8PJr)y>th^1sgP^VH=VLM3~McQ9HxfT{o<VvEnXo*weoC22TAk|oc zUlNZbh{Fi&;KH^5)knDa(p|TM-W3r$Z|N(=1%pRQt!j_GlAX&JoD7si+f6^KFf05f zi;&;^T5$&{=j}LovpQNYjysoAgJ0ZKGP;{w#lY0&z-z}`%spRQ$lusiB178j)y28N z>%E`k{N?vyW=f4JIcLG`EbT%?S%T3Aa=Z>4-toFqi$_;+{WYrGg%({Yxf)t+zxuY% z2C&bt12(JM8|!*Zccx?)z|$hp$KWj}DP@L_hgF94_>Xi_xJI?z1o*t(B8^7ts_KE8 zr@?%o`c5y_W{*u}A<b%t=8BG5@q7)$OOR$|bT_Gr_^30UEW~49!LLGse-3*-?{3rD zYnAV8f`rp)-mThC>>cREE}+<WF%}S*;?t1LtoucciIS3#ctuA&m*=`_$&3B2#Ak8` zh4zt1&FTYds2s_q{Y`iL>Vb|^I3u?;e9)gTX22pW89SD0$t+J{YAUmLl{l0*o}M8Z zZ77E@RHzH;7LI>K0>z}SM(+%z?meawX;Q9T*d;p6AL3o%b8@@!@!>p@95T!z*-N{a zngl0y<?EWcICT(mi7l3N{Jhait>aZtPcx;{vSGofYr@pQVW4~sDUpaJw`PH8XS|i; z5U3y2BN=O>>g<`fOfdn0NR5$k0n7o5TP?~=h?HM+Zc63&VJ3QKw%;H_rlJVcF27u$ zM^6*Nvx+XPVWCzl+EYq2$#5kT2!20vZmK0&_a$3W@)Mi2HVriJo91V(5>@2V#6}m^ zf`sx?o<9e@c%%k!(XuV)>-Na=o)T4vlCGK{rwr6R_G*-#ki=fQX}jUD*x5sIz9Q~N z*F+UJ?cgFVmZSzDe2S1#=(;2{tbVUp<7}WtRgja|4{^9PjDl-4@N!6`<KW`ckllpC z{Pyrcjh2&c#FX1}zEwlDc5JS*5{;^5iE;e+E&Qf*9$qZ?Z~EH#%o1sIR-0{)E1HUB zi3lQ9V$l!c{FpX~Fe$XYju~#6mpG)UnO8p*)VFJ?hEBnCwBua7iduC!Aw91Kv2Mc^ z)b~k)vjZlAPpU<zbVPF|mxxx|O;0Ub&E=IMS>wSUzil}!nJuAj8`OcNq!~mwjE>Kj z=vGEwrvU-njsS4)-P<HI%euQuWf>_Yn#}R<-VPsB7?ndQC4&NJD`Id}X$PI_kK^fG zK9EeM&lA9>n6T_jIJE_me}chpS@9sww-V6ZjU6-CBdiq%y5FC1Gk>vk)l6!({G@VC zx8Kpm4rrJNs1l!P6cwuDvpWrVTb0M`UuW^{`c&Bu-0Bm2{D64OCZxRM?aEWv5AkOD z`Ix!8{mNGDM9@tI>Yi6qNnJw&_RJ4JAuoVE@D-j6OrsW9MLF{|%a#to;81ivpQ*JQ zYkD8LXLi9ghE)A_L5fk*pQ0oslbpK}e)RX+Yd7!LUdT1<a<9g&*LlxX_~vf#T64US ztXs_WAmV)UOgz2OJTw5`equlz3v2cBYDn!7iv23$I;ZChLLn#IgG2Bzj54Y3q-7l* zGj=k}yHoeN_w}#yWez5VsFbVJXL3Chs@g7DxCz{KT7JfDoz(gq41K9rG6!kdudigZ zB{>6qvP%>i`^=`xKS{EI!J&5B<e*nbcL=bVB*S=D7^E(_PO<$Ie00R4s$Fq1_x^S; z&pT71us*Xy2O{ut!}Wu_)ocP<__m+H)xOZ?OtSljs4vRjtK$i(nMRr|T^8(u9r}$9 z&ga4UL0@)X-hM7TnFV`$3_DeiIe6w%FuzCaBsm*>Bqy0@ZsI!fnSEVsBb~4<ny#RZ zTcW$`yg)KxbIvz(cA*3tMR;o0vt=z2+NWMGu1m97q7iz<6Orj=%xR`_i=VYpOeL!9 zcYgmTG|B@EL#u7FPUUfuAk#I`Vxp8nbL%^mDju}Ry!rz!gbeVO_tzLn=FxmhjG?mg zT3D{*1vsEcJ<BUu*yG2b1PtV4RukgPX|zEZ2Zc+N$hqHK&e%oF+s<mzT+y@}vRP{3 z$2Wh%0`i(aI(3q}@qmKCtg`C$T8!$2qA72b$c8>&LwFHOB^(om(+&N4b-OM%(Zc-7 z70X9PpXR>8T~jK}@S9Qzu#Ak#muH+Jchlb(%{^Hy#@iQej?8`*Fo3sSZDK5k*1Lh* z8ygo?fqmS~wu+h<zye)YJT70U<l2vB7-R*u>`#p9#5|v(MGWQ1)N|H0^E7Fg*_5g+ zL&s*T98^<)4euE1EW5`;Sd?P$&@x~-dUw`mF)miJlN_Sd<lI~O&xRZb<oU)0Hjg#5 z|F{t$BQ2e>>HPo%Bvtq0roCGYU!EW*5P=*5a_NJBfPi`KXJ|nDI8@{z3pMP{Df;~O zbti2!E}_GO({5k!FXj_KtcD}?wrZv^1KXWx&YM1q!<_J)-ib(<Iv@ZW2mcl8N?k=O zm|~EDe~@Q2WQ3sq)4dePL#HQns>O5VyV}8hN3R?(%T$%<KrWTKE|*29T6hNaa@g); z*2GTdJzNdlehy7_-@Ms*bPc6`LK=B+x4X98cbjAy6mjPYN1f@(aMI>;JC@_-P{6M6 zX}Io+pcyP>1iD0ugx)Vpe^>kXye|NM^jS@BYFJJ<dt$7%FW=UK+0<Z$2LxqFSE<Sw zqb)Jnzg}ndz6|l=)y@^N8a2#YK22Xs<OZJT!Uk&XcE-4@s%Ia9{-hyAR96s*6++qR zxQ|RyR)!PKQ%yFVK4QD~Xy#U2e5@>b3GJ-#!0VNVi;dJdjt&3W;Px*x2OrI$A9d%< zclK>(%uNj=2Q0!@znkzYu*CwBWPNP!>M3NR+l!g^ZVS;WPPtS0F3skC<m6i*?BOA` z81=eU8ggy<(LQy{!-n?ako|Gt)IE(~7-nuBm;$l%lu-%w*Y6-`+I;8hGIF%vH0|bV z<j-2u-Klt;{7uU>sJ6gy4QE}#gCFs*IC(kMAr?u_nN<u<76Iy_DgsFckmeC)!d9yX zpu=}aU5vv)=Vf$Q5A%&ZBGWwLw$X(+H%;w&{0{%1>14ats~&o}RI4)f^5h)mrLcch zl6jCOH}CQXzkZ@x5pnG(B6qIHaAh;Ec8-7TePo(3>Ad6PGh4d<wHP6nTW^1k_#G!O z4%l8W+X$Q+G@E<u(Kc&ct`IXr<gfR;!hxKL9f+mH+ra9`4Hpu`pef0lGX1`whDe(# ze4|V%*&WM$$b!RkLWfKISHz2=Ujgi@iuON*k0t(KxX+VT%A0;c)tvr~`+0h_GNFyp z{HTh4!l}XqRz5>Bp5r{#Bq|vLDId{^IV)o1FJ}+lkU$(V_r#H!Oa-~99!!zzpZ0IL zkrB2Y9>Eh6!C-Ln>fTJL$t`aQDK$3-&kpk?-u47z+B<`>l`dL{;Rh?Mh(r#K2$N;U zLN|epwaIuP0B!w@1O^r3Wpb(KanHeE;XiOKpFI>&jssNRj@7Dd5!Cg{C+VzaU@-Ie zRxQqMvwm<QXJ~59$3LdsV9d_`xXfimlR6duddp*X@ga0A0$eQCj&#m;+9aD_h_hls zqkX6v+=DRxq6JrXDp5Rw05ucvG7&>Mjd=DX^td`UEKBRQ=WfR{i=ty4eXk!%pHS>s zbKUzrJd$0N#?T-mxcRy@7M?%E5e!x;=Cy%Xv7SZxs-27K+DyNuOxnO|;}GM(8e^$k zAkGC?NZMl<V4m)}<N51n9El7eWlq|NDr4xlDEmOix5UIgLGPI?vo*AVOf&tRd`)#l zY1Wq`sD5TPdHIcn>@kh8gRzdgs86Vm7DZJ=`}@1sMIl)gXX0A?!OR3Pn%IPjqV_XI zWz!C3As?Ao2}p%!$Hp35%QEo<{tSiEqGOFjPlf3Xt2*-l{h;0f7IQB1FrgVC!Ob@D z3i^fW!3nvrIfhmPotMe1k>O9D<8e&NU+DZN9BDfw5jw60-Na8XA8E9{kgGXqcRl@` zY=?D}zGZzJ=S(axF~`f}dNd%Fw1MvI$c=xNUBbUIt3WkQ$Sd@s(YAH>%shaG@}<^- zKF#Wgu1RC!mj4$lEXLuPsWFo63l{g~0jUf(;@5%{gAJgs`?=CGGC|-~o{kco5;$x6 zBha5`hGTEoUmRQHcwg!xPV-N&R)FKWoCH(6B9LKYvWdyYmgoscz$+(_Gy=WTUO)K2 z)rA{4(SNlP`x}0|iH}PWb4}>pidSA85VC>mKi{>4uSOy!t6|e+t21(sL5J%<WZE+y zAx=w<5H}?w$up_fm`|KzZxmTcJx&e|$SyiV7SSU(*n)sOf358>8<GXZ8#Bf9iwK{~ zO`mJ1uC&tadEfh?I2k@%kCS4=%<b{fX4;KPcBdL*_fa#i>x%q<3E}UAEuUMH<O?8% zmgd?>ri1|ZLr`gUzhi@N(xJ;)8mfmwkn8E@Z4$y$)%Ma+KSw+|Ev~8MiNr?Kk#B2c zgv>xHcN^^|z9Fq9SG(?~={q~9C^t<*++#|6VRu?<w2fg&U@*c}BCyQAwmj5?1=yZ6 z{g9SFw2@?~-2t*r(105upFF|QBTuG8fHIQvN4`U|dzwcgb84?mrxJy4zQ|?PS03!C zdD0QXc<06z2~Mx8x#ZnMiuc3P+10HPz(iQ0BWa0t*(T`J{MUMDPLOF@tV2*DiOiLz z>?N1L=#=Ym7WkvR^3=?l>8#I+*6P)R;c;4~j=;{(=}CYay@RTYs_T&*w03;m_2$zD zZVsE$+QRxi_S37STN@qGyXB#~wLccZFpKl7C%OmRbv~ocw*z@tm+udLzsjE`epCw~ zKjm_3AMeul6A6-*_58H~`Kaou{IidG<#wb@Nw(ovH7w00P=FE`d18@KoNk(^QRnj; zXjtpibr?(-W`)y3y*e`ekuUf1x7GX1Ee4tuza#(i@XX`IJm^7S<j;>YBVqj?Ep6%S zd}IsKZTN75I8b;@9)YT#p8>on`tS*Hn{Rej4y`!*-D=ofcB;5O$GVN#snA_FoAj-S z_K6Sy46#jI(tOt`hr*N_s*IAga_jeIDW824ap-9i!iz^r(ia5&c)s%v5j%gB=CC<* z=H&Wv#Meb&LHOqA0O4G<+kJ3*5#gS!`Yjuek8JYu>KjE-(b1`B<1N?e*HKcU%sRcK zVJU(>5cu~v0<g|N9GcYtm31Pcpvh3`_H&~WG&I?@xY@C~l8OT@U5_K~J{?j50(N4L z!8CqsUXH}8X^pGbT4Cbxa|dBd-OSbT#0%2eI}y9Tx@!-75dCj->A7CWB@^EOia%FM zBtJGWFlNHUSWl|cP}SG7=O8uJ4D}@@Q?G7Li5yyzHN;Ha5gePs?ZrSB#dlbwWE>vx zfRq%n{yDk&ux~}d?x2c~)f0+l@{n|aR>V79czSkWqDW?!7hrpo8D<)ixH70!->b&Y zV%|_DijJy*eXpkFN0(;2*!JL;BE{OY4-=0>C0{rybGwKoee?Dy<$Y;38kH$&P}aDu zq^dZ4aDo?>A%|w-X|<po_Q2=W`0PUMQq{B&SIUvg+bRh@OeSbN;X=PLy4)Yr>4THU z$*&t1NCRsQ0pDIHHG_wsi?^YMmU`WmLfGo+mx)qnCf}@<C)~38BaISswhO##>vILO zg8q3Oi5ZPAfnB>X8LPH^^odgFg-ljOfLXASn+_fp>se0`?F%)Hu5Gi>6e}F8ktqxc zN{v$Lw<AZDgy*taL;ieiWLS#Lg_<raHg{&8N3Z&hpEyv8Nr|YgWNirliHD!={e~&C z0<(PTq0ac?xWv;j-;1yC@x!}`fprPEqjRMW;g-)yvF++eW><2O1~IPzK#v$ExBxwl zNj%Ar^jWhU3Hv~uOfVrrtD?K03hO@Rh@hG6?FeXklAcu_&~1;w5#@J#G#|8B(MXYQ zO!Gu1!YeM{qBX7*Ej;(EPP108%ii3HJSb)VIpdJRNm0Hz(_H^mKiiRQrlNJ?h@i$) zEqU5U>X%a`m=b|Zk6)X3B-cJe2kC_%-PRAkhbmeAi5i*LOp+=OQV^VfIZ5WT#wXzW zs7Z^ju@Z-aI}x{}nMZjCIx1e8&AB!v+RhQTlCV(%CnWURb})9Q^5#@7|5+J>XZb9- z2hJ2`;>U=}Dt|I@Ev?P%CDou4k;_q)xG`#*<E@InB#v2i$bBIrSkVA+<mI{|x-q7y zP|DGQmELS!we!bgtLq~7&&|C=S)1ct(aVRlAW-@PKqY-BT5d4n`^?59tIcE$4@kPV zg`HX38TaG-;`P+J4eaT5e8vG4o4VkotY!aGsDDVIeUp27R$$=EZE5qBR9`7*afdCQ zsTHbWz>|`kQ%f#EE*{r6u_vx#0{omK!qVF_6Mp8vc)DE&G~qlFdss~St(IY=wX8C2 zV(j$@D*j4-R6#l?H(o?owv(VBR)|c(ay7cOIsIKu^}CJ6iv;TgV=i7tYNO)H-2?Qd z=<cn=@$NsvRu+CEoU5>5y}eXJn&b(^AWijw-IL(8&#Fs=eYR}`pq5Rf1zSt>u*710 z>QEo^-3GIF+4vgYaca6_jzm}|EQ+9JFtTx1cWc86w$1(o#lS|JOW^u<s@H-HbqIT7 z@xp>j=F!$tsOj}@Qc~vRwtz_Nqu`Q8=ul`!CcgI<Qx6d<L9ZnY@W-FEuQ|xbU8N#Y z4lJ0qKcZH1sK^;clxX#CS?upetKE!tG$%#xc!iNJ<bG-jt~uwbGR+);dZ$G<{u*3_ zh3WqUmsCMr*of*6H(86hrg5IvDN^r`G&g1JqeK1E<Y=bI7qEA>+4YZFp-e-PXN&Th zaxT#lR9?5ew&E667^NQoLaU`yR8*u~;gLNmoT|Sw0$|u8<E*Yudy2G-yMM~#nQt6# zE@~ZUxf_D$0g>TteBYa3N=ggec<wwvqb?*YS6b^;zj@dZPg*<evuiq)-}%+BYem%& z5rj^j2PDo=6agfNuO{u?q1wUC+>v+V<L3dhb#=|88;+=;Y5U}Slz;u+3JutKfBZ!n z-x>Xz8|c67K2q%gB+4_{9}GRwB~ONLrtGIZt!`~HUvEFr$Wxht1%W`-*uY+t_jVg) z@Xx?Qic<ZBbO&$m3R{;rau7&~13;sd;tI(%=L8>HHY1w4hCm>CAoUW6zHhCeaY3== z{Ms-a@ag)uqJYz@Lw3;DU<p7729Sl`e?6#se>=Pefs%8b8;eL4QeA}ULN}U2Z*BJJ z<i>~UP^M4{o92G@d+9eh1zMbgKtoS~_DhrcGShDAb`PzyHK=rLb2j!P`6u^(vlGlD z4)8`7RAh6rDrk5Y?o4RguWn>s^p6p*msK0dgFwAVb9EI3z&~GOK5B@6yP>9Vt(K4T zFxe#L(NeR5D}sTo7yrwDRQyys@l~_E9O3m_OD%7!;PZSoG`?tN-+lkb?qg`bbaGjY zV54n<rj<{tQ;|wOUScY=Yi#oHQLeG>1iY!KX3Or2$GP{ijj;U=QNi7-ni}nKyQ7t! zj(toa`|~|T;)39=&pPf37R3q$h3!WCio7Ok;cf$JDV}qvkX6o4VNx4pe6JBEF!!<o zt6Xq*WE+1tSlV0ZV#dqJEhfKYf9t%jG2%JGbqOo-7DZ)|$vc%OTO)EN#mmVJ%xP|} zRl-@LlV_ikQw&u+&uTmt!=q(ZIenCyZK|DDKkIb$m0>L|k<g;y)JC&T%4?#z+|V)g z%^Q?vn7dI#UGAs1eTi;c4e1=|lBOsi&mcR%7WQ5KH+GAlnsFQdD%JD+>q6W&JlU9} zN*d>dFp0sG?zF~P=s6uFCDWOvL}fLf*v7sEd-z;olf9ZpwuUkJbOq&vPPS~8H<Xf+ zbE20iGGHMXJfCdWJMh!6D4r;0rsy_Ekt6Qof^1?+H6*7PlkB5am^<HI0X~-U?FQC< zEMN9jq?ZhoJ#Ji8@=Ebq=e?+*dFS09?A=+<9<bR$8n)SV<|G(gK$eePmb$Arcs$iJ zq|sKlTA(1uZH}V7ZZdt`V7|WhMk0TyP)tWYB>c0^b9#syf2_l&@-(U24T`DORQ>+B z=z>|8DpzgcUW`^1WbAreV~k8zQ`N~tC6;J#AM~~B7m%}u&S>snvAO`W=3AAyX7MbI z{`$V|;@kXyg}c%h$x(mq3YI)%^9xl(UFNHHu%I6ogSDY-|EsF2k7hGV<Eefax^-4t za#|m=)tdS!rlyRKWrl`T7LiDpkYw71jvzi0Q$%fNT0=}V2;(D2HANyxwL+25Z7G6S zm4-;ss&rbBXsKug-8ZJYd-kt;&b{~d+<V`1?s<OCy}$dsR$6TH!VFvRu6ljT;&|fV z$s!n*xl8C=m4PQU?Q=B9^h?l&s9Wx)aLJKBX8G&ER_SEQ`Fyvft<08ZiuK+mg49SI z{u3d-4Ylde9`MhtiTIZyhlto_uPn(`y9z<l7Cx4^B;Bcu#U#FE=}$}bEm!M;nh)83 z_vbas)nB6K8LOH+Y3t;;nl;YH4_PWS^Ha|{-A+evG{T4YVER;Ye~6PRX!_(Ou2VM7 zy1IkJer`t<Sah9@O>igu*#CM8(bYV;Fa8O)ujfsrEz|Hu{%>V}|0ey`E!D>l1JB2D zfp4RGxQ?u<ZP|(&o>GLFk?i};qK~w%<Do)oYq{Rll9~g?6?Y+!QwT^ent2b>n|#X; zg3AoS@$RvE6Xma~&DjWFZ+^Gv40G}7_0CI0`;+sBJzoby%FdjI982OATC-9UdZd^J z$wAu8shBvU71B!sb>CccRVwOm>3Q!MeEEWD2S>hQ<+2y8N+<uGfqc{`>lpi!!@~-b zCeolI0+*sErt8NT)=*Z4owyBjXY(dRV1~O=jrz)330E!`rrtq%kppO{3=2>$UndBK zvF`Nve{okVI6Lc#3xuEc=~L|w#u4%MnYB)CWj0)kz+VS$2269W_zzNUUI5Hp)=Kd5 zLp7=M7p*0A(t)|+u@Sq&@e`J+sGPple{DTMYY#-ZUz{`vr@Xu!#EDR)ryP9xrD@25 z2KN)Y8xuc!<eY5cjvv~q_2cF>ub%857#nud%%T8Hv?bFYVqS46+7P`cz`spGaBSHb z9W)PW;eh36B|5$VH}q=cL1R$UiVb7W1e=K0riu7PmR-+GLC~@7l@vTC1?-&?mmXQU zCAnx~Tv1JQ&GE(LJn9aE7OB#Or0@<n)JSS;CFX}F7*~wQ?#{7fO#xzYB#_FT*%^#6 ze?QqIN&}Kq$MwbJJlY~+$b%gJYjN$L|B_W+{4w~s>d*xS;m~@eW(0M7GoP4-;0)j9 zDWZ9qv^`Y!=@3~T_HT*o41#wl+Ozaqj`Ifw-4T8Pe$ApWb=1Sfn~%b2;M$oNv58){ zOUg2F4a#>rI$~`dvVt;9Bu5Dm^Nq!AH0X$kOKb>dlE1boNfW30Y!Lh)%b{<_D4<;4 zXIyK2P-$xQSS!5}+#Y2hwW(CxR9CfR6%8enZ4UkH?(t&4aW#^DPAp%VJvB=|U`)*u zdj&yoByo+&;E5BoR(f!>dF0GS=?8T~$PA}@{YHGx*;%xs2};8w{Pu7>BquL((cR@$ z@!)0P*0YEcj(Lh%Ws<qqyN1Et45w)L?@y)W(r%Q-iC6V-T<ln@q}3^b@f3@41vtv1 z{&P8jhvE%L8e$i@oA3GDvQ2gra)BYyq8&WxX0;w{(sYhZ4H_?u!9KAtZt0c%5V$)l zuyiwL!rmuhQgxHKqH+eG)B{0%D=I7Z8~3LaV1<|^*HPfDLi&c{O}(l{&vg#ghl>7= zW$p<hcE&1NOrw<AM9(%sDCj=e=YIzqIx?N>bRF8rLK9(W*N~j<@T2Es^>E1=5XtLq z{O^MY7ORi$TUe6hM~WqdE*n&0N!|k>7SOqO06D0AIjT-tI|c$_iBFFMDrUg*f07CP zUuhRqhmxsnzt_G6Jij5}`Ey0fUDy{!XRjyexPZuiyXHpdMOE?>Z;=lTkL5om-&@En z+U6Hq7N3Dkj)${<#<Sb9OabwM9Qub)=)c_IKOEvySx{#~Su=n~2=#R{%64f6zta&i zAWKC|7-gpw!beyA-f+_GRoEv(urKssCfj_(scB?a=kn^e{V8pk&bofN4e=#m+-##0 z2Mwb<{`gBBeC^d({fK#6@lG%+Z!hnP?~&S3_$T=8{*>hpg1Ac9VYTS7DT()M_7z_S z)aFDxZ(3&!lt57ww&TwyYV51_ObA1VS(t*a?@_n4GzpYB2-0;Iw|t^N&PR9_43YOH zm79URkpe)j4+O^=skdb^!e8!oCvHfe6Pyb>xH~3T5BConUF%<ilvjj&vzS@OaikDS z14SeTgl*Ix+g69j&jFih(O>q0>84L`-oaH-AN*w-<5kr$pY&B+as^Ugm0n_yjCfQ3 n<u+n;cn{^9p{w`#RMp2FEuo&>88X~$9psE(2!!kV{pEiHZ_@(= From d822d666c7a770bf44dd13ef32df98a275b66c23 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Sat, 30 Sep 2023 22:09:59 +0200 Subject: [PATCH 143/716] [prtester] improvements and fixes for prtester (#3721) --- .github/.gitignore | 6 +++ .github/prtester.py | 54 +++++++++++++++++++-------- .github/workflows/prhtmlgenerator.yml | 2 + templates/exception.html.php | 4 +- 4 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 .github/.gitignore diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 00000000..6310b3dd --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1,6 @@ +# Visual Studio Code +.vscode/* + +# Generated files +comment*.md +comment*.txt diff --git a/.github/prtester.py b/.github/prtester.py index a2a7ab84..103ecbe6 100644 --- a/.github/prtester.py +++ b/.github/prtester.py @@ -1,5 +1,6 @@ import argparse import requests +import re from bs4 import BeautifulSoup from datetime import datetime from typing import Iterable @@ -16,18 +17,18 @@ class Instance: name = '' url = '' -def main(instances: Iterable[Instance], with_upload: bool, comment_title: str): +def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: bool, title: str, output_file: str): start_date = datetime.now() table_rows = [] for instance in instances: page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page - table_rows += testBridges(instance, bridge_cards, with_upload) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version - with open(file=os.getcwd() + '/comment.txt', mode='w+', encoding='utf-8') as file: + table_rows += testBridges(instance, bridge_cards, with_upload, with_reduced_upload) # run the main scraping code with the list of bridges + with open(file=output_file, mode='w+', encoding='utf-8') as file: table_rows_value = '\n'.join(sorted(table_rows)) file.write(f''' -## {comment_title} +## {title} | Bridge | Context | Status | | - | - | - | {table_rows_value} @@ -35,7 +36,7 @@ def main(instances: Iterable[Instance], with_upload: bool, comment_title: str): *last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}* '''.strip()) -def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) -> Iterable: +def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool) -> Iterable: instance_suffix = '' if instance.name: instance_suffix = f' ({instance.name})' @@ -43,7 +44,7 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) - for bridge_card in bridge_cards: bridgeid = bridge_card.get('id') bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata - print(f'{bridgeid}{instance_suffix}\n') + print(f'{bridgeid}{instance_suffix}') bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html' bridge_name = bridgeid.replace('Bridge', '') context_forms = bridge_card.find_all("form") @@ -112,20 +113,26 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) - page_text = response.text.replace('<head>','<head><base href="https://rss-bridge.org/bridge01/" target="_blank">') page_text = page_text.encode("utf_8") soup = BeautifulSoup(page_text, "html.parser") - status_messages = list(map(lambda e: f'⚠️ `{e.text.strip().splitlines()[0]}`', soup.find_all('pre'))) + status_messages = [] if response.status_code != 200: - status_messages = [f'❌ `HTTP status {response.status_code} {response.reason}`'] + status_messages + status_messages += [f'❌ `HTTP status {response.status_code} {response.reason}`'] else: feed_items = soup.select('.feeditem') feed_items_length = len(feed_items) if feed_items_length <= 0: status_messages += [f'⚠️ `The feed has no items`'] elif feed_items_length == 1 and len(soup.select('.error')) > 0: - status_messages = [f'❌ `{feed_items[0].text.strip().splitlines()[0]}`'] + status_messages + status_messages += [f'❌ `{getFirstLine(feed_items[0].text)}`'] + status_messages += map(lambda e: f'❌ `{getFirstLine(e.text)}`', soup.select('.error .error-type') + soup.select('.error .error-message')) + for item_element in soup.select('.feeditem'): # remove all feed items to not accidentally selected <pre> tags from item content + item_element.decompose() + status_messages += map(lambda e: f'⚠️ `{getFirstLine(e.text)}`', soup.find_all('pre')) + status_messages = list(dict.fromkeys(status_messages)) # remove duplicates status = '<br>'.join(status_messages) - if status.strip() == '': + status_is_ok = status == ''; + if status_is_ok: status = '✔️' - if with_upload: + if with_upload and (not with_reduced_upload or not status_is_ok): termpad = requests.post(url="https://termpad.com/", data=page_text) termpad_url = termpad.text.strip() termpad_url = termpad_url.replace('termpad.com/','termpad.com/raw/') @@ -133,11 +140,22 @@ def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool) - form_number += 1 return table_rows +def getFirstLine(value: str) -> str: + # trim whitespace and remove text that can break the table or is simply unnecessary + clean_value = re.sub('^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip()) + first_line = next(iter(clean_value.splitlines()), '') + max_length = 250 + if (len(first_line) > max_length): + first_line = first_line[:max_length] + '...' + return first_line + if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('-i', '--instances', nargs='+') - parser.add_argument('-nu', '--no-upload', action='store_true') - parser.add_argument('-t', '--comment-title', default='Pull request artifacts') + parser.add_argument('--instances', nargs='+') + parser.add_argument('--no-upload', action='store_true') + parser.add_argument('--reduced-upload', action='store_true') + parser.add_argument('--title', default='Pull request artifacts') + parser.add_argument('--output-file', default=os.getcwd() + '/comment.txt') args = parser.parse_args() instances = [] if args.instances: @@ -156,4 +174,10 @@ if __name__ == '__main__': instance.name = 'pr' instance.url = 'http://localhost:3001' instances.append(instance) - main(instances=instances, with_upload=not args.no_upload, comment_title=args.comment_title); \ No newline at end of file + main( + instances=instances, + with_upload=not args.no_upload, + with_reduced_upload=args.reduced_upload and not args.no_upload, + title=args.title, + output_file=args.output_file + ); \ No newline at end of file diff --git a/.github/workflows/prhtmlgenerator.yml b/.github/workflows/prhtmlgenerator.yml index cfedca13..7985250a 100644 --- a/.github/workflows/prhtmlgenerator.yml +++ b/.github/workflows/prhtmlgenerator.yml @@ -8,6 +8,8 @@ jobs: test-pr: name: Generate HTML runs-on: ubuntu-latest + env: + PYTHONUNBUFFERED: 1 # Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989 steps: - name: Check out self diff --git a/templates/exception.html.php b/templates/exception.html.php index 6ea747f2..dac0ad26 100644 --- a/templates/exception.html.php +++ b/templates/exception.html.php @@ -60,7 +60,7 @@ <h2>Details</h2> <div style="margin-bottom: 15px"> - <div> + <div class="error-type"> <strong>Type:</strong> <?= e(get_class($e)) ?> </div> @@ -68,7 +68,7 @@ <strong>Code:</strong> <?= e($e->getCode()) ?> </div> - <div> + <div class="error-message"> <strong>Message:</strong> <?= e(sanitize_root($e->getMessage())) ?> </div> From 7273a05f02a1be053c5c5ecfc10cbea0da06cf18 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 1 Oct 2023 18:53:50 +0200 Subject: [PATCH 144/716] fix: google play and tiktok (#3722) * fix(googleplay) * fix(tiktok) --- bridges/GooglePlayStoreBridge.php | 32 ++++++++++++------------------- bridges/TikTokBridge.php | 4 ++++ lib/RssBridge.php | 3 --- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/bridges/GooglePlayStoreBridge.php b/bridges/GooglePlayStoreBridge.php index d61be2c8..86343520 100644 --- a/bridges/GooglePlayStoreBridge.php +++ b/bridges/GooglePlayStoreBridge.php @@ -21,30 +21,22 @@ class GooglePlayStoreBridge extends BridgeAbstract ] ]]; - const INFORMATION_MAP = [ - 'Updated' => 'timestamp', - 'Current Version' => 'title', - 'Offered By' => 'author' - ]; - public function collectData() { - $appuri = static::URI . '/details?id=' . $this->getInput('id'); - $html = getSimpleHTMLDOM($appuri); + $id = $this->getInput('id'); + $url = 'https://play.google.com/store/apps/details?id=' . $id; + $html = getSimpleHTMLDOM($url); + + $updatedAtElement = $html->find('div.TKjAsc div', 2); + // Updated onSep 27, 2023 + $updatedAt = $updatedAtElement->plaintext; + $description = $html->find('div.bARER', 0); $item = []; - $item['uri'] = $appuri; - $item['content'] = $html->find('div[itemprop=description]', 1)->innertext; - - // Find other fields from Additional Information section - foreach ($html->find('.hAyfc') as $info) { - $index = self::INFORMATION_MAP[$info->first_child()->plaintext] ?? null; - if (is_null($index)) { - continue; - } - $item[$index] = $info->children(1)->plaintext; - } - + $item['uri'] = $url; + $item['title'] = $id . ' ' . $updatedAt; + $item['content'] = $description->innertext ?? ''; + $item['uid'] = 'GooglePlayStoreBridge/' . $updatedAt; $this->items[] = $item; } diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php index 769bc625..73a18b04 100644 --- a/bridges/TikTokBridge.php +++ b/bridges/TikTokBridge.php @@ -40,6 +40,10 @@ class TikTokBridge extends BridgeAbstract $SIGI_STATE_RAW = $var->innertext; $SIGI_STATE = Json::decode($SIGI_STATE_RAW, false); + if (!isset($SIGI_STATE->ItemModule)) { + return; + } + foreach ($SIGI_STATE->ItemModule as $key => $value) { $item = []; diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 6ba952eb..2fb21323 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -39,9 +39,6 @@ final class RssBridge $line ); self::$logger->warning($text); - if (Debug::isEnabled()) { - print sprintf("<pre>%s</pre>\n", e($text)); - } }); // There might be some fatal errors which are not caught by set_error_handler() or \Throwable. From 0c92cf32d471cc0722727b7cd40779882f7b923a Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Sun, 1 Oct 2023 19:00:13 +0200 Subject: [PATCH 145/716] [ImgsedBridge] Fix and improvements (#3710) * [ImgsedBridge] Fix and improvements - Display an error if the user doesn't select at least an content type to display - Unsplit the regular expression to make the URL of imgsed.com work too - Remove the "hour part" of the publication date : the website shows only the number of days if the content is older than one day * [ImgsedBridge] Fix and improvements Fix syntax * [ImgsedBridge] Fix and improvements - Fix TEST_DETECT_PARAMETERS - change detectParameters regular expression to match more instagram.com URLs * [ImgsedBridge] Fix and improvements - Fix date parsing for interval 'a day' * lint --------- Co-authored-by: Dag <me@dvikan.no> --- bridges/ImgsedBridge.php | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index 1fa5b827..b6385361 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -36,6 +36,12 @@ class ImgsedBridge extends BridgeAbstract ], ] ]; + const TEST_DETECT_PARAMETERS = [ + 'https://www.instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram', 'post' => 'on', 'story' => 'on', 'tagged' => 'on'], + 'https://instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram', 'post' => 'on', 'story' => 'on', 'tagged' => 'on'], + 'https://imgsed.com/instagram/' => ['context' => 'Username', 'u' => 'instagram', 'post' => 'on', 'story' => 'on', 'tagged' => 'on'], + 'https://www.imgsed.com/instagram/' => ['context' => 'Username', 'u' => 'instagram', 'post' => 'on', 'story' => 'on', 'tagged' => 'on'], + ]; public function getURI() { @@ -213,9 +219,16 @@ HTML, { $date = date_create(); $dateString = str_replace(' ago', '', $content); + // Special case : 'a day' is not a valid interval in PHP, so replace it with it's PHP equivalenbt : '1 day' + if ($dateString == 'a day') { + $dateString = '1 day'; + } + $relativeDate = date_interval_create_from_date_string($dateString); if ($relativeDate) { date_sub($date, $relativeDate); + // As the relative interval has the precision of a day for date older than 24 hours, we can remove the hour of the date, as it is not relevant + date_time_set($date, 0, 0, 0, 0); } else { $this->logger->info(sprintf('Unable to parse date string: %s', $dateString)); } @@ -244,7 +257,13 @@ HTML, if ($this->getInput('tagged')) { $types[] = 'Tags'; } + + // If no content type is selected, this bridge does nothing, so we return an error + if (count($types) == 0) { + returnClientError('You must select at least one of the content type : Post, Stories or Tags !'); + } $typesText = $types[0] ?? ''; + if (count($types) > 1) { for ($i = 1; $i < count($types) - 1; $i++) { $typesText .= ', ' . $types[$i]; @@ -262,10 +281,9 @@ HTML, $params = [ 'post' => 'on', 'story' => 'on', - 'tagged' => 'on' + 'tagged' => 'on', ]; - $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})\/(reels\/|tagged\/|) -|(www\.|)(imgsed.com)\/(stories\/|tagged\/|)([a-zA-Z0-9_\.]{1,30})\/)/'; + $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})(\/reels\/|\/tagged\/|\/|)|(www\.|)(imgsed.com)\/(stories\/|tagged\/|)([a-zA-Z0-9_\.]{1,30})\/)/'; if (preg_match($regex, $url, $matches) > 0) { $params['context'] = 'Username'; // Extract detected domain using the regex @@ -273,7 +291,7 @@ HTML, if ($domain == 'imgsed.com') { $params['u'] = $matches[10]; return $params; - } else if ($domain == 'instagram.com') { + } elseif ($domain == 'instagram.com') { $params['u'] = $matches[5]; return $params; } else { From 41df17bc464832118a767ceff6c9217071699783 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 1 Oct 2023 19:23:30 +0200 Subject: [PATCH 146/716] refactor (#3712) * test: refactor test suite * docs * refactor * yup * docs --- README.md | 2 + actions/ConnectivityAction.php | 12 - actions/DetectAction.php | 12 - actions/DisplayAction.php | 4 +- actions/ListAction.php | 28 +- actions/SetBridgeCacheAction.php | 12 - bridges/DemoBridge.php | 2 +- lib/ApiAuthenticationMiddleware.php | 12 - lib/AuthenticationMiddleware.php | 12 - lib/BridgeAbstract.php | 162 +++++------ lib/BridgeCard.php | 20 -- lib/BridgeFactory.php | 8 +- lib/Configuration.php | 12 - lib/FeedExpander.php | 12 - lib/FormatFactory.php | 12 - lib/ParameterValidator.php | 272 +++++++----------- lib/bootstrap.php | 14 +- lib/http.php | 6 +- lib/php8backports.php | 34 --- .../BridgeImplementationTest.php | 103 ++++--- tests/CacheImplementationTest.php | 36 +++ tests/Caches/CacheImplementationTest.php | 48 ---- tests/ParameterValidatorTest.php | 40 +++ 23 files changed, 309 insertions(+), 566 deletions(-) rename tests/{Bridges => }/BridgeImplementationTest.php (65%) create mode 100644 tests/CacheImplementationTest.php delete mode 100644 tests/Caches/CacheImplementationTest.php create mode 100644 tests/ParameterValidatorTest.php diff --git a/README.md b/README.md index a1e5fdc7..7037095e 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,8 @@ Modify `report_limit` so that an error must occur 3 times before it is reported. ; Defines how often an error must occur before it is reported to the user report_limit = 3 +The report count is reset to 0 each day. + ### How to password-protect the instance HTTP basic access authentication: diff --git a/actions/ConnectivityAction.php b/actions/ConnectivityAction.php index cfffd195..1568333a 100644 --- a/actions/ConnectivityAction.php +++ b/actions/ConnectivityAction.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - /** * Checks if the website for a given bridge is reachable. * diff --git a/actions/DetectAction.php b/actions/DetectAction.php index 49b7ced7..bbacde38 100644 --- a/actions/DetectAction.php +++ b/actions/DetectAction.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - class DetectAction implements ActionInterface { public function execute(array $request) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 8c9dd057..e3b25fef 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -101,8 +101,8 @@ class DisplayAction implements ActionInterface try { $bridge->loadConfiguration(); // Remove parameters that don't concern bridges - $bridgeData = array_diff_key($request, array_fill_keys(['action', 'bridge', 'format', '_noproxy', '_cache_timeout', '_error_time'], '')); - $bridge->setDatas($bridgeData); + $input = array_diff_key($request, array_fill_keys(['action', 'bridge', 'format', '_noproxy', '_cache_timeout', '_error_time'], '')); + $bridge->setInput($input); $bridge->collectData(); $items = $bridge->getItems(); if (isset($items[0]) && is_array($items[0])) { diff --git a/actions/ListAction.php b/actions/ListAction.php index 9025bf6e..19bb4d37 100644 --- a/actions/ListAction.php +++ b/actions/ListAction.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - class ListAction implements ActionInterface { public function execute(array $request) @@ -26,14 +14,14 @@ class ListAction implements ActionInterface $bridge = $bridgeFactory->create($bridgeClassName); $list->bridges[$bridgeClassName] = [ - 'status' => $bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive', - 'uri' => $bridge->getURI(), - 'donationUri' => $bridge->getDonationURI(), - 'name' => $bridge->getName(), - 'icon' => $bridge->getIcon(), - 'parameters' => $bridge->getParameters(), - 'maintainer' => $bridge->getMaintainer(), - 'description' => $bridge->getDescription() + 'status' => $bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive', + 'uri' => $bridge->getURI(), + 'donationUri' => $bridge->getDonationURI(), + 'name' => $bridge->getName(), + 'icon' => $bridge->getIcon(), + 'parameters' => $bridge->getParameters(), + 'maintainer' => $bridge->getMaintainer(), + 'description' => $bridge->getDescription() ]; } $list->total = count($list->bridges); diff --git a/actions/SetBridgeCacheAction.php b/actions/SetBridgeCacheAction.php index c9264a27..2e9d7147 100644 --- a/actions/SetBridgeCacheAction.php +++ b/actions/SetBridgeCacheAction.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - class SetBridgeCacheAction implements ActionInterface { private CacheInterface $cache; diff --git a/bridges/DemoBridge.php b/bridges/DemoBridge.php index 15ab7377..18582aa6 100644 --- a/bridges/DemoBridge.php +++ b/bridges/DemoBridge.php @@ -4,7 +4,7 @@ class DemoBridge extends BridgeAbstract { const MAINTAINER = 'teromene'; const NAME = 'DemoBridge'; - const URI = 'http://github.com/rss-bridge/rss-bridge'; + const URI = 'https://github.com/rss-bridge/rss-bridge'; const DESCRIPTION = 'Bridge used for demos'; const CACHE_TIMEOUT = 15; diff --git a/lib/ApiAuthenticationMiddleware.php b/lib/ApiAuthenticationMiddleware.php index 6a59e760..62886314 100644 --- a/lib/ApiAuthenticationMiddleware.php +++ b/lib/ApiAuthenticationMiddleware.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - final class ApiAuthenticationMiddleware { public function __invoke($request): void diff --git a/lib/AuthenticationMiddleware.php b/lib/AuthenticationMiddleware.php index 8c2f6b29..a91420f8 100644 --- a/lib/AuthenticationMiddleware.php +++ b/lib/AuthenticationMiddleware.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - final class AuthenticationMiddleware { public function __construct() diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index e074ce74..a7b811a8 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -90,17 +90,70 @@ abstract class BridgeAbstract return static::CACHE_TIMEOUT; } - /** - * Sets the input values for a given context. - * - * @param array $inputs Associative array of inputs - * @param string $queriedContext The context name - * @return void - */ - protected function setInputs(array $inputs, $queriedContext) + public function loadConfiguration() + { + foreach (static::CONFIGURATION as $optionName => $optionValue) { + $section = $this->getShortName(); + $configurationOption = Configuration::getConfig($section, $optionName); + + if ($configurationOption !== null) { + $this->configuration[$optionName] = $configurationOption; + continue; + } + + if (isset($optionValue['required']) && $optionValue['required'] === true) { + throw new \Exception(sprintf('Missing configuration option: %s', $optionName)); + } elseif (isset($optionValue['defaultValue'])) { + $this->configuration[$optionName] = $optionValue['defaultValue']; + } + } + } + + public function setInput(array $input) + { + $context = $input['context'] ?? null; + if ($context) { + // Context hinting (optional) + $this->queriedContext = $context; + unset($input['context']); + } + + $parameters = $this->getParameters(); + + if (!$parameters) { + if ($input) { + throw new \Exception('Invalid parameters value(s)'); + } + return; + } + + $validator = new ParameterValidator(); + + // $input is passed by reference! + if (!$validator->validateInput($input, $parameters)) { + $invalidParameterKeys = array_column($validator->getInvalidParameters(), 'name'); + throw new \Exception(sprintf('Invalid parameters value(s): %s', implode(', ', $invalidParameterKeys))); + } + + // Guess the context from input data + if (empty($this->queriedContext)) { + $queriedContext = $validator->getQueriedContext($input, $parameters); + $this->queriedContext = $queriedContext; + } + + if (is_null($this->queriedContext)) { + throw new \Exception('Required parameter(s) missing'); + } elseif ($this->queriedContext === false) { + throw new \Exception('Mixed context parameters'); + } + + $this->setInputWithContext($input, $this->queriedContext); + } + + private function setInputWithContext(array $input, $queriedContext) { // Import and assign all inputs to their context - foreach ($inputs as $name => $value) { + foreach ($input as $name => $value) { foreach (static::PARAMETERS as $context => $set) { if (array_key_exists($name, static::PARAMETERS[$context])) { $this->inputs[$context][$name]['value'] = $value; @@ -128,7 +181,7 @@ abstract class BridgeAbstract switch ($type) { case 'checkbox': - $this->inputs[$context][$name]['value'] = $inputs[$context][$name]['value'] ?? false; + $this->inputs[$context][$name]['value'] = $input[$context][$name]['value'] ?? false; break; case 'list': if (!isset($properties['defaultValue'])) { @@ -153,8 +206,8 @@ abstract class BridgeAbstract // Copy global parameter values to the guessed context if (array_key_exists('global', static::PARAMETERS)) { foreach (static::PARAMETERS['global'] as $name => $properties) { - if (isset($inputs[$name])) { - $value = $inputs[$name]; + if (isset($input[$name])) { + $value = $input[$name]; } else { if ($properties['type'] ?? null === 'checkbox') { $value = false; @@ -176,91 +229,6 @@ abstract class BridgeAbstract } } - /** - * Set inputs for the bridge - * - * Returns errors and aborts execution if the provided input parameters are - * invalid. - * - * @param array List of input parameters. Each element in this list must - * relate to an item in {@see BridgeAbstract::PARAMETERS} - * @return void - */ - public function setDatas(array $inputs) - { - if (isset($inputs['context'])) { // Context hinting (optional) - $this->queriedContext = $inputs['context']; - unset($inputs['context']); - } - - if (empty(static::PARAMETERS)) { - if (!empty($inputs)) { - throw new \Exception('Invalid parameters value(s)'); - } - - return; - } - - $validator = new ParameterValidator(); - - if (!$validator->validateData($inputs, static::PARAMETERS)) { - $parameters = array_map( - function ($i) { - return $i['name']; - }, // Just display parameter names - $validator->getInvalidParameters() - ); - - throw new \Exception(sprintf('Invalid parameters value(s): %s', implode(', ', $parameters))); - } - - // Guess the context from input data - if (empty($this->queriedContext)) { - $this->queriedContext = $validator->getQueriedContext($inputs, static::PARAMETERS); - } - - if (is_null($this->queriedContext)) { - throw new \Exception('Required parameter(s) missing'); - } elseif ($this->queriedContext === false) { - throw new \Exception('Mixed context parameters'); - } - - $this->setInputs($inputs, $this->queriedContext); - } - - /** - * Loads configuration for the bridge - * - * Returns errors and aborts execution if the provided configuration is - * invalid. - * - * @return void - */ - public function loadConfiguration() - { - foreach (static::CONFIGURATION as $optionName => $optionValue) { - $section = $this->getShortName(); - $configurationOption = Configuration::getConfig($section, $optionName); - - if ($configurationOption !== null) { - $this->configuration[$optionName] = $configurationOption; - continue; - } - - if (isset($optionValue['required']) && $optionValue['required'] === true) { - throw new \Exception(sprintf('Missing configuration option: %s', $optionName)); - } elseif (isset($optionValue['defaultValue'])) { - $this->configuration[$optionName] = $optionValue['defaultValue']; - } - } - } - - /** - * Returns the value for the provided input - * - * @param string $input The input name - * @return mixed|null The input value or null if the input is not defined - */ protected function getInput($input) { return $this->inputs[$this->queriedContext][$input]['value'] ?? null; diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index 99c44fff..b2fda192 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -1,25 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * A generator class for a single bridge card on the home page of RSS-Bridge. - * - * This class generates the HTML content for a single bridge card for the home - * page of RSS-Bridge. - * - * @todo Return error if a caller creates an object of this class. - */ final class BridgeCard { /** diff --git a/lib/BridgeFactory.php b/lib/BridgeFactory.php index c3da4bfe..ad433287 100644 --- a/lib/BridgeFactory.php +++ b/lib/BridgeFactory.php @@ -4,9 +4,9 @@ final class BridgeFactory { private CacheInterface $cache; private Logger $logger; - private $bridgeClassNames = []; - private $enabledBridges = []; - private $missingEnabledBridges = []; + private array $bridgeClassNames = []; + private array $enabledBridges = []; + private array $missingEnabledBridges = []; public function __construct() { @@ -22,7 +22,7 @@ final class BridgeFactory $enabledBridges = Configuration::getConfig('system', 'enabled_bridges'); if ($enabledBridges === null) { - throw new \Exception('No bridges are enabled... wtf?'); + throw new \Exception('No bridges are enabled...'); } foreach ($enabledBridges as $enabledBridge) { if ($enabledBridge === '*') { diff --git a/lib/Configuration.php b/lib/Configuration.php index c38d7cc9..d699178f 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - /** * Configuration module for RSS-Bridge. * diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index 14c931e6..76f570b6 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - /** * An abstract class for bridges that need to transform existing RSS or Atom * feeds. diff --git a/lib/FormatFactory.php b/lib/FormatFactory.php index 9cded40f..042dcf31 100644 --- a/lib/FormatFactory.php +++ b/lib/FormatFactory.php @@ -1,17 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - class FormatFactory { private $folder; diff --git a/lib/ParameterValidator.php b/lib/ParameterValidator.php index 31934432..e8de754c 100644 --- a/lib/ParameterValidator.php +++ b/lib/ParameterValidator.php @@ -1,154 +1,21 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** - * Validator for bridge parameters - */ class ParameterValidator { - /** - * Holds the list of invalid parameters - * - * @var array - */ - private $invalid = []; + private array $invalid = []; /** - * Add item to list of invalid parameters + * Check that inputs are actually present in the bridge parameters. * - * @param string $name The name of the parameter - * @param string $reason The reason for that parameter being invalid - * @return void + * Also check whether input values are allowed. */ - private function addInvalidParameter($name, $reason) + public function validateInput(&$input, $parameters): bool { - $this->invalid[] = [ - 'name' => $name, - 'reason' => $reason, - ]; - } - - /** - * Return list of invalid parameters. - * - * Each element is an array of 'name' and 'reason'. - * - * @return array List of invalid parameters - */ - public function getInvalidParameters() - { - return $this->invalid; - } - - /** - * Validate value for a text input - * - * @param string $value The value of a text input - * @param string|null $pattern (optional) A regex pattern - * @return string|null The filtered value or null if the value is invalid - */ - private function validateTextValue($value, $pattern = null) - { - if (!is_null($pattern)) { - $filteredValue = filter_var( - $value, - FILTER_VALIDATE_REGEXP, - ['options' => [ - 'regexp' => '/^' . $pattern . '$/' - ] - ] - ); - } else { - $filteredValue = filter_var($value); - } - - if ($filteredValue === false) { - return null; - } - - return $filteredValue; - } - - /** - * Validate value for a number input - * - * @param int $value The value of a number input - * @return int|null The filtered value or null if the value is invalid - */ - private function validateNumberValue($value) - { - $filteredValue = filter_var($value, FILTER_VALIDATE_INT); - - if ($filteredValue === false) { - return null; - } - - return $filteredValue; - } - - /** - * Validate value for a checkbox - * - * @param bool $value The value of a checkbox - * @return bool The filtered value - */ - private function validateCheckboxValue($value) - { - return filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); - } - - /** - * Validate value for a list - * - * @param string $value The value of a list - * @param array $expectedValues A list of expected values - * @return string|null The filtered value or null if the value is invalid - */ - private function validateListValue($value, $expectedValues) - { - $filteredValue = filter_var($value); - - if ($filteredValue === false) { - return null; - } - - if (!in_array($filteredValue, $expectedValues)) { // Check sub-values? - foreach ($expectedValues as $subName => $subValue) { - if (is_array($subValue) && in_array($filteredValue, $subValue)) { - return $filteredValue; - } - } - return null; - } - - return $filteredValue; - } - - /** - * Check if all required parameters are satisfied - * - * @param array $data (ref) A list of input values - * @param array $parameters The bridge parameters - * @return bool True if all parameters are satisfied - */ - public function validateData(&$data, $parameters) - { - if (!is_array($data)) { + if (!is_array($input)) { return false; } - foreach ($data as $name => $value) { + foreach ($input as $name => $value) { // Some RSS readers add a cache-busting parameter (_=<timestamp>) to feed URLs, detect and ignore them. if ($name === '_') { continue; @@ -156,54 +23,60 @@ class ParameterValidator $registered = false; foreach ($parameters as $context => $set) { - if (array_key_exists($name, $set)) { - $registered = true; - if (!isset($set[$name]['type'])) { - $set[$name]['type'] = 'text'; - } + if (!array_key_exists($name, $set)) { + continue; + } + $registered = true; + if (!isset($set[$name]['type'])) { + // Default type is text + $set[$name]['type'] = 'text'; + } - switch ($set[$name]['type']) { - case 'number': - $data[$name] = $this->validateNumberValue($value); - break; - case 'checkbox': - $data[$name] = $this->validateCheckboxValue($value); - break; - case 'list': - $data[$name] = $this->validateListValue($value, $set[$name]['values']); - break; - default: - case 'text': - if (isset($set[$name]['pattern'])) { - $data[$name] = $this->validateTextValue($value, $set[$name]['pattern']); - } else { - $data[$name] = $this->validateTextValue($value); - } - break; - } + switch ($set[$name]['type']) { + case 'number': + $input[$name] = $this->validateNumberValue($value); + break; + case 'checkbox': + $input[$name] = $this->validateCheckboxValue($value); + break; + case 'list': + $input[$name] = $this->validateListValue($value, $set[$name]['values']); + break; + default: + case 'text': + if (isset($set[$name]['pattern'])) { + $input[$name] = $this->validateTextValue($value, $set[$name]['pattern']); + } else { + $input[$name] = $this->validateTextValue($value); + } + break; + } - if (is_null($data[$name]) && isset($set[$name]['required']) && $set[$name]['required']) { - $this->addInvalidParameter($name, 'Parameter is invalid!'); - } + if ( + is_null($input[$name]) + && isset($set[$name]['required']) + && $set[$name]['required'] + ) { + $this->invalid[] = ['name' => $name, 'reason' => 'Parameter is invalid!']; } } if (!$registered) { - $this->addInvalidParameter($name, 'Parameter is not registered!'); + $this->invalid[] = ['name' => $name, 'reason' => 'Parameter is not registered!']; } } - return empty($this->invalid); + return $this->invalid === []; } /** * Get the name of the context matching the provided inputs * - * @param array $data Associative array of user data + * @param array $input Associative array of user data * @param array $parameters Array of bridge parameters * @return string|null Returns the context name or null if no match was found */ - public function getQueriedContext($data, $parameters) + public function getQueriedContext($input, $parameters) { $queriedContexts = []; @@ -212,7 +85,7 @@ class ParameterValidator $queriedContexts[$context] = null; // Ensure all user data exist in the current context - $notInContext = array_diff_key($data, $set); + $notInContext = array_diff_key($input, $set); if (array_key_exists('global', $parameters)) { $notInContext = array_diff_key($notInContext, $parameters['global']); } @@ -222,7 +95,7 @@ class ParameterValidator // Check if all parameters of the context are satisfied foreach ($set as $id => $properties) { - if (isset($data[$id]) && !empty($data[$id])) { + if (isset($input[$id]) && !empty($input[$id])) { $queriedContexts[$context] = true; } elseif ( isset($properties['type']) @@ -248,8 +121,8 @@ class ParameterValidator switch (array_sum($queriedContexts)) { case 0: // Found no match, is there a context without parameters? - if (isset($data['context'])) { - return $data['context']; + if (isset($input['context'])) { + return $input['context']; } foreach ($queriedContexts as $context => $queried) { if (is_null($queried)) { @@ -264,4 +137,55 @@ class ParameterValidator return false; } } + + public function getInvalidParameters(): array + { + return $this->invalid; + } + + private function validateTextValue($value, $pattern = null) + { + if (is_null($pattern)) { + // No filtering taking place + $filteredValue = filter_var($value); + } else { + $filteredValue = filter_var($value, FILTER_VALIDATE_REGEXP, ['options' => ['regexp' => '/^' . $pattern . '$/']]); + } + if ($filteredValue === false) { + return null; + } + return $filteredValue; + } + + private function validateNumberValue($value) + { + $filteredValue = filter_var($value, FILTER_VALIDATE_INT); + if ($filteredValue === false) { + return null; + } + return $filteredValue; + } + + private function validateCheckboxValue($value) + { + return filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); + } + + private function validateListValue($value, $expectedValues) + { + $filteredValue = filter_var($value); + if ($filteredValue === false) { + return null; + } + if (!in_array($filteredValue, $expectedValues)) { + // Check sub-values? + foreach ($expectedValues as $subName => $subValue) { + if (is_array($subValue) && in_array($filteredValue, $subValue)) { + return $filteredValue; + } + } + return null; + } + return $filteredValue; + } } diff --git a/lib/bootstrap.php b/lib/bootstrap.php index bc584541..a95de9dd 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -1,18 +1,6 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -/** Path to the formats library */ +// Path to the formats library const PATH_LIB_FORMATS = __DIR__ . '/../formats/'; /** Path to the caches library */ diff --git a/lib/http.php b/lib/http.php index 10ce86c9..3d65b2d1 100644 --- a/lib/http.php +++ b/lib/http.php @@ -211,12 +211,12 @@ final class Response } } - public function getBody() + public function getBody(): string { return $this->body; } - public function getCode() + public function getCode(): int { return $this->code; } @@ -226,7 +226,7 @@ final class Response return self::STATUS_CODES[$this->code] ?? ''; } - public function getHeaders() + public function getHeaders(): array { return $this->headers; } diff --git a/lib/php8backports.php b/lib/php8backports.php index 5b103e3d..ccef6016 100644 --- a/lib/php8backports.php +++ b/lib/php8backports.php @@ -1,39 +1,5 @@ <?php -/** - * This file is part of RSS-Bridge, a PHP project capable of generating RSS and - * Atom feeds for websites that don't have one. - * - * For the full license information, please view the UNLICENSE file distributed - * with this source code. - * - * @package Core - * @license http://unlicense.org/ UNLICENSE - * @link https://github.com/rss-bridge/rss-bridge - */ - -// based on https://github.com/laravel/framework/blob/8.x/src/Illuminate/Support/Str.php -// -// Copyright (c) Taylor Otwell -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is furnished -// to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - if (!function_exists('str_starts_with')) { function str_starts_with($haystack, $needle) { diff --git a/tests/Bridges/BridgeImplementationTest.php b/tests/BridgeImplementationTest.php similarity index 65% rename from tests/Bridges/BridgeImplementationTest.php rename to tests/BridgeImplementationTest.php index af9d7db1..d2f74931 100644 --- a/tests/Bridges/BridgeImplementationTest.php +++ b/tests/BridgeImplementationTest.php @@ -1,6 +1,6 @@ <?php -namespace RssBridge\Tests\Bridges; +namespace RssBridge\Tests; use BridgeAbstract; use FeedExpander; @@ -8,8 +8,8 @@ use PHPUnit\Framework\TestCase; class BridgeImplementationTest extends TestCase { - private $class; - private $obj; + private string $className; + private BridgeAbstract $bridge; /** * @dataProvider dataBridgesProvider @@ -17,9 +17,9 @@ class BridgeImplementationTest extends TestCase public function testClassName($path) { $this->setBridge($path); - $this->assertTrue($this->class === ucfirst($this->class), 'class name must start with uppercase character'); - $this->assertEquals(0, substr_count($this->class, ' '), 'class name must not contain spaces'); - $this->assertStringEndsWith('Bridge', $this->class, 'class name must end with "Bridge"'); + $this->assertTrue($this->className === ucfirst($this->className), 'class name must start with uppercase character'); + $this->assertEquals(0, substr_count($this->className, ' '), 'class name must not contain spaces'); + $this->assertStringEndsWith('Bridge', $this->className, 'class name must end with "Bridge"'); } /** @@ -28,7 +28,7 @@ class BridgeImplementationTest extends TestCase public function testClassType($path) { $this->setBridge($path); - $this->assertInstanceOf(BridgeAbstract::class, $this->obj); + $this->assertInstanceOf(BridgeAbstract::class, $this->bridge); } /** @@ -38,18 +38,18 @@ class BridgeImplementationTest extends TestCase { $this->setBridge($path); - $this->assertIsString($this->obj::NAME, 'class::NAME'); - $this->assertNotEmpty($this->obj::NAME, 'class::NAME'); - $this->assertIsString($this->obj::URI, 'class::URI'); - $this->assertNotEmpty($this->obj::URI, 'class::URI'); - $this->assertIsString($this->obj::DESCRIPTION, 'class::DESCRIPTION'); - $this->assertNotEmpty($this->obj::DESCRIPTION, 'class::DESCRIPTION'); - $this->assertIsString($this->obj::MAINTAINER, 'class::MAINTAINER'); - $this->assertNotEmpty($this->obj::MAINTAINER, 'class::MAINTAINER'); + $this->assertIsString($this->bridge::NAME, 'class::NAME'); + $this->assertNotEmpty($this->bridge::NAME, 'class::NAME'); + $this->assertIsString($this->bridge::URI, 'class::URI'); + $this->assertNotEmpty($this->bridge::URI, 'class::URI'); + $this->assertIsString($this->bridge::DESCRIPTION, 'class::DESCRIPTION'); + $this->assertNotEmpty($this->bridge::DESCRIPTION, 'class::DESCRIPTION'); + $this->assertIsString($this->bridge::MAINTAINER, 'class::MAINTAINER'); + $this->assertNotEmpty($this->bridge::MAINTAINER, 'class::MAINTAINER'); - $this->assertIsArray($this->obj::PARAMETERS, 'class::PARAMETERS'); - $this->assertIsInt($this->obj::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); - $this->assertGreaterThanOrEqual(0, $this->obj::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); + $this->assertIsArray($this->bridge::PARAMETERS, 'class::PARAMETERS'); + $this->assertIsInt($this->bridge::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); + $this->assertGreaterThanOrEqual(0, $this->bridge::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); } /** @@ -60,23 +60,22 @@ class BridgeImplementationTest extends TestCase $this->setBridge($path); $multiMinimum = 2; - if (isset($this->obj::PARAMETERS['global'])) { + if (isset($this->bridge::PARAMETERS['global'])) { ++$multiMinimum; } - $multiContexts = (count($this->obj::PARAMETERS) >= $multiMinimum); + $multiContexts = (count($this->bridge::PARAMETERS) >= $multiMinimum); $paramsSeen = []; $allowedTypes = [ 'text', 'number', 'list', - 'checkbox' + 'checkbox', ]; - foreach ($this->obj::PARAMETERS as $context => $params) { + foreach ($this->bridge::PARAMETERS as $context => $params) { if ($multiContexts) { $this->assertIsString($context, 'invalid context name'); - $this->assertNotEmpty($context, 'The context name cannot be empty'); } @@ -152,11 +151,10 @@ class BridgeImplementationTest extends TestCase } } - foreach ($this->obj::TEST_DETECT_PARAMETERS as $url => $params) { - $this->assertEquals($this->obj->detectParameters($url), $params); + foreach ($this->bridge::TEST_DETECT_PARAMETERS as $url => $params) { + $detectedParameters = $this->bridge->detectParameters($url); + $this->assertEquals($detectedParameters, $params); } - - $this->assertTrue(true); } /** @@ -164,19 +162,21 @@ class BridgeImplementationTest extends TestCase */ public function testVisibleMethods($path) { - $allowedBridgeAbstract = get_class_methods(BridgeAbstract::class); - sort($allowedBridgeAbstract); - $allowedFeedExpander = get_class_methods(FeedExpander::class); - sort($allowedFeedExpander); + $bridgeAbstractMethods = get_class_methods(BridgeAbstract::class); + sort($bridgeAbstractMethods); + $feedExpanderMethods = get_class_methods(FeedExpander::class); + sort($feedExpanderMethods); $this->setBridge($path); - $methods = get_class_methods($this->obj); - sort($methods); - if ($this->obj instanceof FeedExpander) { - $this->assertEquals($allowedFeedExpander, $methods); - } else { - $this->assertEquals($allowedBridgeAbstract, $methods); + $publicMethods = get_class_methods($this->bridge); + sort($publicMethods); + foreach ($publicMethods as $publicMethod) { + if ($this->bridge instanceof FeedExpander) { + $this->assertContains($publicMethod, $feedExpanderMethods); + } else { + $this->assertContains($publicMethod, $bridgeAbstractMethods); + } } } @@ -187,23 +187,23 @@ class BridgeImplementationTest extends TestCase { $this->setBridge($path); - $value = $this->obj->getDescription(); + $value = $this->bridge->getDescription(); $this->assertIsString($value, '$class->getDescription()'); $this->assertNotEmpty($value, '$class->getDescription()'); - $value = $this->obj->getMaintainer(); + $value = $this->bridge->getMaintainer(); $this->assertIsString($value, '$class->getMaintainer()'); $this->assertNotEmpty($value, '$class->getMaintainer()'); - $value = $this->obj->getName(); + $value = $this->bridge->getName(); $this->assertIsString($value, '$class->getName()'); $this->assertNotEmpty($value, '$class->getName()'); - $value = $this->obj->getURI(); + $value = $this->bridge->getURI(); $this->assertIsString($value, '$class->getURI()'); $this->assertNotEmpty($value, '$class->getURI()'); - $value = $this->obj->getIcon(); + $value = $this->bridge->getIcon(); $this->assertIsString($value, '$class->getIcon()'); } @@ -214,14 +214,14 @@ class BridgeImplementationTest extends TestCase { $this->setBridge($path); - $this->checkUrl($this->obj::URI); - $this->checkUrl($this->obj->getURI()); + $this->assertNotFalse(filter_var($this->bridge::URI, FILTER_VALIDATE_URL)); + $this->assertNotFalse(filter_var($this->bridge->getURI(), FILTER_VALIDATE_URL)); } public function dataBridgesProvider() { $bridges = []; - foreach (glob(__DIR__ . '/../../bridges/*Bridge.php') as $path) { + foreach (glob(__DIR__ . '/../bridges/*Bridge.php') as $path) { $bridges[basename($path, '.php')] = [$path]; } return $bridges; @@ -229,16 +229,11 @@ class BridgeImplementationTest extends TestCase private function setBridge($path) { - $this->class = '\\' . basename($path, '.php'); - $this->assertTrue(class_exists($this->class), 'class ' . $this->class . ' doesn\'t exist'); - $this->obj = new $this->class( + $this->className = '\\' . basename($path, '.php'); + $this->assertTrue(class_exists($this->className), 'class ' . $this->className . ' doesn\'t exist'); + $this->bridge = new $this->className( new \NullCache(), - new \NullLogger() + new \NullLogger(), ); } - - private function checkUrl($url) - { - $this->assertNotFalse(filter_var($url, FILTER_VALIDATE_URL), 'no valid URL: ' . $url); - } } diff --git a/tests/CacheImplementationTest.php b/tests/CacheImplementationTest.php new file mode 100644 index 00000000..e6ed352b --- /dev/null +++ b/tests/CacheImplementationTest.php @@ -0,0 +1,36 @@ +<?php + +namespace RssBridge\Tests; + +use CacheInterface; +use PHPUnit\Framework\TestCase; + +class CacheImplementationTest extends TestCase +{ + public function getCacheClassNames() + { + $caches = []; + foreach (glob(PATH_LIB_CACHES . '*.php') as $path) { + $caches[] = [basename($path, '.php')]; + } + return $caches; + } + + /** + * @dataProvider getCacheClassNames + */ + public function testClassName($path) + { + $this->assertTrue($path === ucfirst($path), 'class name must start with uppercase character'); + $this->assertEquals(0, substr_count($path, ' '), 'class name must not contain spaces'); + $this->assertStringEndsWith('Cache', $path, 'class name must end with "Cache"'); + } + + /** + * @dataProvider getCacheClassNames + */ + public function testClassType($path) + { + $this->assertTrue(is_subclass_of($path, CacheInterface::class), 'class must be subclass of CacheInterface'); + } +} diff --git a/tests/Caches/CacheImplementationTest.php b/tests/Caches/CacheImplementationTest.php deleted file mode 100644 index a3ad5f79..00000000 --- a/tests/Caches/CacheImplementationTest.php +++ /dev/null @@ -1,48 +0,0 @@ -<?php - -namespace RssBridge\Tests\Caches; - -use CacheInterface; -use PHPUnit\Framework\TestCase; - -class CacheImplementationTest extends TestCase -{ - private $class; - - /** - * @dataProvider dataCachesProvider - */ - public function testClassName($path) - { - $this->setCache($path); - $this->assertTrue($this->class === ucfirst($this->class), 'class name must start with uppercase character'); - $this->assertEquals(0, substr_count($this->class, ' '), 'class name must not contain spaces'); - $this->assertStringEndsWith('Cache', $this->class, 'class name must end with "Cache"'); - } - - /** - * @dataProvider dataCachesProvider - */ - public function testClassType($path) - { - $this->setCache($path); - $this->assertTrue(is_subclass_of($this->class, CacheInterface::class), 'class must be subclass of CacheInterface'); - } - - //////////////////////////////////////////////////////////////////////////// - - public function dataCachesProvider() - { - $caches = []; - foreach (glob(PATH_LIB_CACHES . '*.php') as $path) { - $caches[basename($path, '.php')] = [$path]; - } - return $caches; - } - - private function setCache($path) - { - $this->class = '\\' . basename($path, '.php'); - $this->assertTrue(class_exists($this->class), 'class ' . $this->class . ' doesn\'t exist'); - } -} diff --git a/tests/ParameterValidatorTest.php b/tests/ParameterValidatorTest.php new file mode 100644 index 00000000..1b241c2c --- /dev/null +++ b/tests/ParameterValidatorTest.php @@ -0,0 +1,40 @@ +<?php + +declare(strict_types=1); + +namespace RssBridge\Tests; + +use PHPUnit\Framework\TestCase; + +class ParameterValidatorTest extends TestCase +{ + public function test1() + { + $sut = new \ParameterValidator(); + $input = ['user' => 'joe']; + $parameters = [ + [ + 'user' => [ + 'name' => 'User', + 'type' => 'text', + ], + ] + ]; + $this->assertTrue($sut->validateInput($input, $parameters)); + } + + public function test2() + { + $sut = new \ParameterValidator(); + $input = ['username' => 'joe']; + $parameters = [ + [ + 'user' => [ + 'name' => 'User', + 'type' => 'text', + ], + ] + ]; + $this->assertFalse($sut->validateInput($input, $parameters)); + } +} From 69da0dd5830e1e8a48e4b22066ae645e790a5a9e Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Sun, 1 Oct 2023 20:46:51 +0200 Subject: [PATCH 147/716] [refactoring] replace direct use of curl with getContents (#3723) + some fixed warnings --- bridges/DealabsBridge.php | 1 + bridges/FDroidBridge.php | 28 +++++------------------ bridges/HotUKDealsBridge.php | 1 + bridges/MydealsBridge.php | 1 + bridges/PepperBridgeAbstract.php | 39 ++++++++------------------------ lib/contents.php | 6 ++++- 6 files changed, 24 insertions(+), 52 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index 6e5ba9e3..a904c3ff 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -1909,6 +1909,7 @@ class DealabsBridge extends PepperBridgeAbstract 'context-group' => 'Deals par groupe', 'context-talk' => 'Surveillance Discussion', 'uri-group' => 'groupe/', + 'uri-deal' => 'bons-plans/', 'request-error' => 'Impossible de joindre Dealabs', 'thread-error' => 'Impossible de déterminer l\'ID de la discussion. Vérifiez l\'URL que vous avez entré', 'no-results' => 'Il n'y a rien à afficher pour le moment :(', diff --git a/bridges/FDroidBridge.php b/bridges/FDroidBridge.php index 099a4121..8d3b7808 100644 --- a/bridges/FDroidBridge.php +++ b/bridges/FDroidBridge.php @@ -21,34 +21,18 @@ class FDroidBridge extends BridgeAbstract public function getIcon() { - return self::URI . 'assets/favicon.ico?v=8j6PKzW9Mk'; + return self::URI . 'assets/favicon.ico'; } private function getTimestamp($url) { $curlOptions = [ - CURLOPT_RETURNTRANSFER => true, - CURLOPT_HEADER => true, - CURLOPT_NOBODY => true, - CURLOPT_CONNECTTIMEOUT => 19, - CURLOPT_TIMEOUT => 19, + CURLOPT_CUSTOMREQUEST => 'HEAD', + CURLOPT_NOBODY => true, ]; - $ch = curl_init($url); - curl_setopt_array($ch, $curlOptions); - $curlHeaders = curl_exec($ch); - $curlError = curl_error($ch); - curl_close($ch); - if (!empty($curlError)) { - return false; - } - $curlHeaders = explode("\n", $curlHeaders); - $timestamp = false; - foreach ($curlHeaders as $header) { - if (strpos($header, 'Last-Modified') !== false) { - $timestamp = str_replace('Last-Modified: ', '', $header); - $timestamp = strtotime($timestamp); - } - } + $reponse = getContents($url, [], $curlOptions, true); + $lastModified = $reponse['headers']['last-modified'][0] ?? null; + $timestamp = strtotime($lastModified ?? 'today'); return $timestamp; } diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index 7328ca04..fe4ef507 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3273,6 +3273,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'context-group' => 'Deals per group', 'context-talk' => 'Discussion Monitoring', 'uri-group' => 'tag/', + 'uri-deal' => 'deals/', 'request-error' => 'Could not request HotUKDeals', 'thread-error' => 'Unable to determine the thread ID. Check the URL you entered', 'no-results' => 'Ooops, looks like we could', diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 257ef561..0ef9c201 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2020,6 +2020,7 @@ class MydealsBridge extends PepperBridgeAbstract 'context-group' => 'Deals pro Gruppen', 'context-talk' => 'Überwachung Diskussion', 'uri-group' => 'gruppe/', + 'uri-deal' => 'deals/', 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', 'no-results' => 'Ups, wir konnten keine Deals zu', diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 280b185d..4e520acc 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -165,7 +165,7 @@ class PepperBridgeAbstract extends BridgeAbstract $url = $this->i8n('bridge-uri') . 'graphql'; // Get Cookies header to do the query - $cookies = $this->getCookies($url); + $cookiesHeaderValue = $this->getCookiesHeaderValue($url); // GraphQL String // This was extracted from https://www.dealabs.com/assets/js/modern/common_211b99.js @@ -209,7 +209,7 @@ HEREDOC; 'X-Pepper-Txn: threads.show', 'X-Request-Type: application/vnd.pepper.v1+json', 'X-Requested-With: XMLHttpRequest', - $cookies, + "Cookie: $cookiesHeaderValue", ]; // CURL Options $opts = [ @@ -245,26 +245,12 @@ HEREDOC; * Extract the cookies obtained from the URL * @return array the array containing the cookies set by the URL */ - private function getCookies($url) + private function getCookiesHeaderValue($url) { - $ch = curl_init($url); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); - // get headers too with this line - curl_setopt($ch, CURLOPT_HEADER, 1); - $result = curl_exec($ch); - // get cookie - // multi-cookie variant contributed by @Combuster in comments - preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $result, $matches); - $cookies = []; - foreach ($matches[1] as $item) { - parse_str($item, $cookie); - $cookies = array_merge($cookies, $cookie); - } - $header = 'Cookie: '; - foreach ($cookies as $name => $content) { - $header .= $name . '=' . $content . '; '; - } - return $header; + $response = getContents($url, [], [], true); + $setCookieHeaders = $response['headers']['set-cookie'] ?? []; + $cookies = array_map(fn($c): string => explode(';', $c)[0], $setCookieHeaders); + return implode('; ', $cookies); } /** @@ -330,7 +316,7 @@ HEREDOC; private function getTalkTitle() { $html = getSimpleHTMLDOMCached($this->getInput('url')); - $title = $html->find('h1[class=thread-title]', 0)->plaintext; + $title = $html->find('.thread-title', 0)->plaintext; return $title; } @@ -356,13 +342,8 @@ HEREDOC; */ private function getDealURI($deal) { - $uriA = $deal->find('div[class*=threadGrid-title]', 0)->find('a[class*=thread-link]', 0); - if ($uriA === null) { - $uri = ''; - } else { - $uri = $uriA->href; - } - + $dealId = $deal->attr['id']; + $uri = $this->i8n('bridge-uri') . $this->i8n('uri-deal') . str_replace('_', '-', $dealId); return $uri; } diff --git a/lib/contents.php b/lib/contents.php index 432d9139..27f0af31 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -52,7 +52,11 @@ function getContents( $config['proxy'] = Configuration::getConfig('proxy', 'url'); } - $cacheKey = 'server_' . $url; + $requestBodyHash = null; + if (isset($curlOptions[CURLOPT_POSTFIELDS])) { + $requestBodyHash = md5(json_encode($curlOptions[CURLOPT_POSTFIELDS])); + } + $cacheKey = implode('_', ['server', $url, $requestBodyHash]); /** @var Response $cachedResponse */ $cachedResponse = $cache->get($cacheKey); From 547af0d0d2fe70ae4a54ab238b970750c1463301 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 1 Oct 2023 20:54:28 +0200 Subject: [PATCH 148/716] refactor: use Json::encode instead of json_encode (#3724) --- lib/contents.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/contents.php b/lib/contents.php index 27f0af31..cfb9f36a 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -54,7 +54,7 @@ function getContents( $requestBodyHash = null; if (isset($curlOptions[CURLOPT_POSTFIELDS])) { - $requestBodyHash = md5(json_encode($curlOptions[CURLOPT_POSTFIELDS])); + $requestBodyHash = md5(Json::encode($curlOptions[CURLOPT_POSTFIELDS], false)); } $cacheKey = implode('_', ['server', $url, $requestBodyHash]); From 64582a64f13ba4d7ba160ad6f57199d7ea010f42 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Sun, 1 Oct 2023 21:19:27 +0200 Subject: [PATCH 149/716] fix(tpb): add category (#3725) --- bridges/ThePirateBayBridge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/bridges/ThePirateBayBridge.php b/bridges/ThePirateBayBridge.php index 4b130800..fd4a31ac 100644 --- a/bridges/ThePirateBayBridge.php +++ b/bridges/ThePirateBayBridge.php @@ -91,6 +91,7 @@ class ThePirateBayBridge extends BridgeAbstract '504' => 'Games', '505' => 'HD-Movies', '506' => 'Movie Clips', + '507' => 'UHD/4k Movies', '599' => 'Other', '601' => 'E-books', '602' => 'Comics', From 59dd49671d4b301f64cc1299c3156f6a31d5fd00 Mon Sep 17 00:00:00 2001 From: User123698745 <User123698745@users.noreply.github.com> Date: Mon, 2 Oct 2023 03:02:57 +0200 Subject: [PATCH 150/716] [BridgeCard] add example value to info hint and allow using it by right click (#3726) --- lib/BridgeCard.php | 13 +++++++++++-- static/rss-bridge.js | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index b2fda192..4781ebc1 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -177,9 +177,18 @@ This bridge is not fetching its content through a secure connection</div>'; $form .= self::getCheckboxInput($inputEntry, $idArg, $id); } + $infoText = []; + $infoTextScript = ''; if (isset($inputEntry['title'])) { - $title_filtered = filter_var($inputEntry['title'], FILTER_SANITIZE_FULL_SPECIAL_CHARS); - $form .= '<i class="info" title="' . $title_filtered . '">i</i>'; + $infoText[] = filter_var($inputEntry['title'], FILTER_SANITIZE_FULL_SPECIAL_CHARS); + } + if ($inputEntry['exampleValue'] !== '') { + $infoText[] = "Example (right click to use):\n" . filter_var($inputEntry['exampleValue'], FILTER_SANITIZE_FULL_SPECIAL_CHARS); + $infoTextScript = 'rssbridge_use_placeholder_value(this);'; + } + + if (count($infoText) > 0) { + $form .= '<i class="info" data-for="' . $idArg . '" title="' . implode("\n\n", $infoText) . '" oncontextmenu="' . $infoTextScript . 'return false">i</i>'; } else { $form .= '<i class="no-info"></i>'; } diff --git a/static/rss-bridge.js b/static/rss-bridge.js index 2c45294c..b9b466d6 100644 --- a/static/rss-bridge.js +++ b/static/rss-bridge.js @@ -48,6 +48,12 @@ function rssbridge_toggle_bridge(){ } } +function rssbridge_use_placeholder_value(sender) { + let inputId = sender.getAttribute('data-for'); + let inputElement = document.getElementById(inputId); + inputElement.value = inputElement.getAttribute("placeholder"); +} + var rssbridge_feed_finder = (function() { /* * Code for "Find feed by URL" feature From 1cbe1a6f98ec536be7f894ebcc255d7417c0060b Mon Sep 17 00:00:00 2001 From: sysadminstory <sysadminstory@users.noreply.github.com> Date: Tue, 3 Oct 2023 23:15:10 +0200 Subject: [PATCH 151/716] [PepperBridge] Fix date parsing (#3727) Website changed the date display. This fix adapt the date parsing to the new website date display --- bridges/HotUKDealsBridge.php | 1 + bridges/PepperBridgeAbstract.php | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index fe4ef507..69301c42 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3316,6 +3316,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'and ' ], 'date-prefixes' => [ + 'Posted ', 'Found ', 'Refreshed ', 'Made hot ' diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 4e520acc..5d2e552b 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -128,9 +128,9 @@ class PepperBridgeAbstract extends BridgeAbstract $clock = end($clocks); // Find the text corresponding to the clock - $spanDateDiv = $clock->parent()->find('span[class=hide--toW3]', 0); - $itemDate = $spanDateDiv->plaintext ?? ''; - // In case of a Local deal, there is no date, but we can use + $spanDateDiv = $clock->next_sibling(); + $itemDate = $spanDateDiv->plaintext; + // In some case of a Local deal, there is no date, but we can use // this case for other reason (like date not in the last field) if ($this->contains($itemDate, $this->i8n('localdeal'))) { $item['timestamp'] = time(); From e376805249dcc16fe3730371e9c3858967681ecf Mon Sep 17 00:00:00 2001 From: Niehztog <Niehztog@users.noreply.github.com> Date: Thu, 5 Oct 2023 02:31:04 +0200 Subject: [PATCH 152/716] [NiusBridge] fix parse error, fix image content-type (#3728) --- bridges/NiusBridge.php | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bridges/NiusBridge.php b/bridges/NiusBridge.php index 05d0dacc..c76b29f0 100644 --- a/bridges/NiusBridge.php +++ b/bridges/NiusBridge.php @@ -12,29 +12,31 @@ class NiusBridge extends XPathAbstract const FEED_SOURCE_URL = 'https://www.nius.de/news'; const XPATH_EXPRESSION_ITEM = './/div[contains(@class, "compact-story") or contains(@class, "regular-story")]'; const XPATH_EXPRESSION_ITEM_TITLE = './/h2[@class="title"]//node()'; - const XPATH_EXPRESSION_ITEM_CONTENT = './/h2[@class="title"]//node()'; + const XPATH_EXPRESSION_ITEM_CONTENT = self::XPATH_EXPRESSION_ITEM_TITLE; const XPATH_EXPRESSION_ITEM_URI = './/a[1]/@href'; - const XPATH_EXPRESSION_ITEM_AUTHOR = 'normalize-space(.//span[@class="author"]/text()[3])'; + const XPATH_EXPR_AUTHOR_PART1 = 'normalize-space(.//span[@class="author"]/text()[1])'; + const XPATH_EXPR_AUTHOR_PART2 = 'normalize-space(.//span[@class="author"]/text()[2])'; + const XPATH_EXPRESSION_ITEM_AUTHOR = 'substring-after(concat(' . self::XPATH_EXPR_AUTHOR_PART1 . ', " ", ' . self::XPATH_EXPR_AUTHOR_PART2 . '), " ")'; - const XPATH_EXPRESSION_ITEM_TIMESTAMP = 'normalize-space(.//span[@class="author"]/text()[1])'; - const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img[@sizes]/@src'; + const XPATH_EXPRESSION_ITEM_ENCLOSURES = './/img[@sizes and @alt="Article background picture"]/@src'; const XPATH_EXPRESSION_ITEM_CATEGORIES = './/div[@class="subtitle"]/text()'; const SETTING_FIX_ENCODING = false; - protected function formatItemTimestamp($value) + protected function formatItemTitle($value) { - return DateTimeImmutable::createFromFormat( - false !== strpos($value, ' Uhr') ? 'H:i \U\h\r' : 'd.m.y', - $value, - new DateTimeZone('Europe/Berlin') - )->format('U'); + return strip_tags($value); + } + + protected function formatItemContent($value) + { + return strip_tags($value); } protected function cleanMediaUrl($mediaUrl) { $result = preg_match('~https:\/\/www\.nius\.de\/_next\/image\?url=(.*)\?~', $mediaUrl, $matches); - return $result ? $matches[1] : $mediaUrl; + return $result ? $matches[1] . '#.jpg' : $mediaUrl; } protected function generateItemId(FeedItem $item) From 5f777d4126fe79c2095a0716a2ba070e51173be0 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Thu, 5 Oct 2023 15:36:35 +0200 Subject: [PATCH 153/716] fix(codeberg): add temp fix (#3730) they changed html for tag and commit --- bridges/CodebergBridge.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bridges/CodebergBridge.php b/bridges/CodebergBridge.php index 9515d514..2a450477 100644 --- a/bridges/CodebergBridge.php +++ b/bridges/CodebergBridge.php @@ -403,6 +403,9 @@ EOD; */ private function stripSvg($html) { + if ($html === null) { + return null; + } if ($html->find('svg', 0)) { $html->find('svg', 0)->outertext = ''; } From f97a3fa4d9f0bbd5ce1b4541518664c21761b368 Mon Sep 17 00:00:00 2001 From: Park0 <Park0@users.noreply.github.com> Date: Mon, 9 Oct 2023 08:46:24 +0200 Subject: [PATCH 154/716] Fia.com document bridge (#3733) * Create FiaBridge.php F1 documents from fia.com * Update FiaBridge.php Fixed concat --- bridges/FiaBridge.php | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 bridges/FiaBridge.php diff --git a/bridges/FiaBridge.php b/bridges/FiaBridge.php new file mode 100644 index 00000000..37d26267 --- /dev/null +++ b/bridges/FiaBridge.php @@ -0,0 +1,43 @@ +<?php + +class FiaBridge extends BridgeAbstract +{ + const NAME = 'Federation Internationale de l\'Automobile site feed'; + const URI = 'https://fia.com'; + const DESCRIPTION = 'Get the latest F1 documents from the fia site'; + const PARAMETERS = []; + const CACHE_TIMEOUT = 900; + + public function collectData() + { + $url = 'https://www.fia.com/documents/championships/fia-formula-one-world-championship-14/'; + $html = getSimpleHTMLDOM($url); + $items = $html->find('li.document-row'); + foreach ($items as $item) { + /** @var simple_html_dom $item */ + // Do something with each list item + $title = trim($item->find('div.title', 0)->plaintext); + $href = $item->find('a', 0)->href; + $url = 'https://www.fia.com' . $href; + + $date = $item->find('span.date-display-single', 0)->plaintext; + + $item = []; + $item['uri'] = $url; + $item['title'] = $title; + $item['timestamp'] = (string) DateTime::createFromFormat('d.m.y H:i', $date)->getTimestamp(); + ; + $item['author'] = 'Fia'; + $item['content'] = "Document on date $date: $title <br /><a href='$url'>$url</a>"; + $item['categories'] = 'Document'; + $item['uid'] = $title . $date; + + $count = count($this->items); + if ($count > 20) { + break; + } else { + $this->items[] = $item; + } + } + } +} From 47f52b5912f442389e9b5a867044efc6f4680b7b Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Mon, 9 Oct 2023 08:48:21 +0200 Subject: [PATCH 155/716] Add CSS Selector Feed Expander (#3732) * Add CSS Selector Feed Expander This bridge combines CssSelectorBridge with FeedExpander Allows expanding a feed using CSS selectors * Fix code linting --------- Co-authored-by: ORelio <ORelio> --- bridges/CssSelectorBridge.php | 9 ++- bridges/CssSelectorFeedExpanderBridge.php | 98 +++++++++++++++++++++++ bridges/SitemapBridge.php | 6 +- 3 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 bridges/CssSelectorFeedExpanderBridge.php diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index dd8fe228..f6ab8d15 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -60,11 +60,12 @@ class CssSelectorBridge extends BridgeAbstract ] ]; - private $feedName = ''; + protected $feedName = ''; + protected $homepageUrl = ''; public function getURI() { - $url = $this->getInput('home_page'); + $url = $this->homepageUrl; if (empty($url)) { $url = parent::getURI(); } @@ -81,7 +82,7 @@ class CssSelectorBridge extends BridgeAbstract public function collectData() { - $url = $this->getInput('home_page'); + $this->homepageUrl = $this->getInput('home_page'); $url_selector = $this->getInput('url_selector'); $url_pattern = $this->getInput('url_pattern'); $content_selector = $this->getInput('content_selector'); @@ -90,7 +91,7 @@ class CssSelectorBridge extends BridgeAbstract $discard_thumbnail = $this->getInput('discard_thumbnail'); $limit = $this->getInput('limit') ?? 10; - $html = defaultLinkTo(getSimpleHTMLDOM($url), $url); + $html = defaultLinkTo(getSimpleHTMLDOM($this->homepageUrl), $this->homepageUrl); $this->feedName = $this->titleCleanup($this->getPageTitle($html), $title_cleanup); $items = $this->htmlFindEntries($html, $url_selector, $url_pattern, $limit, $content_cleanup); diff --git a/bridges/CssSelectorFeedExpanderBridge.php b/bridges/CssSelectorFeedExpanderBridge.php new file mode 100644 index 00000000..7e1f630f --- /dev/null +++ b/bridges/CssSelectorFeedExpanderBridge.php @@ -0,0 +1,98 @@ +<?php + +if (!class_exists('CssSelectorFeedExpanderBridgeInternal')) { + // Utility class used internally by CssSelectorFeedExpanderBridge + class CssSelectorFeedExpanderBridgeInternal extends FeedExpander + { + public function collectData() + { + // Unused. Call collectExpandableDatas($url) inherited from FeedExpander instead + } + } +} + +class CssSelectorFeedExpanderBridge extends CssSelectorBridge +{ + const MAINTAINER = 'ORelio'; + const NAME = 'CSS Selector Feed Expander'; + const URI = 'https://github.com/RSS-Bridge/rss-bridge/'; + const DESCRIPTION = 'Expand any site RSS feed using CSS selectors (Advanced Users)'; + const PARAMETERS = [ + [ + 'feed' => [ + 'name' => 'Feed: URL of truncated RSS feed', + 'exampleValue' => 'https://example.com/feed.xml', + 'required' => true + ], + 'content_selector' => [ + 'name' => 'Selector for each article content', + 'title' => <<<EOT + This bridge works using CSS selectors, e.g. "div.article" will match <div class="article">. + Everything inside that element becomes feed item content. + EOT, + 'exampleValue' => 'article.content', + 'required' => true + ], + 'content_cleanup' => [ + 'name' => '[Optional] Content cleanup: List of items to remove', + 'title' => 'Selector for unnecessary elements to remove inside article contents.', + 'exampleValue' => 'div.ads, div.comments', + ], + 'dont_expand_metadata' => [ + 'name' => '[Optional] Don\'t expand metadata', + 'title' => "This bridge will attempt to fill missing fields using metadata from the webpage.\nCheck to disable.", + 'type' => 'checkbox', + ], + 'discard_thumbnail' => [ + 'name' => '[Optional] Discard thumbnail set by site author', + 'title' => 'Some sites set their logo as thumbnail for every article. Use this option to discard it.', + 'type' => 'checkbox', + ], + 'limit' => self::LIMIT + ] + ]; + + public function collectData() + { + $url = $this->getInput('feed'); + $content_selector = $this->getInput('content_selector'); + $content_cleanup = $this->getInput('content_cleanup'); + $dont_expand_metadata = $this->getInput('dont_expand_metadata'); + $discard_thumbnail = $this->getInput('discard_thumbnail'); + $limit = $this->getInput('limit'); + + $feed_expander = new CssSelectorFeedExpanderBridgeInternal(); + $items = $feed_expander->collectExpandableDatas($url)->getItems(); + + $this->homepageUrl = urljoin($url, '/'); + $this->feedName = $feed_expander->getName(); + + foreach ($items as $item_from_feed) { + $item_expanded = $this->expandEntryWithSelector( + $item_from_feed['uri'], + $content_selector, + $content_cleanup + ); + + if ($dont_expand_metadata) { + // Take feed item, only replace content from expanded data + $content = $item_expanded['content']; + $item_expanded = $item_from_feed; + $item_expanded['content'] = $content; + } else { + // Take expanded item, but give priority to metadata already in source item + foreach ($item_from_feed as $field => $val) { + if ($field !== 'content') { + $item_expanded[$field] = $val; + } + } + } + + if ($discard_thumbnail && isset($item_expanded['enclosures'])) { + unset($item_expanded['enclosures']); + } + + $this->items[] = $item_expanded; + } + } +} diff --git a/bridges/SitemapBridge.php b/bridges/SitemapBridge.php index 78526e6e..bdf662ee 100644 --- a/bridges/SitemapBridge.php +++ b/bridges/SitemapBridge.php @@ -64,7 +64,7 @@ class SitemapBridge extends CssSelectorBridge public function collectData() { - $url = $this->getInput('home_page'); + $this->homepageUrl = $this->getInput('home_page'); $url_pattern = $this->getInput('url_pattern'); $content_selector = $this->getInput('content_selector'); $content_cleanup = $this->getInput('content_cleanup'); @@ -73,8 +73,8 @@ class SitemapBridge extends CssSelectorBridge $discard_thumbnail = $this->getInput('discard_thumbnail'); $limit = $this->getInput('limit'); - $this->feedName = $this->titleCleanup($this->getPageTitle($url), $title_cleanup); - $sitemap_url = empty($site_map) ? $url : $site_map; + $this->feedName = $this->titleCleanup($this->getPageTitle($this->homepageUrl), $title_cleanup); + $sitemap_url = empty($site_map) ? $this->homepageUrl : $site_map; $sitemap_xml = $this->getSitemapXml($sitemap_url, !empty($site_map)); $links = $this->sitemapXmlToList($sitemap_xml, $url_pattern, empty($limit) ? 10 : $limit); From 143f90da604929e642184a6f830574266b8ac0e7 Mon Sep 17 00:00:00 2001 From: ORelio <ORelio@users.noreply.github.com> Date: Tue, 10 Oct 2023 19:34:16 +0200 Subject: [PATCH 156/716] [WeLiveSecurity] Fix content extraction (#3734) --- bridges/WeLiveSecurityBridge.php | 35 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/bridges/WeLiveSecurityBridge.php b/bridges/WeLiveSecurityBridge.php index 6434a13a..f54f6b29 100644 --- a/bridges/WeLiveSecurityBridge.php +++ b/bridges/WeLiveSecurityBridge.php @@ -16,19 +16,36 @@ class WeLiveSecurityBridge extends FeedExpander { $item = parent::parseItem($item); - $article_html = getSimpleHTMLDOMCached($item['uri']); - if (!$article_html) { - $item['content'] .= '<p><em>Could not request ' . $this->getName() . ': ' . $item['uri'] . '</em></p>'; + $html = getSimpleHTMLDOMCached($item['uri']); + if (!$html) { + $item['content'] .= '<br /><p><em>Could not request ' . $this->getName() . ': ' . $item['uri'] . '</em></p>'; return $item; } - $article_content = $article_html->find('div.formatted', 0)->innertext; - $article_content = stripWithDelimiters($article_content, '<script', '</script>'); - $article_content = stripRecursiveHTMLSection($article_content, 'div', '<div class="comments'); - $article_content = stripRecursiveHTMLSection($article_content, 'div', '<div class="similar-articles'); - $article_content = stripRecursiveHTMLSection($article_content, 'span', '<span class="meta'); - $item['content'] = trim($article_content); + $html = $html->find('.article-page', 0); + $content_html = $html->find('.article-body', 0); + // Remove social media footer + foreach ($content_html->find('blockquote') as $blockquote) { + if (str_starts_with(trim($blockquote->plaintext), 'Connect with us on')) { + $blockquote->outertext = ''; + } + } + + // Headline subtitle + $content = $content_html->innertext; + $subtitle = $html->find('.sub-title', 0); + if ($subtitle) { + $content = '<p><b>' . $subtitle->plaintext . '</b></p>' . $content; + } + + // Author + $author = $html->find('.article-author', 0); + if ($author && !isset($item['author'])) { + $item['author'] = trim($author->plaintext); + } + + $item['content'] = trim($content); return $item; } From b6a9baff9494d4f7820b640848afa02492a6f865 Mon Sep 17 00:00:00 2001 From: Dag <me@dvikan.no> Date: Tue, 10 Oct 2023 21:41:57 +0200 Subject: [PATCH 157/716] fix(cvedetails,tldrtech) (#3735) --- bridges/CVEDetailsBridge.php | 131 ++++++++++++++++------------------- bridges/TldrTechBridge.php | 20 ++++-- lib/http.php | 1 + lib/utils.php | 2 +- 4 files changed, 75 insertions(+), 79 deletions(-) diff --git a/bridges/CVEDetailsBridge.php b/bridges/CVEDetailsBridge.php index 5334c170..27b4008c 100644 --- a/bridges/CVEDetailsBridge.php +++ b/bridges/CVEDetailsBridge.php @@ -36,12 +36,65 @@ class CVEDetailsBridge extends BridgeAbstract private $vendor = ''; private $product = ''; - // Return the URL to query. - // Because of the optional product ID, we need to attach it if it is - // set. The search result page has the exact same structure (with and - // without the product ID). - private function buildUrl() + public function collectData() { + if ($this->html == null) { + $this->fetchContent(); + } + + foreach ($this->html->find('#searchresults > .row') as $i => $tr) { + // There are some optional vulnerability types, which will be + // added to the categories as well as the CWE number -- which is + // always given. + $categories = [$this->vendor]; + $enclosures = []; + + $detailLink = $tr->find('h3 > a', 0); + $detailHtml = getSimpleHTMLDOM($detailLink->href); + + // The CVE number itself + $title = $tr->find('h3 > a', 0)->innertext; + $content = $tr->find('.cvesummarylong', 0)->innertext; + $cweList = $detailHtml->find('h2', 2)->next_sibling(); + foreach ($cweList->find('li') as $li) { + $cweWithDescription = $li->find('a', 0)->innertext; + + if (preg_match('/CWE-(\d+)/', $cweWithDescription, $cwe)) { + $categories[] = 'CWE-' . $cwe[1]; + $enclosures[] = 'https://cwe.mitre.org/data/definitions/' . $cwe[1] . '.html'; + } + } + + if ($this->product != '') { + $categories[] = $this->product; + } + + $this->items[] = [ + 'uri' => 'https://cvedetails.com/' . $detailHtml->find('h1 > a', 0)->href, + 'title' => $title, + 'timestamp' => $tr->find('[data-tsvfield="publishDate"]', 0)->innertext, + 'content' => $content, + 'categories' => $categories, + 'enclosures' => $enclosures, + 'uid' => $title, + ]; + + // We only want to fetch the latest 10 CVEs + if (count($this->items) >= 10) { + break; + } + } + } + + // Make the actual request to cvedetails.com and stores the response + // (HTML) for later use and extract vendor and product from it. + private function fetchContent() + { + // build url + // Return the URL to query. + // Because of the optional product ID, we need to attach it if it is + // set. The search result page has the exact same structure (with and + // without the product ID). $url = self::URI . '/vulnerability-list/vendor_id-' . $this->getInput('vendor_id'); if ($this->getInput('product_id') !== '') { $url .= '/product_id-' . $this->getInput('product_id'); @@ -51,22 +104,12 @@ class CVEDetailsBridge extends BridgeAbstract // number, which should be mostly accurate. $url .= '?order=1'; // Order by CVE number DESC - return $url; - } - - // Make the actual request to cvedetails.com and stores the response - // (HTML) for later use and extract vendor and product from it. - private function fetchContent() - { - $html = getSimpleHTMLDOM($this->buildUrl()); + $html = getSimpleHTMLDOM($url); $this->html = defaultLinkTo($html, self::URI); $vendor = $html->find('#contentdiv h1 > a', 0); if ($vendor == null) { - returnServerError('Invalid Vendor ID ' . - $this->getInput('vendor_id') . - ' or Product ID ' . - $this->getInput('product_id')); + returnServerError('Invalid Vendor ID ' . $this->getInput('vendor_id') . ' or Product ID ' . $this->getInput('product_id')); } $this->vendor = $vendor->innertext; @@ -76,7 +119,6 @@ class CVEDetailsBridge extends BridgeAbstract } } - // Build the name of the feed. public function getName() { if ($this->getInput('vendor_id') == '') { @@ -94,57 +136,4 @@ class CVEDetailsBridge extends BridgeAbstract return $name; } - - // Pull the data from the HTML response and fill the items.. - public function collectData() - { - if ($this->html == null) { - $this->fetchContent(); - } - - foreach ($this->html->find('#searchresults > .row') as $i => $tr) { - // There are some optional vulnerability types, which will be - // added to the categories as well as the CWE number -- which is - // always given. - $categories = [$this->vendor]; - $enclosures = []; - - $detailLink = $tr->find('.cveheader > h3 > a', 0); - $detailHtml = getSimpleHTMLDOM($detailLink->href); - - $div = $detailHtml->find('.cvedetailssummary', 0); - - // The CVE number itself - $title = $div->find('h1 > a', 0)->innertext; - $content = $div->find('.ssc-paragraph', 0)->innertext; - $cweList = $detailHtml->find('h2', 2)->next_sibling(); - foreach ($cweList->find('li') as $li) { - $cweWithDescription = $li->find('a', 0)->innertext; - preg_match('/CWE-(\d+)/', $cweWithDescription, $cwe); - if (count($cwe) > 1) { - $categories[] = 'CWE-' . $cwe[1]; - $enclosures[] = 'https://cwe.mitre.org/data/definitions/' . $cwe[1] . '.html'; - } - } - - if ($this->product != '') { - $categories[] = $this->product; - } - - $this->items[] = [ - 'uri' => 'https://cvedetails.com/' . $detailHtml->find('h1 > a', 0)->href, - 'title' => $title, - 'timestamp' => $tr->find('td', 5)->innertext, - 'content' => $content, - 'categories' => $categories, - 'enclosures' => $enclosures, - 'uid' => $title, - ]; - - // We only want to fetch the latest 10 CVEs - if (count($this->items) >= 10) { - break; - } - } - } } diff --git a/bridges/TldrTechBridge.php b/bridges/TldrTechBridge.php index b89686bb..984117b2 100644 --- a/bridges/TldrTechBridge.php +++ b/bridges/TldrTechBridge.php @@ -35,7 +35,10 @@ class TldrTechBridge extends BridgeAbstract public function collectData() { - $html = getSimpleHTMLDOM(self::URI . $this->getInput('topic') . '/archives'); + $topic = $this->getInput('topic'); + $limit = $this->getInput('limit'); + $url = self::URI . $topic . '/archives'; + $html = getSimpleHTMLDOM($url); $entries_root = $html->find('div.content-center.mt-5', 0); $added = 0; foreach ($entries_root->children() as $child) { @@ -46,22 +49,25 @@ class TldrTechBridge extends BridgeAbstract $date_items = explode('/', $child->href); $date = strtotime(end($date_items)); $this->items[] = [ - 'uri' => self::URI . $child->href, - 'title' => $child->plaintext, + 'uri' => self::URI . $child->href, + 'title' => $child->plaintext, 'timestamp' => $date, - 'content' => $this->parseEntry(self::URI . $child->href) + 'content' => $this->extractContent(self::URI . $child->href), ]; $added++; - if ($added >= $this->getInput('limit')) { + if ($added >= $limit) { break; } } } - private function parseEntry($uri) + private function extractContent($url) { - $html = getSimpleHTMLDOM($uri); + $html = getSimpleHTMLDOM($url); $content = $html->find('div.content-center.mt-5', 0); + if (!$content) { + return ''; + } $subscribe_form = $content->find('div.mt-5 > div > form', 0); if ($subscribe_form) { $content->removeChild($subscribe_form->parent->parent); diff --git a/lib/http.php b/lib/http.php index 3d65b2d1..c5c57d05 100644 --- a/lib/http.php +++ b/lib/http.php @@ -13,6 +13,7 @@ final class CloudFlareException extends HttpException '<title>Please Wait...', '<title>Attention Required!', '<title>Security | Glassdoor', + '<title>Access denied', // cf as seen on patreon.com ]; foreach ($cloudflareTitles as $cloudflareTitle) { if (str_contains($response->getBody(), $cloudflareTitle)) { diff --git a/lib/utils.php b/lib/utils.php index 4c58d258..e8f00f54 100644 --- a/lib/utils.php +++ b/lib/utils.php @@ -140,7 +140,7 @@ function _sanitize_path_name(string $s, string $pathName): string } /** - * This is buggy because strip tags removes a lot that isn't html + * This is buggy because strip_tags() removes a lot that isn't html */ function is_html(string $text): bool { From 145bd10f4c00f3aef9d178266cb3282920dc6f41 Mon Sep 17 00:00:00 2001 From: Eugene Molotov Date: Wed, 11 Oct 2023 21:16:57 +0500 Subject: [PATCH 158/716] [VkBridge] Revert more universal regex for title generation (#3736) In practice it lead to feed items to have "untitled". Using previous regex with more covered cases. Credits to https://t.me/votkot as author of regex --- bridges/VkBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 90b2586e..a21ee665 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -453,7 +453,7 @@ class VkBridge extends BridgeAbstract { $content = explode('
', $content)[0]; $content = strip_tags($content); - preg_match('/.+?(?=[\.\n])/mu', htmlspecialchars_decode($content), $result); + preg_match('/^[:\,"\w\ \p{L}\(\)\?#«»\-\–\—||&\.%\\₽\/+\;\!]+/mu', htmlspecialchars_decode($content), $result); if (count($result) == 0) { return 'untitled'; } From 7e183915a956a24b235e6c30914b2d8f1018a65a Mon Sep 17 00:00:00 2001 From: Eugene Molotov Date: Wed, 11 Oct 2023 21:28:54 +0500 Subject: [PATCH 159/716] [VkBridge] Fix missing feed title (#3737) --- bridges/VkBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index a21ee665..d86b9cf9 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -76,10 +76,10 @@ class VkBridge extends BridgeAbstract break; } } - $pageName = $html->find('.page_name', 0); + $pageName = $html->find('meta[property="og:title"]', 0); if (is_object($pageName)) { - $pageName = $pageName->plaintext; - $this->pageName = htmlspecialchars_decode($pageName); + $pageName = $pageName->getAttribute('content'); + $this->pageName = $pageName; } foreach ($html->find('div.replies') as $comment_block) { $comment_block->outertext = ''; From d21f8cebf617aa1f8154b1ec03d5af1b420d489d Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 11 Oct 2023 18:37:01 +0200 Subject: [PATCH 160/716] fix(imgsed): parsing of datetime string (#3738) * refactor * fix(imgsed): parsing of date date_interval_create_from_date_string(): Unknown or bad format (an hour) at position 0 (a) --- bridges/ImgsedBridge.php | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index b6385361..e605cf4f 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -43,15 +43,6 @@ class ImgsedBridge extends BridgeAbstract 'https://www.imgsed.com/instagram/' => ['context' => 'Username', 'u' => 'instagram', 'post' => 'on', 'story' => 'on', 'tagged' => 'on'], ]; - public function getURI() - { - if (!is_null($this->getInput('u'))) { - return urljoin(self::URI, '/' . $this->getInput('u') . '/'); - } - - return parent::getURI(); - } - public function collectData() { $username = $this->getInput('u'); @@ -100,9 +91,6 @@ class ImgsedBridge extends BridgeAbstract $isMoreContent = (bool) $post->find('svg', 0); $moreContentNote = $isMoreContent ? '

(multiple images and/or videos)

' : ''; - - - $this->items[] = [ 'uri' => $url, 'author' => $author, @@ -214,15 +202,18 @@ HTML, } } - // Parse date, and transform the date into a timetamp, even in a case of a relative date private function parseDate($content) { + // Parse date, and transform the date into a timetamp, even in a case of a relative date $date = date_create(); $dateString = str_replace(' ago', '', $content); // Special case : 'a day' is not a valid interval in PHP, so replace it with it's PHP equivalenbt : '1 day' if ($dateString == 'a day') { $dateString = '1 day'; } + if ($dateString === 'an hour') { + $dateString = '1 hour'; + } $relativeDate = date_interval_create_from_date_string($dateString); if ($relativeDate) { @@ -235,6 +226,15 @@ HTML, return date_format($date, 'r'); } + public function getURI() + { + if (!is_null($this->getInput('u'))) { + return urljoin(self::URI, '/' . $this->getInput('u') . '/'); + } + + return parent::getURI(); + } + private function convertURLToInstagram($url) { return str_replace(self::URI, self::INSTAGRAMURI, $url); From 6a72c56cdd9230fedddc3d5eb23ebe94bfdc3310 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 12 Oct 2023 19:49:04 +0200 Subject: [PATCH 161/716] fix: various fixes (#3741) --- bridges/CarThrottleBridge.php | 30 +++++++++++++++++++----------- bridges/GatesNotesBridge.php | 6 ++++-- lib/contents.php | 11 +++++------ 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/bridges/CarThrottleBridge.php b/bridges/CarThrottleBridge.php index 5b95dd28..913b686c 100644 --- a/bridges/CarThrottleBridge.php +++ b/bridges/CarThrottleBridge.php @@ -25,20 +25,28 @@ class CarThrottleBridge extends BridgeAbstract $articlePage = getSimpleHTMLDOMCached($item['uri']) or returnServerError('could not retrieve page'); - $item['author'] = $articlePage->find('div.author div')[1]->innertext; - - $dinges = $articlePage->find('div.main-body')[0]; - //remove ads - foreach ($dinges->find('aside') as $ad) { - $ad->outertext = ''; - $dinges->save(); + $authorDiv = $articlePage->find('div.author div'); + if ($authorDiv) { + $item['author'] = $authorDiv[1]->innertext; } - $item['content'] = $articlePage->find('div.summary')[0] . - $articlePage->find('figure.main-image')[0] . - $dinges; + $dinges = $articlePage->find('div.main-body')[0] ?? null; + //remove ads + if ($dinges) { + foreach ($dinges->find('aside') as $ad) { + $ad->outertext = ''; + $dinges->save(); + } + } + + $var = $articlePage->find('div.summary')[0] ?? ''; + $var1 = $articlePage->find('figure.main-image')[0] ?? ''; + $dinges1 = $dinges ?? ''; + + $item['content'] = $var . + $var1 . + $dinges1; - //add the item to the list array_push($this->items, $item); } } diff --git a/bridges/GatesNotesBridge.php b/bridges/GatesNotesBridge.php index 61dccb1a..24ba9b2e 100644 --- a/bridges/GatesNotesBridge.php +++ b/bridges/GatesNotesBridge.php @@ -23,9 +23,11 @@ class GatesNotesBridge extends BridgeAbstract $cleanedContent = str_replace([ '', '', + '\r\n', ], '', $rawContent); + $cleanedContent = str_replace('\"', '"', $cleanedContent); + $cleanedContent = trim($cleanedContent, '"'); - // The content is actually a json between quotes with \r\n inserted $json = Json::decode($cleanedContent, false); foreach ($json as $article) { @@ -98,7 +100,7 @@ class GatesNotesBridge extends BridgeAbstract } $article_body = sanitize($article_body->innertext); - $content = $top_description . $hero_image . $article_body; + $content = $top_description . ($hero_image ?? '') . $article_body; return $content; } diff --git a/lib/contents.php b/lib/contents.php index cfb9f36a..a3830ca7 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -158,13 +158,12 @@ function getSimpleHTMLDOM( $defaultBRText = DEFAULT_BR_TEXT, $defaultSpanText = DEFAULT_SPAN_TEXT ) { - $content = getContents( - $url, - $header ?? [], - $opts ?? [] - ); + $html = getContents($url, $header ?? [], $opts ?? []); + if ($html === '') { + throw new \Exception('Unable to parse dom because the http response was the empty string'); + } return str_get_html( - $content, + $html, $lowercase, $forceTagsClosed, $target_charset, From e55a88fb8ea6133bdc23076f1af80726b48c245a Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 12 Oct 2023 20:32:17 +0200 Subject: [PATCH 162/716] refactor(nyaa) (#3742) --- bridges/NyaaTorrentsBridge.php | 76 +++++++++++----------------------- 1 file changed, 25 insertions(+), 51 deletions(-) diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php index da3c34f5..3129bfc4 100644 --- a/bridges/NyaaTorrentsBridge.php +++ b/bridges/NyaaTorrentsBridge.php @@ -6,20 +6,6 @@ class NyaaTorrentsBridge extends FeedExpander const NAME = 'NyaaTorrents'; const URI = 'https://nyaa.si/'; const DESCRIPTION = 'Returns the newest torrents, with optional search criteria.'; - const MAX_ITEMS = 20; - const CUSTOM_FIELD_PREFIX = 'nyaa:'; - const CUSTOM_FIELDS = [ - self::CUSTOM_FIELD_PREFIX . 'seeders' => 'seeders', - self::CUSTOM_FIELD_PREFIX . 'leechers' => 'leechers', - self::CUSTOM_FIELD_PREFIX . 'downloads' => 'downloads', - self::CUSTOM_FIELD_PREFIX . 'infoHash' => 'infoHash', - self::CUSTOM_FIELD_PREFIX . 'categoryId' => 'categoryId', - self::CUSTOM_FIELD_PREFIX . 'category' => 'category', - self::CUSTOM_FIELD_PREFIX . 'size' => 'size', - self::CUSTOM_FIELD_PREFIX . 'comments' => 'comments', - self::CUSTOM_FIELD_PREFIX . 'trusted' => 'trusted', - self::CUSTOM_FIELD_PREFIX . 'remake' => 'remake' - ]; const PARAMETERS = [ [ 'f' => [ @@ -74,58 +60,31 @@ class NyaaTorrentsBridge extends FeedExpander ] ]; - public function getIcon() - { - return self::URI . 'static/favicon.png'; - } - - public function getURI() - { - return self::URI . '?page=rss&s=id&o=desc&' - . http_build_query([ - 'f' => $this->getInput('f'), - 'c' => $this->getInput('c'), - 'q' => $this->getInput('q'), - 'u' => $this->getInput('u') - ]); - } - public function collectData() { - $content = getContents($this->getURI()); - $content = $this->fixCustomFields($content); - $rssContent = simplexml_load_string(trim($content)); - $this->collectRss2($rssContent, self::MAX_ITEMS); - } - - private function fixCustomFields($content) - { - $broken = array_keys(self::CUSTOM_FIELDS); - $fixed = array_values(self::CUSTOM_FIELDS); - return str_replace($broken, $fixed, $content); + $this->collectExpandableDatas($this->getURI()); } protected function parseItem($newItem) { $item = parent::parseRss2Item($newItem); - - // Add nyaa custom fields $item['id'] = str_replace(['https://nyaa.si/download/', '.torrent'], '', $item['uri']); - foreach (array_values(self::CUSTOM_FIELDS) as $value) { - $item[$value] = (string) $newItem->$value; - } - //Convert URI from torrent file to web page + $nyaaFields = (array)($newItem->children('nyaa', true)); + $item = array_merge($item, $nyaaFields); + + // Convert URI from torrent file to web page $item['uri'] = str_replace('/download/', '/view/', $item['uri']); $item['uri'] = str_replace('.torrent', '', $item['uri']); - if ($item_html = getSimpleHTMLDOMCached($item['uri'])) { - //Retrieve full description from page contents + $item_html = getSimpleHTMLDOMCached($item['uri']); + if ($item_html) { + // Retrieve full description from page contents $item_desc = str_get_html( markdownToHtml(html_entity_decode($item_html->find('#torrent-description', 0)->innertext)) ); - //Retrieve image for thumbnail or generic logo fallback + // Retrieve image for thumbnail or generic logo fallback $item_image = $this->getURI() . 'static/img/avatar/default.png'; foreach ($item_desc->find('img') as $img) { if (strpos($img->src, 'prez') === false) { @@ -134,11 +93,26 @@ class NyaaTorrentsBridge extends FeedExpander } } - //Add expanded fields to the current item $item['enclosures'] = [$item_image]; $item['content'] = $item_desc; } return $item; } + + public function getIcon() + { + return self::URI . 'static/favicon.png'; + } + + public function getURI() + { + $params = [ + 'f' => $this->getInput('f'), + 'c' => $this->getInput('c'), + 'q' => $this->getInput('q'), + 'u' => $this->getInput('u'), + ]; + return self::URI . '?page=rss&s=id&o=desc&' . http_build_query($params); + } } From 6634291c67194428241f84d48f236a3037492bee Mon Sep 17 00:00:00 2001 From: Jisagi Date: Thu, 12 Oct 2023 21:24:08 +0200 Subject: [PATCH 163/716] NyaaTorrentsBridge - add max items again (#3743) --- bridges/NyaaTorrentsBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php index 3129bfc4..22225be8 100644 --- a/bridges/NyaaTorrentsBridge.php +++ b/bridges/NyaaTorrentsBridge.php @@ -62,7 +62,7 @@ class NyaaTorrentsBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas($this->getURI()); + $this->collectExpandableDatas($this->getURI(), 20); } protected function parseItem($newItem) From 9bda9e246a21a44daeeb08dd1f41665e5a9f16e3 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 12 Oct 2023 22:14:04 +0200 Subject: [PATCH 164/716] refactor: FeedExpander (#3740) * refactor: FeedExpander --- bridges/CssSelectorFeedExpanderBridge.php | 4 + bridges/FeedExpanderExampleBridge.php | 17 +- bridges/MastodonBridge.php | 4 +- bridges/NyaaTorrentsBridge.php | 2 +- bridges/RaceDepartmentBridge.php | 2 +- index.php | 50 +++ lib/FeedExpander.php | 482 +++------------------- lib/FeedParser.php | 205 +++++++++ lib/RssBridge.php | 53 --- 9 files changed, 315 insertions(+), 504 deletions(-) create mode 100644 lib/FeedParser.php diff --git a/bridges/CssSelectorFeedExpanderBridge.php b/bridges/CssSelectorFeedExpanderBridge.php index 7e1f630f..008921b1 100644 --- a/bridges/CssSelectorFeedExpanderBridge.php +++ b/bridges/CssSelectorFeedExpanderBridge.php @@ -61,6 +61,10 @@ class CssSelectorFeedExpanderBridge extends CssSelectorBridge $discard_thumbnail = $this->getInput('discard_thumbnail'); $limit = $this->getInput('limit'); + //$xmlString = getContents($url); + //$feed = (new FeedParser())->parseFeed($xmlString); + //$items = $feed['items']; + $feed_expander = new CssSelectorFeedExpanderBridgeInternal(); $items = $feed_expander->collectExpandableDatas($url)->getItems(); diff --git a/bridges/FeedExpanderExampleBridge.php b/bridges/FeedExpanderExampleBridge.php index a6b37f65..0e9ae386 100644 --- a/bridges/FeedExpanderExampleBridge.php +++ b/bridges/FeedExpanderExampleBridge.php @@ -46,21 +46,6 @@ class FeedExpanderExampleBridge extends FeedExpander protected function parseItem($newsItem) { - switch ($this->getInput('version')) { - case 'rss_0_9_1': - return $this->parseRss091Item($newsItem); - break; - case 'rss_1_0': - return $this->parseRss1Item($newsItem); - break; - case 'rss_2_0': - return $this->parseRss2Item($newsItem); - break; - case 'atom_1_0': - return $this->parseATOMItem($newsItem); - break; - default: - returnClientError('Unknown version ' . $this->getInput('version') . '!'); - } + return (array) $newsItem; } } diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index cae556b8..01d425b4 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -82,14 +82,14 @@ class MastodonBridge extends BridgeAbstract } $items = $content['orderedItems'] ?? $content['items']; foreach ($items as $status) { - $item = $this->parseItem($status); + $item = $this->parseStatus($status); if ($item) { $this->items[] = $item; } } } - protected function parseItem($content) + protected function parseStatus($content) { $item = []; switch ($content['type']) { diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php index 22225be8..217db377 100644 --- a/bridges/NyaaTorrentsBridge.php +++ b/bridges/NyaaTorrentsBridge.php @@ -67,7 +67,7 @@ class NyaaTorrentsBridge extends FeedExpander protected function parseItem($newItem) { - $item = parent::parseRss2Item($newItem); + $item = parent::parseItem($newItem); $item['id'] = str_replace(['https://nyaa.si/download/', '.torrent'], '', $item['uri']); $nyaaFields = (array)($newItem->children('nyaa', true)); diff --git a/bridges/RaceDepartmentBridge.php b/bridges/RaceDepartmentBridge.php index c33ee67a..7fe92b4a 100644 --- a/bridges/RaceDepartmentBridge.php +++ b/bridges/RaceDepartmentBridge.php @@ -14,7 +14,7 @@ class RaceDepartmentBridge extends FeedExpander protected function parseItem($feedItem) { - $item = parent::parseRss2Item($feedItem); + $item = parent::parseItem($feedItem); //fetch page $articlePage = getSimpleHTMLDOMCached($feedItem->link); diff --git a/index.php b/index.php index 9181c0b0..123f6ecd 100644 --- a/index.php +++ b/index.php @@ -6,6 +6,56 @@ if (version_compare(\PHP_VERSION, '7.4.0') === -1) { require_once __DIR__ . '/lib/bootstrap.php'; +Configuration::verifyInstallation(); +$customConfig = []; +if (file_exists(__DIR__ . '/config.ini.php')) { + $customConfig = parse_ini_file(__DIR__ . '/config.ini.php', true, INI_SCANNER_TYPED); +} +Configuration::loadConfiguration($customConfig, getenv()); + +// Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); +date_default_timezone_set(Configuration::getConfig('system', 'timezone')); + $rssBridge = new RssBridge(); +set_exception_handler(function (\Throwable $e) { + http_response_code(500); + print render(__DIR__ . '/templates/exception.html.php', ['e' => $e]); + RssBridge::getLogger()->error('Uncaught Exception', ['e' => $e]); + exit(1); +}); + +set_error_handler(function ($code, $message, $file, $line) { + if ((error_reporting() & $code) === 0) { + return false; + } + // In the future, uncomment this: + //throw new \ErrorException($message, 0, $code, $file, $line); + $text = sprintf( + '%s at %s line %s', + sanitize_root($message), + sanitize_root($file), + $line + ); + RssBridge::getLogger()->warning($text); +}); + +// There might be some fatal errors which are not caught by set_error_handler() or \Throwable. +register_shutdown_function(function () { + $error = error_get_last(); + if ($error) { + $message = sprintf( + '(shutdown) %s: %s in %s line %s', + $error['type'], + sanitize_root($error['message']), + sanitize_root($error['file']), + $error['line'] + ); + RssBridge::getLogger()->error($message); + if (Debug::isEnabled()) { + print sprintf("
%s
\n", e($message)); + } + } +}); + $rssBridge->main($argv ?? []); diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index 76f570b6..70c4560d 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -1,95 +1,37 @@ feedParser = new FeedParser(); + } + + public function collectExpandableDatas(string $url, $maxItems = -1) + { + if (!$url) { throw new \Exception('There is no $url for this RSS expander'); } - - Debug::log(sprintf('Loading from %s', $url)); - - /* Notice we do not use cache here on purpose: - * we want a fresh view of the RSS stream each time - */ - - $mimeTypes = [ - MrssFormat::MIME_TYPE, - AtomFormat::MIME_TYPE, - '*/*', - ]; - $httpHeaders = ['Accept: ' . implode(', ', $mimeTypes)]; - $xml = getContents($url, $httpHeaders); - if ($xml === '') { + if ($maxItems === -1) { + $maxItems = 999; + } + $accept = [MrssFormat::MIME_TYPE, AtomFormat::MIME_TYPE, '*/*']; + $httpHeaders = ['Accept: ' . implode(', ', $accept)]; + // Notice we do not use cache here on purpose. We want a fresh view of the RSS stream each time + $xmlString = getContents($url, $httpHeaders); + if ($xmlString === '') { throw new \Exception(sprintf('Unable to parse xml from `%s` because we got the empty string', $url), 10); } // Maybe move this call earlier up the stack frames @@ -97,8 +39,8 @@ abstract class FeedExpander extends BridgeAbstract libxml_use_internal_errors(true); // Consider replacing libxml with https://www.php.net/domdocument // Intentionally not using the silencing operator (@) because it has no effect here - $rssContent = simplexml_load_string(trim($xml)); - if ($rssContent === false) { + $xml = simplexml_load_string(trim($xmlString)); + if ($xml === false) { $xmlErrors = libxml_get_errors(); foreach ($xmlErrors as $xmlError) { Debug::log(trim($xmlError->message)); @@ -112,384 +54,62 @@ abstract class FeedExpander extends BridgeAbstract // Restore previous behaviour in case other code relies on it being off libxml_use_internal_errors(false); - // Commented out because it's spammy - // Debug::log(sprintf("RSS content is ===========\n%s===========", var_export($rssContent, true))); + // Currently only feed metadata (not items) are plucked out + $this->parsedFeed = $this->feedParser->parseFeed($xmlString); - switch (true) { - case isset($rssContent->item[0]): - Debug::log('Detected RSS 1.0 format'); - $this->feedType = self::FEED_TYPE_RSS_1_0; - $this->collectRss1($rssContent, $maxItems); - break; - case isset($rssContent->channel[0]): - Debug::log('Detected RSS 0.9x or 2.0 format'); - $this->feedType = self::FEED_TYPE_RSS_2_0; - $this->collectRss2($rssContent, $maxItems); - break; - case isset($rssContent->entry[0]): - Debug::log('Detected ATOM format'); - $this->feedType = self::FEED_TYPE_ATOM_1_0; - $this->collectAtom1($rssContent, $maxItems); - break; - default: - Debug::log(sprintf('Unable to detect feed format from `%s`', $url)); - throw new \Exception(sprintf('Unable to detect feed format from `%s`', $url)); + if (isset($xml->item[0])) { + $this->feedType = self::FEED_TYPE_RSS_1_0; + $items = $xml->item; + } elseif (isset($xml->channel[0])) { + $this->feedType = self::FEED_TYPE_RSS_2_0; + $items = $xml->channel[0]->item; + } elseif (isset($xml->entry[0])) { + $this->feedType = self::FEED_TYPE_ATOM_1_0; + $items = $xml->entry; + } else { + throw new \Exception(sprintf('Unable to detect feed format from `%s`', $url)); + } + foreach ($items as $item) { + $parsedItem = $this->parseItem($item); + if ($parsedItem) { + $this->items[] = $parsedItem; + } + if (count($this->items) >= $maxItems) { + break; + } } - return $this; } /** - * Collect data from an RSS 1.0 compatible feed - * - * @link http://web.resource.org/rss/1.0/spec RDF Site Summary (RSS) 1.0 - * - * @param string $rssContent The RSS content - * @param int $maxItems Maximum number of items to collect from the feed - * (`-1`: no limit). - * @return void - * - * @todo Instead of passing $maxItems to all functions, just add all items - * and remove excessive items later. - */ - protected function collectRss1($rssContent, $maxItems) - { - $this->loadRss2Data($rssContent->channel[0]); - foreach ($rssContent->item as $item) { - $tmp_item = $this->parseItem($item); - if (!empty($tmp_item)) { - $this->items[] = $tmp_item; - } - if ($maxItems !== -1 && count($this->items) >= $maxItems) { - break; - } - } - } - - /** - * Collect data from a RSS 2.0 compatible feed - * - * @link http://www.rssboard.org/rss-specification RSS 2.0 Specification - * - * @param int $maxItems Maximum number of items to collect from the feed (`-1`: no limit). - * @return void - * - * @todo Instead of passing $maxItems to all functions, just add all items and remove excessive items later. - */ - protected function collectRss2(\SimpleXMLElement $rssContent, $maxItems) - { - $rssContent = $rssContent->channel[0]; - $this->loadRss2Data($rssContent); - foreach ($rssContent->item as $item) { - $tmp_item = $this->parseItem($item); - if (!empty($tmp_item)) { - $this->items[] = $tmp_item; - } - if ($maxItems !== -1 && count($this->items) >= $maxItems) { - break; - } - } - } - - /** - * Collect data from a Atom 1.0 compatible feed - * - * @link https://tools.ietf.org/html/rfc4287 The Atom Syndication Format - * - * @param object $content The Atom content - * @param int $maxItems Maximum number of items to collect from the feed - * (`-1`: no limit). - * @return void - * - * @todo Instead of passing $maxItems to all functions, just add all items - * and remove excessive items later. - */ - protected function collectAtom1($content, $maxItems) - { - $this->loadAtomData($content); - foreach ($content->entry as $item) { - $tmp_item = $this->parseItem($item); - if (!empty($tmp_item)) { - $this->items[] = $tmp_item; - } - if ($maxItems !== -1 && count($this->items) >= $maxItems) { - break; - } - } - } - - /** - * Load RSS 2.0 feed data into RSS-Bridge - * - * @param object $rssContent The RSS content - * @return void - * - * @todo set title, link, description, language, and so on - */ - protected function loadRss2Data($rssContent) - { - $this->title = trim((string)$rssContent->title); - $this->uri = trim((string)$rssContent->link); - - if (!empty($rssContent->image)) { - $this->icon = trim((string)$rssContent->image->url); - } - } - - /** - * Load Atom feed data into RSS-Bridge - * - * @param object $content The Atom content - * @return void - */ - protected function loadAtomData($content) - { - $this->title = (string)$content->title; - - // Find best link (only one, or first of 'alternate') - if (!isset($content->link)) { - $this->uri = ''; - } elseif (count($content->link) === 1) { - $this->uri = (string)$content->link[0]['href']; - } else { - $this->uri = ''; - foreach ($content->link as $link) { - if (strtolower($link['rel']) === 'alternate') { - $this->uri = (string)$link['href']; - break; - } - } - } - - if (!empty($content->icon)) { - $this->icon = (string)$content->icon; - } elseif (!empty($content->logo)) { - $this->icon = (string)$content->logo; - } - } - - /** - * Parse the contents of a single Atom feed item into a RSS-Bridge item for - * further transformation. - * - * @param object $feedItem A single feed item - * @return object The RSS-Bridge item - * - * @todo To reduce confusion, the RSS-Bridge item should maybe have a class - * of its own? - */ - protected function parseATOMItem($feedItem) - { - // Some ATOM entries also contain RSS 2.0 fields - $item = $this->parseRss2Item($feedItem); - - if (isset($feedItem->id)) { - $item['uri'] = (string)$feedItem->id; - } - if (isset($feedItem->title)) { - $item['title'] = html_entity_decode((string)$feedItem->title); - } - if (isset($feedItem->updated)) { - $item['timestamp'] = strtotime((string)$feedItem->updated); - } - if (isset($feedItem->author)) { - $item['author'] = (string)$feedItem->author->name; - } - if (isset($feedItem->content)) { - $contentChildren = $feedItem->content->children(); - if (count($contentChildren) > 0) { - $content = ''; - foreach ($contentChildren as $contentChild) { - $content .= $contentChild->asXML(); - } - $item['content'] = $content; - } else { - $item['content'] = (string)$feedItem->content; - } - } - - //When "link" field is present, URL is more reliable than "id" field - if (count($feedItem->link) === 1) { - $item['uri'] = (string)$feedItem->link[0]['href']; - } else { - foreach ($feedItem->link as $link) { - if (strtolower($link['rel']) === 'alternate') { - $item['uri'] = (string)$link['href']; - } - if (strtolower($link['rel']) === 'enclosure') { - $item['enclosures'][] = (string)$link['href']; - } - } - } - - return $item; - } - - /** - * Parse the contents of a single RSS 0.91 feed item into a RSS-Bridge item - * for further transformation. - * - * @param object $feedItem A single feed item - * @return object The RSS-Bridge item - * - * @todo To reduce confusion, the RSS-Bridge item should maybe have a class - * of its own? - */ - protected function parseRss091Item($feedItem) - { - $item = []; - if (isset($feedItem->link)) { - $item['uri'] = (string)$feedItem->link; - } - if (isset($feedItem->title)) { - $item['title'] = html_entity_decode((string)$feedItem->title); - } - // rss 0.91 doesn't support timestamps - // rss 0.91 doesn't support authors - // rss 0.91 doesn't support enclosures - if (isset($feedItem->description)) { - $item['content'] = (string)$feedItem->description; - } - return $item; - } - - /** - * Parse the contents of a single RSS 1.0 feed item into a RSS-Bridge item - * for further transformation. - * - * @param object $feedItem A single feed item - * @return object The RSS-Bridge item - * - * @todo To reduce confusion, the RSS-Bridge item should maybe have a class - * of its own? - */ - protected function parseRss1Item($feedItem) - { - // 1.0 adds optional elements around the 0.91 standard - $item = $this->parseRss091Item($feedItem); - - $namespaces = $feedItem->getNamespaces(true); - if (isset($namespaces['dc'])) { - $dc = $feedItem->children($namespaces['dc']); - if (isset($dc->date)) { - $item['timestamp'] = strtotime((string)$dc->date); - } - if (isset($dc->creator)) { - $item['author'] = (string)$dc->creator; - } - } - - return $item; - } - - /** - * Parse the contents of a single RSS 2.0 feed item into a RSS-Bridge item - * for further transformation. - * - * @param object $feedItem A single feed item - * @return object The RSS-Bridge item - * - * @todo To reduce confusion, the RSS-Bridge item should maybe have a class - * of its own? - */ - protected function parseRss2Item($feedItem) - { - // Primary data is compatible to 0.91 with some additional data - $item = $this->parseRss091Item($feedItem); - - $namespaces = $feedItem->getNamespaces(true); - if (isset($namespaces['dc'])) { - $dc = $feedItem->children($namespaces['dc']); - } - if (isset($namespaces['media'])) { - $media = $feedItem->children($namespaces['media']); - } - - if (isset($feedItem->guid)) { - foreach ($feedItem->guid->attributes() as $attribute => $value) { - if ( - $attribute === 'isPermaLink' - && ( - $value === 'true' || ( - filter_var($feedItem->guid, FILTER_VALIDATE_URL) - && (empty($item['uri']) || !filter_var($item['uri'], FILTER_VALIDATE_URL)) - ) - ) - ) { - $item['uri'] = (string)$feedItem->guid; - break; - } - } - } - - if (isset($feedItem->pubDate)) { - $item['timestamp'] = strtotime((string)$feedItem->pubDate); - } elseif (isset($dc->date)) { - $item['timestamp'] = strtotime((string)$dc->date); - } - - if (isset($feedItem->author)) { - $item['author'] = (string)$feedItem->author; - } elseif (isset($feedItem->creator)) { - $item['author'] = (string)$feedItem->creator; - } elseif (isset($dc->creator)) { - $item['author'] = (string)$dc->creator; - } elseif (isset($media->credit)) { - $item['author'] = (string)$media->credit; - } - - if (isset($feedItem->enclosure) && !empty($feedItem->enclosure['url'])) { - $item['enclosures'] = [(string)$feedItem->enclosure['url']]; - } - - return $item; - } - - /** - * Parse the contents of a single feed item, depending on the current feed - * type, into a RSS-Bridge item. - * - * @param object $item The current feed item - * @return object A RSS-Bridge item, with (hopefully) the whole content + * @param \SimpleXMLElement $item The feed item to be parsed */ protected function parseItem($item) { switch ($this->feedType) { case self::FEED_TYPE_RSS_1_0: - return $this->parseRss1Item($item); + return $this->feedParser->parseRss1Item($item); case self::FEED_TYPE_RSS_2_0: - return $this->parseRss2Item($item); + return $this->feedParser->parseRss2Item($item); case self::FEED_TYPE_ATOM_1_0: - return $this->parseATOMItem($item); + return $this->feedParser->parseAtomItem($item); default: throw new \Exception(sprintf('Unknown version %s!', $this->getInput('version'))); } } - /** {@inheritdoc} */ public function getURI() { - if (!empty($this->uri)) { - return $this->uri; - } - return parent::getURI(); + return $this->parsedFeed['uri'] ?? parent::getURI(); } - /** {@inheritdoc} */ public function getName() { - if (!empty($this->title)) { - return $this->title; - } - return parent::getName(); + return $this->parsedFeed['title'] ?? parent::getName(); } - /** {@inheritdoc} */ public function getIcon() { - if (!empty($this->icon)) { - return $this->icon; - } - return parent::getIcon(); + return $this->parsedFeed['icon'] ?? parent::getIcon(); } } diff --git a/lib/FeedParser.php b/lib/FeedParser.php new file mode 100644 index 00000000..90df548d --- /dev/null +++ b/lib/FeedParser.php @@ -0,0 +1,205 @@ + null, + 'url' => null, + 'icon' => null, + 'items' => [], + ]; + if (isset($xml->item[0])) { + // rss 1.0 + $channel = $xml->channel[0]; + $feed['title'] = trim((string)$channel->title); + $feed['uri'] = trim((string)$channel->link); + if (!empty($channel->image)) { + $feed['icon'] = trim((string)$channel->image->url); + } + foreach ($xml->item as $item) { + $feed['items'][] = $this->parseRss1Item($item); + } + } elseif (isset($xml->channel[0])) { + // rss 2.0 + $channel = $xml->channel[0]; + $feed['title'] = trim((string)$channel->title); + $feed['uri'] = trim((string)$channel->link); + if (!empty($channel->image)) { + $feed['icon'] = trim((string)$channel->image->url); + } + foreach ($channel->item as $item) { + $feed['items'][] = $this->parseRss2Item($item); + } + } elseif (isset($xml->entry[0])) { + // atom 1.0 + $feed['title'] = (string)$xml->title; + // Find best link (only one, or first of 'alternate') + if (!isset($xml->link)) { + $feed['uri'] = ''; + } elseif (count($xml->link) === 1) { + $feed['uri'] = (string)$xml->link[0]['href']; + } else { + $feed['uri'] = ''; + foreach ($xml->link as $link) { + if (strtolower((string) $link['rel']) === 'alternate') { + $feed['uri'] = (string)$link['href']; + break; + } + } + } + if (!empty($xml->icon)) { + $feed['icon'] = (string)$xml->icon; + } elseif (!empty($xml->logo)) { + $feed['icon'] = (string)$xml->logo; + } + foreach ($xml->entry as $item) { + $feed['items'][] = $this->parseAtomItem($item); + } + } else { + throw new \Exception(sprintf('Unable to detect feed format from `%s`', $url)); + } + + return $feed; + } + + public function parseAtomItem(\SimpleXMLElement $feedItem): array + { + // Some ATOM entries also contain RSS 2.0 fields + $item = $this->parseRss2Item($feedItem); + + if (isset($feedItem->id)) { + $item['uri'] = (string)$feedItem->id; + } + if (isset($feedItem->title)) { + $item['title'] = html_entity_decode((string)$feedItem->title); + } + if (isset($feedItem->updated)) { + $item['timestamp'] = strtotime((string)$feedItem->updated); + } + if (isset($feedItem->author)) { + $item['author'] = (string)$feedItem->author->name; + } + if (isset($feedItem->content)) { + $contentChildren = $feedItem->content->children(); + if (count($contentChildren) > 0) { + $content = ''; + foreach ($contentChildren as $contentChild) { + $content .= $contentChild->asXML(); + } + $item['content'] = $content; + } else { + $item['content'] = (string)$feedItem->content; + } + } + + // When "link" field is present, URL is more reliable than "id" field + if (count($feedItem->link) === 1) { + $item['uri'] = (string)$feedItem->link[0]['href']; + } else { + foreach ($feedItem->link as $link) { + if (strtolower((string) $link['rel']) === 'alternate') { + $item['uri'] = (string)$link['href']; + } + if (strtolower((string) $link['rel']) === 'enclosure') { + $item['enclosures'][] = (string)$link['href']; + } + } + } + return $item; + } + + public function parseRss2Item(\SimpleXMLElement $feedItem): array + { + // Primary data is compatible to 0.91 with some additional data + $item = $this->parseRss091Item($feedItem); + + $namespaces = $feedItem->getNamespaces(true); + if (isset($namespaces['dc'])) { + $dc = $feedItem->children($namespaces['dc']); + } + if (isset($namespaces['media'])) { + $media = $feedItem->children($namespaces['media']); + } + + if (isset($feedItem->guid)) { + foreach ($feedItem->guid->attributes() as $attribute => $value) { + if ( + $attribute === 'isPermaLink' + && ( + $value === 'true' || ( + filter_var($feedItem->guid, FILTER_VALIDATE_URL) + && (empty($item['uri']) || !filter_var($item['uri'], FILTER_VALIDATE_URL)) + ) + ) + ) { + $item['uri'] = (string)$feedItem->guid; + break; + } + } + } + + if (isset($feedItem->pubDate)) { + $item['timestamp'] = strtotime((string)$feedItem->pubDate); + } elseif (isset($dc->date)) { + $item['timestamp'] = strtotime((string)$dc->date); + } + + if (isset($feedItem->author)) { + $item['author'] = (string)$feedItem->author; + } elseif (isset($feedItem->creator)) { + $item['author'] = (string)$feedItem->creator; + } elseif (isset($dc->creator)) { + $item['author'] = (string)$dc->creator; + } elseif (isset($media->credit)) { + $item['author'] = (string)$media->credit; + } + + if (isset($feedItem->enclosure) && !empty($feedItem->enclosure['url'])) { + $item['enclosures'] = [(string)$feedItem->enclosure['url']]; + } + return $item; + } + + public function parseRss1Item(\SimpleXMLElement $feedItem): array + { + // 1.0 adds optional elements around the 0.91 standard + $item = $this->parseRss091Item($feedItem); + $namespaces = $feedItem->getNamespaces(true); + if (isset($namespaces['dc'])) { + $dc = $feedItem->children($namespaces['dc']); + if (isset($dc->date)) { + $item['timestamp'] = strtotime((string)$dc->date); + } + if (isset($dc->creator)) { + $item['author'] = (string)$dc->creator; + } + } + return $item; + } + + public function parseRss091Item(\SimpleXMLElement $feedItem): array + { + $item = []; + if (isset($feedItem->link)) { + $item['uri'] = (string)$feedItem->link; + } + if (isset($feedItem->title)) { + $item['title'] = html_entity_decode((string)$feedItem->title); + } + // rss 0.91 doesn't support timestamps + // rss 0.91 doesn't support authors + // rss 0.91 doesn't support enclosures + if (isset($feedItem->description)) { + $item['content'] = (string)$feedItem->description; + } + return $item; + } +} diff --git a/lib/RssBridge.php b/lib/RssBridge.php index 2fb21323..d56c59b8 100644 --- a/lib/RssBridge.php +++ b/lib/RssBridge.php @@ -8,66 +8,13 @@ final class RssBridge public function __construct() { - Configuration::verifyInstallation(); - - $customConfig = []; - if (file_exists(__DIR__ . '/../config.ini.php')) { - $customConfig = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); - } - Configuration::loadConfiguration($customConfig, getenv()); - - // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); - date_default_timezone_set(Configuration::getConfig('system', 'timezone')); - - set_exception_handler(function (\Throwable $e) { - self::$logger->error('Uncaught Exception', ['e' => $e]); - http_response_code(500); - print render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]); - exit(1); - }); - - set_error_handler(function ($code, $message, $file, $line) { - if ((error_reporting() & $code) === 0) { - return false; - } - // In the future, uncomment this: - //throw new \ErrorException($message, 0, $code, $file, $line); - $text = sprintf( - '%s at %s line %s', - sanitize_root($message), - sanitize_root($file), - $line - ); - self::$logger->warning($text); - }); - - // There might be some fatal errors which are not caught by set_error_handler() or \Throwable. - register_shutdown_function(function () { - $error = error_get_last(); - if ($error) { - $message = sprintf( - '(shutdown) %s: %s in %s line %s', - $error['type'], - sanitize_root($error['message']), - sanitize_root($error['file']), - $error['line'] - ); - self::$logger->error($message); - if (Debug::isEnabled()) { - print sprintf("
%s
\n", e($message)); - } - } - }); - self::$logger = new SimpleLogger('rssbridge'); if (Debug::isEnabled()) { self::$logger->addHandler(new StreamHandler(Logger::DEBUG)); } else { self::$logger->addHandler(new StreamHandler(Logger::INFO)); } - self::$httpClient = new CurlHttpClient(); - $cacheFactory = new CacheFactory(self::$logger); if (Debug::isEnabled()) { self::$cache = $cacheFactory->create('array'); From 382648fc22c232bc8c66111fc4d3ab6570946437 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 00:25:34 +0200 Subject: [PATCH 165/716] refactor: FeedExpander::parseItem() descendants (#3744) --- bridges/AcrimedBridge.php | 13 +++-- bridges/ArsTechnicaBridge.php | 4 +- bridges/BleepingComputerBridge.php | 12 ++--- bridges/CNETFranceBridge.php | 7 ++- bridges/CaschyBridge.php | 4 +- bridges/CommonDreamsBridge.php | 4 +- bridges/CourrierInternationalBridge.php | 6 +-- bridges/DarkReadingBridge.php | 5 +- bridges/DauphineLibereBridge.php | 4 +- bridges/DeveloppezDotComBridge.php | 36 +++++++------ bridges/EconomistBridge.php | 15 +++--- bridges/EngadgetBridge.php | 20 +++---- bridges/EsquerdaNetBridge.php | 38 +++++++------ bridges/FeedExpanderExampleBridge.php | 5 -- bridges/FeedExpanderTestBridge.php | 23 ++++++++ bridges/FilterBridge.php | 4 +- bridges/ForGifsBridge.php | 10 ++-- bridges/FreeCodeCampBridge.php | 14 ++--- bridges/FuturaSciencesBridge.php | 11 ++-- bridges/HardwareInfoBridge.php | 10 ++-- bridges/HeiseBridge.php | 5 +- bridges/IGNBridge.php | 11 ++-- bridges/LeMondeInformatiqueBridge.php | 5 +- bridges/ListverseBridge.php | 9 ++-- bridges/MediapartBridge.php | 14 +++-- bridges/MsnMondeBridge.php | 12 +++-- bridges/NYTBridge.php | 11 ++-- bridges/NextInpactBridge.php | 5 +- bridges/NextgovBridge.php | 3 +- bridges/NiceMatinBridge.php | 5 +- bridges/NyaaTorrentsBridge.php | 8 +-- bridges/OnVaSortirBridge.php | 16 +++--- bridges/PhoronixBridge.php | 17 +++--- bridges/QwantzBridge.php | 15 +++--- bridges/RaceDepartmentBridge.php | 7 ++- bridges/ScribbleHubBridge.php | 18 +++---- bridges/SplCenterBridge.php | 11 ++-- bridges/TapasBridge.php | 30 +++++------ bridges/TheGuardianBridge.php | 13 ++--- bridges/TwitterEngineeringBridge.php | 24 ++++----- bridges/VarietyBridge.php | 6 +-- bridges/ViceBridge.php | 10 ++-- bridges/WiredBridge.php | 9 ++-- bridges/WordPressBridge.php | 72 ++++++++++++------------- bridges/WorldOfTanksBridge.php | 4 +- bridges/ZeitBridge.php | 10 ++-- 46 files changed, 314 insertions(+), 281 deletions(-) create mode 100644 bridges/FeedExpanderTestBridge.php diff --git a/bridges/AcrimedBridge.php b/bridges/AcrimedBridge.php index d37f3ce4..93890f35 100644 --- a/bridges/AcrimedBridge.php +++ b/bridges/AcrimedBridge.php @@ -20,17 +20,16 @@ class AcrimedBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas( - static::URI . 'spip.php?page=backend', - $this->getInput('limit') - ); + $url = 'https://www.acrimed.org/spip.php?page=backend'; + $limit = $this->getInput('limit'); + $this->collectExpandableDatas($url, $limit); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOM($newsItem->link); + $articlePage = getSimpleHTMLDOM($item['uri']); $article = sanitize($articlePage->find('article.article1', 0)->innertext); $article = defaultLinkTo($article, static::URI); $item['content'] = $article; diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index 1e3e6379..98e5566b 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -33,9 +33,9 @@ class ArsTechnicaBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($newItem) + protected function parseItem($item) { - $item = parent::parseItem($newItem); + $item = parent::parseItem($item); $item_html = getSimpleHTMLDOMCached($item['uri'] . '&'); $item_html = defaultLinkTo($item_html, self::URI); diff --git a/bridges/BleepingComputerBridge.php b/bridges/BleepingComputerBridge.php index c1d3d568..bad78561 100644 --- a/bridges/BleepingComputerBridge.php +++ b/bridges/BleepingComputerBridge.php @@ -7,6 +7,12 @@ class BleepingComputerBridge extends FeedExpander const URI = 'https://www.bleepingcomputer.com/'; const DESCRIPTION = 'Returns the newest articles.'; + public function collectData() + { + $feed = static::URI . 'feed/'; + $this->collectExpandableDatas($feed); + } + protected function parseItem($item) { $item = parent::parseItem($item); @@ -23,10 +29,4 @@ class BleepingComputerBridge extends FeedExpander return $item; } - - public function collectData() - { - $feed = static::URI . 'feed/'; - $this->collectExpandableDatas($feed); - } } diff --git a/bridges/CNETFranceBridge.php b/bridges/CNETFranceBridge.php index da808596..d6a766de 100644 --- a/bridges/CNETFranceBridge.php +++ b/bridges/CNETFranceBridge.php @@ -43,9 +43,9 @@ class CNETFranceBridge extends FeedExpander $this->collectExpandableDatas('https://www.cnetfrance.fr/feeds/rss/news/'); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); foreach ($this->bannedTitle as $term) { if (preg_match('/' . $term . '/mi', $item['title']) === 1) { @@ -54,8 +54,7 @@ class CNETFranceBridge extends FeedExpander } foreach ($this->bannedURL as $term) { - $preg_match = preg_match('#' . $term . '#mi', $item['uri']); - if ($preg_match === 1) { + if (preg_match('#' . $term . '#mi', $item['uri'])) { return null; } } diff --git a/bridges/CaschyBridge.php b/bridges/CaschyBridge.php index 5f463852..7d632bf6 100644 --- a/bridges/CaschyBridge.php +++ b/bridges/CaschyBridge.php @@ -34,9 +34,9 @@ class CaschyBridge extends FeedExpander ); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); if (strpos($item['uri'], 'https://stadt-bremerhaven.de/') !== 0) { return $item; diff --git a/bridges/CommonDreamsBridge.php b/bridges/CommonDreamsBridge.php index 99580499..e1a185de 100644 --- a/bridges/CommonDreamsBridge.php +++ b/bridges/CommonDreamsBridge.php @@ -12,9 +12,9 @@ class CommonDreamsBridge extends FeedExpander $this->collectExpandableDatas('http://www.commondreams.org/rss.xml', 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); $item['content'] = $this->extractContent($item['uri']); return $item; } diff --git a/bridges/CourrierInternationalBridge.php b/bridges/CourrierInternationalBridge.php index fdbe2ea6..9e30fd51 100644 --- a/bridges/CourrierInternationalBridge.php +++ b/bridges/CourrierInternationalBridge.php @@ -13,11 +13,11 @@ class CourrierInternationalBridge extends FeedExpander $this->collectExpandableDatas(static::URI . 'feed/all/rss.xml', 20); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOMCached($feedItem->link); + $articlePage = getSimpleHTMLDOMCached($item['uri']); $content = $articlePage->find('.article-text, depeche-text', 0); if (!$content) { return $item; diff --git a/bridges/DarkReadingBridge.php b/bridges/DarkReadingBridge.php index 58087506..aca30490 100644 --- a/bridges/DarkReadingBridge.php +++ b/bridges/DarkReadingBridge.php @@ -56,9 +56,10 @@ class DarkReadingBridge extends FeedExpander $this->collectExpandableDatas($feed_url, $limit); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + $article = getSimpleHTMLDOMCached($item['uri']); $item['content'] = $this->extractArticleContent($article); $item['enclosures'] = []; //remove author profile picture diff --git a/bridges/DauphineLibereBridge.php b/bridges/DauphineLibereBridge.php index 82323036..0ab808cd 100644 --- a/bridges/DauphineLibereBridge.php +++ b/bridges/DauphineLibereBridge.php @@ -43,9 +43,9 @@ class DauphineLibereBridge extends FeedExpander $this->collectExpandableDatas($url, 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); $item['content'] = $this->extractContent($item['uri']); return $item; } diff --git a/bridges/DeveloppezDotComBridge.php b/bridges/DeveloppezDotComBridge.php index d0d54d0a..9dcbc31a 100644 --- a/bridges/DeveloppezDotComBridge.php +++ b/bridges/DeveloppezDotComBridge.php @@ -163,19 +163,6 @@ class DeveloppezDotComBridge extends FeedExpander ] ]; - /** - * Return the RSS url for selected domain - */ - private function getRssUrl() - { - $domain = $this->getInput('domain'); - if (!empty($domain)) { - return 'https://' . $domain . self::DOMAIN . self::RSS_URL; - } - - return self::URI . self::RSS_URL; - } - /** * Grabs the RSS item from Developpez.com */ @@ -189,15 +176,14 @@ class DeveloppezDotComBridge extends FeedExpander * Parse the content of every RSS item. And will try to get the full article * pointed by the item URL intead of the default abstract. */ - protected function parseItem($newsItem) + protected function parseItem($item) { + $item = parent::parseItem($item); + if (count($this->items) >= $this->getInput('limit')) { return null; } - // This function parse each entry in the RSS with the default parse - $item = parent::parseItem($newsItem); - // There is a bug in Developpez RSS, coma are writtent as '~?' in the // title, so I have to fix it manually $item['title'] = $this->fixComaInTitle($item['title']); @@ -229,6 +215,19 @@ class DeveloppezDotComBridge extends FeedExpander return $item; } + /** + * Return the RSS url for selected domain + */ + private function getRssUrl() + { + $domain = $this->getInput('domain'); + if (!empty($domain)) { + return 'https://' . $domain . self::DOMAIN . self::RSS_URL; + } + + return self::URI . self::RSS_URL; + } + /** * Replace '~?' by a proper coma ',' */ @@ -334,6 +333,9 @@ class DeveloppezDotComBridge extends FeedExpander */ private function isHtmlTagNotTxt($txt) { + if ($txt === '') { + return false; + } $html = str_get_html($txt); return $html && $html->root && count($html->root->children) > 0; } diff --git a/bridges/EconomistBridge.php b/bridges/EconomistBridge.php index 9a73a852..0572ab8f 100644 --- a/bridges/EconomistBridge.php +++ b/bridges/EconomistBridge.php @@ -93,21 +93,22 @@ class EconomistBridge extends FeedExpander $limit = 30; } - $this->collectExpandableDatas('https://www.economist.com/' . $category . '/rss.xml', $limit); + $url = 'https://www.economist.com/' . $category . '/rss.xml'; + $this->collectExpandableDatas($url, $limit); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); - $html = getSimpleHTMLDOM($item['uri']); + $item = parent::parseItem($item); + $dom = getSimpleHTMLDOM($item['uri']); - $article = $html->find('#new-article-template', 0); + $article = $dom->find('#new-article-template', 0); if ($article == null) { - $article = $html->find('main', 0); + $article = $dom->find('main', 0); } if ($article) { $elem = $article->find('div', 0); - list($content, $audio_url) = $this->processContent($html, $elem); + list($content, $audio_url) = $this->processContent($dom, $elem); $item['content'] = $content; if ($audio_url != null) { $item['enclosures'] = [$audio_url]; diff --git a/bridges/EngadgetBridge.php b/bridges/EngadgetBridge.php index c219c0ff..3253cc2e 100644 --- a/bridges/EngadgetBridge.php +++ b/bridges/EngadgetBridge.php @@ -10,26 +10,28 @@ class EngadgetBridge extends FeedExpander public function collectData() { + $url = 'https://www.engadget.com/rss.xml'; $max = 10; - $this->collectExpandableDatas(static::URI . 'rss.xml', $max); + $this->collectExpandableDatas($url, $max); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); - $url = (string) $newsItem->link; - if (!$url) { + $item = parent::parseItem($item); + + $itemUrl = trim($item['uri']); + if (!$itemUrl) { return $item; } // todo: remove querystring tracking - $articlePage = getSimpleHTMLDOM($url); + $dom = getSimpleHTMLDOM($itemUrl); // figure contain's the main article image - $article = $articlePage->find('figure', 0); + $article = $dom->find('figure', 0); // .article-text has the actual article - foreach ($articlePage->find('.article-text') as $element) { + foreach ($dom->find('.article-text') as $element) { $article = $article . $element; } - $item['content'] = $article; + $item['content'] = $article ?? ''; return $item; } } diff --git a/bridges/EsquerdaNetBridge.php b/bridges/EsquerdaNetBridge.php index ffb4fd4e..64a6949f 100644 --- a/bridges/EsquerdaNetBridge.php +++ b/bridges/EsquerdaNetBridge.php @@ -1,5 +1,8 @@ getInput('feed'); - return self::URI . '/rss/' . $type; - } - - public function getIcon() - { - return 'https://www.esquerda.net/sites/default/files/favicon_0.ico'; - } - public function collectData() { parent::collectExpandableDatas($this->getURI()); } - protected function parseItem($newsItem) + protected function parseItem($item) { - # Fix Publish date - $badDate = $newsItem->pubDate; - preg_match('|(?P\d\d)/(?P\d\d)/(?P\d\d\d\d) - (?P\d\d):(?P\d\d)|', $badDate, $d); - $newsItem->pubDate = sprintf('%s-%s-%sT%s:%s', $d['year'], $d['month'], $d['day'], $d['hour'], $d['minute']); - $item = parent::parseItem($newsItem); - # Include all the content - $uri = $item['uri']; - $html = getSimpleHTMLDOMCached($uri); + $item = parent::parseItem($item); + + $html = getSimpleHTMLDOMCached($item['uri']); $content = $html->find('div#content div.content', 0); ## Fix author $authorHTML = $html->find('.field-name-field-op-author a', 0); @@ -72,4 +59,15 @@ class EsquerdaNetBridge extends FeedExpander $item['content'] = $content; return $item; } + + public function getURI() + { + $type = $this->getInput('feed'); + return self::URI . '/rss/' . $type; + } + + public function getIcon() + { + return 'https://www.esquerda.net/sites/default/files/favicon_0.ico'; + } } diff --git a/bridges/FeedExpanderExampleBridge.php b/bridges/FeedExpanderExampleBridge.php index 0e9ae386..f0af64f4 100644 --- a/bridges/FeedExpanderExampleBridge.php +++ b/bridges/FeedExpanderExampleBridge.php @@ -43,9 +43,4 @@ class FeedExpanderExampleBridge extends FeedExpander returnClientError('Unknown version ' . $this->getInput('version') . '!'); } } - - protected function parseItem($newsItem) - { - return (array) $newsItem; - } } diff --git a/bridges/FeedExpanderTestBridge.php b/bridges/FeedExpanderTestBridge.php new file mode 100644 index 00000000..9a6e7bb7 --- /dev/null +++ b/bridges/FeedExpanderTestBridge.php @@ -0,0 +1,23 @@ +collectExpandableDatas($url); + } +} diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index 1d920f90..3e3e812d 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -82,9 +82,9 @@ class FilterBridge extends FeedExpander $this->collectExpandableDatas($this->getURI()); } - protected function parseItem($newItem) + protected function parseItem($item) { - $item = parent::parseItem($newItem); + $item = parent::parseItem($item); // Generate title from first 50 characters of content? if ($this->getInput('title_from_content') && array_key_exists('content', $item)) { diff --git a/bridges/ForGifsBridge.php b/bridges/ForGifsBridge.php index 03848d04..e210124a 100644 --- a/bridges/ForGifsBridge.php +++ b/bridges/ForGifsBridge.php @@ -12,12 +12,12 @@ class ForGifsBridge extends FeedExpander $this->collectExpandableDatas('https://forgifs.com/gallery/srss/7'); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); - $content = str_get_html($item['content']); - $img = $content->find('img', 0); + $dom = str_get_html($item['content']); + $img = $dom->find('img', 0); $poster = $img->src; // The actual gif is the same path but its id must be decremented by one. @@ -34,7 +34,7 @@ class ForGifsBridge extends FeedExpander $img->width = 'auto'; $img->height = 'auto'; - $item['content'] = $content; + $item['content'] = (string) $dom; return $item; } diff --git a/bridges/FreeCodeCampBridge.php b/bridges/FreeCodeCampBridge.php index 89d8c53a..141746d2 100644 --- a/bridges/FreeCodeCampBridge.php +++ b/bridges/FreeCodeCampBridge.php @@ -14,15 +14,17 @@ class FreeCodeCampBridge extends FeedExpander $this->collectExpandableDatas('https://www.freecodecamp.org/news/rss/', 15); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); - // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); + $item = parent::parseItem($item); + + $dom = getSimpleHTMLDOM($item['uri']); + // figure contain's the main article image - $article = $articlePage->find('figure', 0); + $article = $dom->find('figure', 0); + // the actual article - foreach ($articlePage->find('.post-full-content') as $element) { + foreach ($dom->find('.post-full-content') as $element) { $article = $article . $element; } $item['content'] = $article; diff --git a/bridges/FuturaSciencesBridge.php b/bridges/FuturaSciencesBridge.php index 3fb8aafa..cfb2d711 100644 --- a/bridges/FuturaSciencesBridge.php +++ b/bridges/FuturaSciencesBridge.php @@ -85,13 +85,14 @@ class FuturaSciencesBridge extends FeedExpander $this->collectExpandableDatas($url, 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + $item['uri'] = str_replace('#xtor%3DRSS-8', '', $item['uri']); - $article = getSimpleHTMLDOMCached($item['uri']); - $item['content'] = $this->extractArticleContent($article); - $author = $this->extractAuthor($article); + $dom = getSimpleHTMLDOMCached($item['uri']); + $item['content'] = $this->extractArticleContent($dom); + $author = $this->extractAuthor($dom); if (!empty($author)) { $item['author'] = $author; } diff --git a/bridges/HardwareInfoBridge.php b/bridges/HardwareInfoBridge.php index e295984c..6a47df66 100644 --- a/bridges/HardwareInfoBridge.php +++ b/bridges/HardwareInfoBridge.php @@ -9,15 +9,15 @@ class HardwareInfoBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas('https://nl.hardware.info/updates/all.rss', 20); + $this->collectExpandableDatas('https://nl.hardware.info/updates/all.rss', 10); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); - //get full article - $articlePage = getSimpleHTMLDOMCached($feedItem->link); + $itemUrl = $item['uri']; + $articlePage = getSimpleHTMLDOMCached($itemUrl); $article = $articlePage->find('div.article__content', 0); diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index dfda311c..434e7514 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -125,9 +125,10 @@ class HeiseBridge extends FeedExpander ); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); + $sessioncookie = $this->getInput('sessioncookie'); // strip rss parameter diff --git a/bridges/IGNBridge.php b/bridges/IGNBridge.php index d00b6a18..c0260cbd 100644 --- a/bridges/IGNBridge.php +++ b/bridges/IGNBridge.php @@ -10,17 +10,16 @@ class IGNBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas('http://feeds.ign.com/ign/all', 15); + $this->collectExpandableDatas('http://feeds.ign.com/ign/all', 2); } // IGNs feed is both hidden and incomplete. This bridge tries to fix this. - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); - // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); + $articlePage = getSimpleHTMLDOM($item['uri']); // List of BS elements $uselessElements = [ @@ -33,7 +32,7 @@ class IGNBridge extends FeedExpander '.jsx-4213937408', '.commerce-container', '.widget-container', - '.newsletter-signup-button' + '.newsletter-signup-button', ]; // Remove useless elements diff --git a/bridges/LeMondeInformatiqueBridge.php b/bridges/LeMondeInformatiqueBridge.php index 678e405f..c91a0437 100644 --- a/bridges/LeMondeInformatiqueBridge.php +++ b/bridges/LeMondeInformatiqueBridge.php @@ -12,9 +12,10 @@ class LeMondeInformatiqueBridge extends FeedExpander $this->collectExpandableDatas(self::URI . 'rss/rss.xml', 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + $article_html = getSimpleHTMLDOMCached($item['uri']); //Deduce thumbnail URL from article image URL diff --git a/bridges/ListverseBridge.php b/bridges/ListverseBridge.php index ba6d7397..b7acbdd0 100644 --- a/bridges/ListverseBridge.php +++ b/bridges/ListverseBridge.php @@ -13,12 +13,11 @@ class ListverseBridge extends FeedExpander $this->collectExpandableDatas('https://listverse.com/feed/', 15); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); - // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); - $article = $articlePage->find('#articlecontentonly', 0); + $item = parent::parseItem($item); + $dom = getSimpleHTMLDOM($item['uri']); + $article = $dom->find('#articlecontentonly', 0); $item['content'] = $article; return $item; } diff --git a/bridges/MediapartBridge.php b/bridges/MediapartBridge.php index 3c8c8317..c4deda61 100644 --- a/bridges/MediapartBridge.php +++ b/bridges/MediapartBridge.php @@ -29,9 +29,11 @@ class MediapartBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + + $itemUrl = $item['uri']; // Mediapart provide multiple type of contents. // We only process items relative to the newspaper @@ -49,12 +51,8 @@ class MediapartBridge extends FeedExpander $opt = []; $opt[CURLOPT_COOKIE] = 'MPSESSID=' . $mpsessid; - // Get the page - $articlePage = getSimpleHTMLDOM( - $newsItem->link . '?onglet=full', - [], - $opt - ); + $pageUrl = $itemUrl . '?onglet=full'; + $articlePage = getSimpleHTMLDOM($pageUrl, [], $opt); // Extract the article content $content = $articlePage->find('div.content-article', 0)->innertext; diff --git a/bridges/MsnMondeBridge.php b/bridges/MsnMondeBridge.php index 844aa4a2..9b308b99 100644 --- a/bridges/MsnMondeBridge.php +++ b/bridges/MsnMondeBridge.php @@ -22,17 +22,19 @@ class MsnMondeBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas(self::FEED_URL, self::LIMIT); + $this->collectExpandableDatas(self::FEED_URL, 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + if (!preg_match('#fr-fr/actualite.*/ar-(?[\w]*)\?#', $item['uri'], $matches)) { - return; + return null; } - $json = json_decode(getContents(self::JSON_URL . $matches['id']), true); + $jsonString = getContents(self::JSON_URL . $matches['id']); + $json = json_decode($jsonString, true); $item['content'] = $json['body']; if (!empty($json['authors'])) { $item['author'] = reset($json['authors'])['name']; diff --git a/bridges/NYTBridge.php b/bridges/NYTBridge.php index 46ede3f8..57c3e2af 100644 --- a/bridges/NYTBridge.php +++ b/bridges/NYTBridge.php @@ -10,17 +10,18 @@ class NYTBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas('https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml', 40); + $url = 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml'; + $this->collectExpandableDatas($url, 40); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + $article = ''; - // $articlePage gets the entire page's contents try { - $articlePage = getSimpleHTMLDOM($newsItem->link); + $articlePage = getSimpleHTMLDOM($item['uri']); } catch (HttpException $e) { // 403 Forbidden, This means we got anti-bot response if ($e->getCode() === 403) { diff --git a/bridges/NextInpactBridge.php b/bridges/NextInpactBridge.php index 408fd783..0260da14 100644 --- a/bridges/NextInpactBridge.php +++ b/bridges/NextInpactBridge.php @@ -88,9 +88,10 @@ class NextInpactBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + $item['content'] = $this->extractContent($item, $item['uri']); if (is_null($item['content'])) { return null; //Filtered article diff --git a/bridges/NextgovBridge.php b/bridges/NextgovBridge.php index ad17f88e..1e096d6c 100644 --- a/bridges/NextgovBridge.php +++ b/bridges/NextgovBridge.php @@ -26,7 +26,8 @@ class NextgovBridge extends FeedExpander public function collectData() { - $this->collectExpandableDatas(self::URI . 'rss/' . $this->getInput('category') . '/', 10); + $url = self::URI . 'rss/' . $this->getInput('category') . '/'; + $this->collectExpandableDatas($url, 10); } protected function parseItem($newsItem) diff --git a/bridges/NiceMatinBridge.php b/bridges/NiceMatinBridge.php index 6e622b42..bcebbbbb 100644 --- a/bridges/NiceMatinBridge.php +++ b/bridges/NiceMatinBridge.php @@ -12,9 +12,10 @@ class NiceMatinBridge extends FeedExpander $this->collectExpandableDatas(self::URI . 'derniere-minute/rss', 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + $item['content'] = $this->extractContent($item['uri']); return $item; } diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php index 217db377..3b5ad3ad 100644 --- a/bridges/NyaaTorrentsBridge.php +++ b/bridges/NyaaTorrentsBridge.php @@ -65,12 +65,14 @@ class NyaaTorrentsBridge extends FeedExpander $this->collectExpandableDatas($this->getURI(), 20); } - protected function parseItem($newItem) + protected function parseItem($newsItem) { - $item = parent::parseItem($newItem); + $item = parent::parseItem($newsItem); + + $nyaaFields = (array)($newsItem->children('nyaa', true)); + $item['id'] = str_replace(['https://nyaa.si/download/', '.torrent'], '', $item['uri']); - $nyaaFields = (array)($newItem->children('nyaa', true)); $item = array_merge($item, $nyaaFields); // Convert URI from torrent file to web page diff --git a/bridges/OnVaSortirBridge.php b/bridges/OnVaSortirBridge.php index af80dd31..9f9a750c 100644 --- a/bridges/OnVaSortirBridge.php +++ b/bridges/OnVaSortirBridge.php @@ -117,18 +117,18 @@ class OnVaSortirBridge extends FeedExpander ] ]; + public function collectData() + { + $url = 'https://' . $this->getInput('city') . '.onvasortir.com/rss.php'; + $this->collectExpandableDatas($url); + } + protected function parseItem($item) { $item = parent::parseItem($item); - $html = getSimpleHTMLDOMCached($item['uri']); - $text = $html->find('div.corpsMax', 0)->innertext; + $dom = getSimpleHTMLDOMCached($item['uri']); + $text = $dom->find('div.corpsMax', 0)->innertext; $item['content'] = utf8_encode($text); return $item; } - - public function collectData() - { - $this->collectExpandableDatas('https://' . - $this->getInput('city') . '.onvasortir.com/rss.php'); - } } diff --git a/bridges/PhoronixBridge.php b/bridges/PhoronixBridge.php index 620fda66..fc0d78e5 100644 --- a/bridges/PhoronixBridge.php +++ b/bridges/PhoronixBridge.php @@ -29,22 +29,25 @@ but some RSS readers don\'t support this. "img" tag are supported by most browse $this->collectExpandableDatas('https://www.phoronix.com/rss.php', $this->getInput('n')); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); - // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); + $item = parent::parseItem($item); + + $itemUrl = $item['uri']; + + $articlePage = getSimpleHTMLDOM($itemUrl); $articlePage = defaultLinkTo($articlePage, $this->getURI()); // Extract final link. From Facebook's like plugin. - parse_str(parse_url($articlePage->find('iframe[src^=//www.facebook.com/plugins]', 0), PHP_URL_QUERY), $facebookQuery); + $parsedUrlQuery = parse_url($articlePage->find('iframe[src^=//www.facebook.com/plugins]', 0), PHP_URL_QUERY); + parse_str($parsedUrlQuery, $facebookQuery); if (array_key_exists('href', $facebookQuery)) { - $newsItem->link = $facebookQuery['href']; + $itemUrl = $facebookQuery['href']; } $item['content'] = $this->extractContent($articlePage); $pages = $articlePage->find('.pagination a[!title]'); foreach ($pages as $page) { - $pageURI = urljoin($newsItem->link, html_entity_decode($page->href)); + $pageURI = urljoin($itemUrl, html_entity_decode($page->href)); $page = getSimpleHTMLDOM($pageURI); $item['content'] .= $this->extractContent($page); } diff --git a/bridges/QwantzBridge.php b/bridges/QwantzBridge.php index e48e948a..2117c33c 100644 --- a/bridges/QwantzBridge.php +++ b/bridges/QwantzBridge.php @@ -6,9 +6,15 @@ class QwantzBridge extends FeedExpander const URI = 'https://qwantz.com/'; const DESCRIPTION = 'Latest comic.'; - protected function parseItem($feedItem) + public function collectData() { - $item = parent::parseItem($feedItem); + $this->collectExpandableDatas(self::URI . 'rssfeed.php'); + } + + protected function parseItem($item) + { + $item = parent::parseItem($item); + $item['author'] = 'Ryan North'; preg_match('/title="(.*?)"/', $item['content'], $matches); @@ -25,11 +31,6 @@ class QwantzBridge extends FeedExpander return $item; } - public function collectData() - { - $this->collectExpandableDatas(self::URI . 'rssfeed.php'); - } - public function getIcon() { return self::URI . 'favicon.ico'; diff --git a/bridges/RaceDepartmentBridge.php b/bridges/RaceDepartmentBridge.php index 7fe92b4a..3783b53e 100644 --- a/bridges/RaceDepartmentBridge.php +++ b/bridges/RaceDepartmentBridge.php @@ -12,12 +12,11 @@ class RaceDepartmentBridge extends FeedExpander $this->collectExpandableDatas('https://www.racedepartment.com/ams/index.rss', 10); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); - //fetch page - $articlePage = getSimpleHTMLDOMCached($feedItem->link); + $articlePage = getSimpleHTMLDOMCached($item['uri']); $coverImage = $articlePage->find('img.js-articleCoverImage', 0); #relative url -> absolute url diff --git a/bridges/ScribbleHubBridge.php b/bridges/ScribbleHubBridge.php index 4fe43df3..8f52d461 100644 --- a/bridges/ScribbleHubBridge.php +++ b/bridges/ScribbleHubBridge.php @@ -42,9 +42,9 @@ class ScribbleHubBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($newItem) + protected function parseItem($item) { - $item = parent::parseItem($newItem); + $item = parent::parseItem($item); //For series, filter out other series from 'All' feed if ( @@ -57,7 +57,7 @@ class ScribbleHubBridge extends FeedExpander $item['comments'] = $item['uri'] . '#comments'; try { - $item_html = getSimpleHTMLDOMCached($item['uri']); + $dom = getSimpleHTMLDOMCached($item['uri']); } catch (HttpException $e) { // 403 Forbidden, This means we got anti-bot response if ($e->getCode() === 403) { @@ -66,22 +66,22 @@ class ScribbleHubBridge extends FeedExpander throw $e; } - $item_html = defaultLinkTo($item_html, self::URI); + $dom = defaultLinkTo($dom, self::URI); //Retrieve full description from page contents - $item['content'] = $item_html->find('#chp_raw', 0); + $item['content'] = $dom->find('#chp_raw', 0); //Retrieve image for thumbnail - $item_image = $item_html->find('.s_novel_img > img', 0)->src; + $item_image = $dom->find('.s_novel_img > img', 0)->src; $item['enclosures'] = [$item_image]; //Restore lost categories - $item_story = html_entity_decode($item_html->find('.chp_byauthor > a', 0)->innertext); - $item_sid = $item_html->find('#mysid', 0)->value; + $item_story = html_entity_decode($dom->find('.chp_byauthor > a', 0)->innertext); + $item_sid = $dom->find('#mysid', 0)->value; $item['categories'] = [$item_story, $item_sid]; //Generate UID - $item_pid = $item_html->find('#mypostid', 0)->value; + $item_pid = $dom->find('#mypostid', 0)->value; $item['uid'] = $item_sid . "/$item_pid"; return $item; diff --git a/bridges/SplCenterBridge.php b/bridges/SplCenterBridge.php index a590558d..ca764846 100644 --- a/bridges/SplCenterBridge.php +++ b/bridges/SplCenterBridge.php @@ -21,6 +21,12 @@ class SplCenterBridge extends FeedExpander const CACHE_TIMEOUT = 3600; // 1 hour + public function collectData() + { + $url = $this->getURI() . '/rss.xml'; + $this->collectExpandableDatas($url); + } + protected function parseItem($item) { $item = parent::parseItem($item); @@ -37,11 +43,6 @@ class SplCenterBridge extends FeedExpander return $item; } - public function collectData() - { - $this->collectExpandableDatas($this->getURI() . '/rss.xml'); - } - public function getURI() { if (!is_null($this->getInput('content'))) { diff --git a/bridges/TapasBridge.php b/bridges/TapasBridge.php index e512ad48..11a9551d 100644 --- a/bridges/TapasBridge.php +++ b/bridges/TapasBridge.php @@ -30,14 +30,17 @@ class TapasBridge extends FeedExpander protected $id; - public function getURI() + public function collectData() { - if ($this->id) { - return self::URI . 'rss/series/' . $this->id; - } else { - return self::URI . 'series/' . $this->getInput('title') . '/info/'; + if (preg_match('/^[\d]+$/', $this->getInput('title'))) { + $this->id = $this->getInput('title'); } - return self::URI; + if ($this->getInput('force_title') or !$this->id) { + $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request ' . $this->getURI()); + $this->id = $html->find('meta[property$=":url"]', 0)->content; + $this->id = str_ireplace(['tapastic://series/', '/info'], '', $this->id); + } + $this->collectExpandableDatas($this->getURI()); } protected function parseItem($feedItem) @@ -72,16 +75,13 @@ class TapasBridge extends FeedExpander return $item; } - public function collectData() + public function getURI() { - if (preg_match('/^[\d]+$/', $this->getInput('title'))) { - $this->id = $this->getInput('title'); + if ($this->id) { + return self::URI . 'rss/series/' . $this->id; + } else { + return self::URI . 'series/' . $this->getInput('title') . '/info/'; } - if ($this->getInput('force_title') or !$this->id) { - $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request ' . $this->getURI()); - $this->id = $html->find('meta[property$=":url"]', 0)->content; - $this->id = str_ireplace(['tapastic://series/', '/info'], '', $this->id); - } - $this->collectExpandableDatas($this->getURI()); + return self::URI; } } diff --git a/bridges/TheGuardianBridge.php b/bridges/TheGuardianBridge.php index e05bde75..98e56506 100644 --- a/bridges/TheGuardianBridge.php +++ b/bridges/TheGuardianBridge.php @@ -52,18 +52,15 @@ class TheGuardianBridge extends FeedExpander public function collectData() { $feed = $this->getInput('feed'); - $feedURL = 'https://feeds.theguardian.com/theguardian/' . $feed; - $this->collectExpandableDatas($feedURL, 10); + $url = 'https://feeds.theguardian.com/theguardian/' . $feed; + $this->collectExpandableDatas($url, 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); - // --- Recovering the article --- - - // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); + $articlePage = getSimpleHTMLDOM($item['uri']); // figure contain's the main article image $article = $articlePage->find('figure', 0); // content__article-body has the actual article diff --git a/bridges/TwitterEngineeringBridge.php b/bridges/TwitterEngineeringBridge.php index fa55b4a4..b98cfb87 100644 --- a/bridges/TwitterEngineeringBridge.php +++ b/bridges/TwitterEngineeringBridge.php @@ -8,18 +8,24 @@ class TwitterEngineeringBridge extends FeedExpander const DESCRIPTION = 'Returns the newest articles.'; const CACHE_TIMEOUT = 21600; // 6h + public function collectData() + { + $url = 'https://blog.twitter.com/engineering/en_us/blog.rss'; + $this->collectExpandableDatas($url); + } + protected function parseItem($item) { $item = parent::parseItem($item); - $article_html = getSimpleHTMLDOMCached($item['uri']); - if (!$article_html) { + $dom = getSimpleHTMLDOMCached($item['uri']); + if (!$dom) { $item['content'] .= '

Could not request ' . $this->getName() . ': ' . $item['uri'] . '

'; return $item; } - $article_html = defaultLinkTo($article_html, $this->getURI()); + $dom = defaultLinkTo($dom, $this->getURI()); - $article_body = $article_html->find('div.column.column-6', 0); + $article_body = $dom->find('div.column.column-6', 0); // Remove elements that are not part of article content $unwanted_selector = 'div.bl02-blog-post-text-masthead, div.tweet-error-text, div.bl13-tweet-template'; @@ -33,8 +39,8 @@ class TwitterEngineeringBridge extends FeedExpander } $item['content'] = $article_body; - $item['timestamp'] = strtotime($article_html->find('span.b02-blog-post-no-masthead__date', 0)->innertext); - $item['categories'] = self::getCategoriesFromTags($article_html); + $item['timestamp'] = strtotime($dom->find('span.b02-blog-post-no-masthead__date', 0)->innertext); + $item['categories'] = self::getCategoriesFromTags($dom); return $item; } @@ -53,12 +59,6 @@ class TwitterEngineeringBridge extends FeedExpander return $categories; } - public function collectData() - { - $feed = static::URI . 'en_us/blog.rss'; - $this->collectExpandableDatas($feed); - } - public function getName() { // Else the original feed returns "English (US)" as the title diff --git a/bridges/VarietyBridge.php b/bridges/VarietyBridge.php index 23d1df3f..6625dca2 100644 --- a/bridges/VarietyBridge.php +++ b/bridges/VarietyBridge.php @@ -13,11 +13,11 @@ class VarietyBridge extends FeedExpander $this->collectExpandableDatas('https://feeds.feedburner.com/variety/headlines', 15); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); + $articlePage = getSimpleHTMLDOM($item['uri']); // Remove Script tags foreach ($articlePage->find('script') as $script_tag) { diff --git a/bridges/ViceBridge.php b/bridges/ViceBridge.php index 35414020..c7ecec33 100644 --- a/bridges/ViceBridge.php +++ b/bridges/ViceBridge.php @@ -32,14 +32,14 @@ class ViceBridge extends FeedExpander $this->collectExpandableDatas($feedURL, 10); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); - // $articlePage gets the entire page's contents - $articlePage = getSimpleHTMLDOM($newsItem->link); + $item = parent::parseItem($item); + + $articlePage = getSimpleHTMLDOM($item['uri']); // text and embedded content $article = $articlePage->find('.article__body', 0); - $item['content'] = $article; + $item['content'] = $article ?? ''; return $item; } diff --git a/bridges/WiredBridge.php b/bridges/WiredBridge.php index d4c7cbbb..7f7f6051 100644 --- a/bridges/WiredBridge.php +++ b/bridges/WiredBridge.php @@ -50,13 +50,16 @@ class WiredBridge extends FeedExpander $this->collectExpandableDatas($feed_url, $limit); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); + + $originalContent = $item['content']; + $article = getSimpleHTMLDOMCached($item['uri']); $item['content'] = $this->extractArticleContent($article); - $headline = strval($newsItem->description); + $headline = $originalContent; if (!empty($headline)) { $item['content'] = '

' . $headline . '

' . $item['content']; } diff --git a/bridges/WordPressBridge.php b/bridges/WordPressBridge.php index 1d46958d..fb27eb31 100644 --- a/bridges/WordPressBridge.php +++ b/bridges/WordPressBridge.php @@ -20,50 +20,56 @@ class WordPressBridge extends FeedExpander ], ]]; - private function cleanContent($content) + public function collectData() { - $content = stripWithDelimiters($content, ''); - $content = preg_replace('/
/', '', $content); - return $content; + $limit = $this->getInput('limit') ?? 10; + if ($this->getInput('url') && substr($this->getInput('url'), 0, strlen('http')) !== 'http') { + // just in case someone find a way to access local files by playing with the url + returnClientError('The url parameter must either refer to http or https protocol.'); + } + try { + $this->collectExpandableDatas($this->getURI() . '/feed/atom/', $limit); + } catch (Exception $e) { + $this->collectExpandableDatas($this->getURI() . '/?feed=atom', $limit); + } } - protected function parseItem($newItem) + protected function parseItem($item) { - $item = parent::parseItem($newItem); + $item = parent::parseItem($item); - $article_html = getSimpleHTMLDOMCached($item['uri']); + $dom = getSimpleHTMLDOMCached($item['uri']); // Find article body $article = null; switch (true) { case !empty($this->getInput('content-selector')): // custom contect selector (manually specified by user) - $article = $article_html->find($this->getInput('content-selector'), 0); + $article = $dom->find($this->getInput('content-selector'), 0); break; - case !is_null($article_html->find('[itemprop=articleBody]', 0)): + case !is_null($dom->find('[itemprop=articleBody]', 0)): // highest priority content div (used for SEO) - $article = $article_html->find('[itemprop=articleBody]', 0); + $article = $dom->find('[itemprop=articleBody]', 0); break; - case !is_null($article_html->find('.article-content', 0)): + case !is_null($dom->find('.article-content', 0)): // more precise than article when present - $article = $article_html->find('.article-content', 0); + $article = $dom->find('.article-content', 0); break; - case !is_null($article_html->find('article', 0)): + case !is_null($dom->find('article', 0)): // most common content div - $article = $article_html->find('article', 0); + $article = $dom->find('article', 0); break; - case !is_null($article_html->find('.single-content', 0)): + case !is_null($dom->find('.single-content', 0)): // another common content div - $article = $article_html->find('.single-content', 0); + $article = $dom->find('.single-content', 0); break; - case !is_null($article_html->find('.post-content', 0)): + case !is_null($dom->find('.post-content', 0)): // another common content div - $article = $article_html->find('.post-content', 0); + $article = $dom->find('.post-content', 0); break; - case !is_null($article_html->find('.post', 0)): + case !is_null($dom->find('.post', 0)): // for old WordPress themes without HTML5 - $article = $article_html->find('.post', 0); + $article = $dom->find('.post', 0); break; } @@ -76,7 +82,7 @@ class WordPressBridge extends FeedExpander // Find article main image $article = convertLazyLoading($article); - $article_image = $article_html->find('img.wp-post-image', 0); + $article_image = $dom->find('img.wp-post-image', 0); if (!empty($item['content']) && (!is_object($article_image) || empty($article_image->src))) { $article_image = str_get_html($item['content'])->find('img.wp-post-image', 0); } @@ -106,6 +112,14 @@ class WordPressBridge extends FeedExpander return $item; } + private function cleanContent($content) + { + $content = stripWithDelimiters($content, ''); + $content = preg_replace('/
/', '', $content); + return $content; + } + public function getURI() { $url = $this->getInput('url'); @@ -114,18 +128,4 @@ class WordPressBridge extends FeedExpander } return $url; } - - public function collectData() - { - $limit = $this->getInput('limit') ?? 10; - if ($this->getInput('url') && substr($this->getInput('url'), 0, strlen('http')) !== 'http') { - // just in case someone find a way to access local files by playing with the url - returnClientError('The url parameter must either refer to http or https protocol.'); - } - try { - $this->collectExpandableDatas($this->getURI() . '/feed/atom/', $limit); - } catch (Exception $e) { - $this->collectExpandableDatas($this->getURI() . '/?feed=atom', $limit); - } - } } diff --git a/bridges/WorldOfTanksBridge.php b/bridges/WorldOfTanksBridge.php index 6e7a594b..52691025 100644 --- a/bridges/WorldOfTanksBridge.php +++ b/bridges/WorldOfTanksBridge.php @@ -30,9 +30,9 @@ class WorldOfTanksBridge extends FeedExpander $this->collectExpandableDatas(sprintf('https://worldoftanks.eu/%s/rss/news/', $this->getInput('lang'))); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); $item['content'] = $this->loadFullArticle($item['uri']); return $item; } diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index b294e9fb..7d7e89aa 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -50,19 +50,19 @@ class ZeitBridge extends FeedExpander 'defaultValue' => 5 ] ]]; - const LIMIT = 5; public function collectData() { - $this->collectExpandableDatas( - $this->getInput('category'), - $this->getInput('limit') ?: static::LIMIT - ); + $url = $this->getInput('category'); + $limit = $this->getInput('limit') ?: 5; + + $this->collectExpandableDatas($url, $limit); } protected function parseItem($item) { $item = parent::parseItem($item); + $item['enclosures'] = []; $headers = [ From 44fb2c98bcc49d5d83186abc2dc05bf9b6eba60c Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 00:26:11 +0200 Subject: [PATCH 166/716] fix: various fixes (#3745) --- bridges/CVEDetailsBridge.php | 2 +- bridges/CubariBridge.php | 18 ++++++++++-------- lib/FeedParser.php | 7 +++++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/bridges/CVEDetailsBridge.php b/bridges/CVEDetailsBridge.php index 27b4008c..b52d290e 100644 --- a/bridges/CVEDetailsBridge.php +++ b/bridges/CVEDetailsBridge.php @@ -57,7 +57,7 @@ class CVEDetailsBridge extends BridgeAbstract $content = $tr->find('.cvesummarylong', 0)->innertext; $cweList = $detailHtml->find('h2', 2)->next_sibling(); foreach ($cweList->find('li') as $li) { - $cweWithDescription = $li->find('a', 0)->innertext; + $cweWithDescription = $li->find('a', 0)->innertext ?? ''; if (preg_match('/CWE-(\d+)/', $cweWithDescription, $cwe)) { $categories[] = 'CWE-' . $cwe[1]; diff --git a/bridges/CubariBridge.php b/bridges/CubariBridge.php index 9a08dbba..a7b6d69d 100644 --- a/bridges/CubariBridge.php +++ b/bridges/CubariBridge.php @@ -47,8 +47,8 @@ class CubariBridge extends BridgeAbstract */ public function collectData() { - $jsonSite = getContents($this->getInput('gist')); - $jsonFile = json_decode($jsonSite, true); + $json = getContents($this->getInput('gist')); + $jsonFile = json_decode($json, true); $this->mangaTitle = $jsonFile['title']; @@ -66,12 +66,14 @@ class CubariBridge extends BridgeAbstract { $url = $this->getInput('gist'); - preg_match('/\/([a-z]*)\.githubusercontent.com(.*)/', $url, $matches); - - // raw or gist is first match. - $unencoded = $matches[1] . $matches[2]; - - return base64_encode($unencoded); + if (preg_match('/\/([a-z]*)\.githubusercontent.com(.*)/', $url, $matches)) { + // raw or gist is first match. + $unencoded = $matches[1] . $matches[2]; + return base64_encode($unencoded); + } else { + // todo: fix this + return ''; + } } private function getSanitizedHash($string) diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 90df548d..a9aabde0 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -64,7 +64,7 @@ final class FeedParser $feed['items'][] = $this->parseAtomItem($item); } } else { - throw new \Exception(sprintf('Unable to detect feed format from `%s`', $url)); + throw new \Exception('Unable to detect feed format'); } return $feed; @@ -163,7 +163,9 @@ final class FeedParser } if (isset($feedItem->enclosure) && !empty($feedItem->enclosure['url'])) { - $item['enclosures'] = [(string)$feedItem->enclosure['url']]; + $item['enclosures'] = [ + (string)$feedItem->enclosure['url'], + ]; } return $item; } @@ -189,6 +191,7 @@ final class FeedParser { $item = []; if (isset($feedItem->link)) { + // todo: trim uri $item['uri'] = (string)$feedItem->link; } if (isset($feedItem->title)) { From e379019db27eb7e6c48a9e8c6ae7d3b379dcce16 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 01:02:19 +0200 Subject: [PATCH 167/716] refactor (#3746) --- bridges/NextgovBridge.php | 33 ++++++++-------- bridges/NyaaTorrentsBridge.php | 71 ++++++++++++++++++++-------------- bridges/TapasBridge.php | 22 +++++------ lib/FeedParser.php | 3 ++ 4 files changed, 72 insertions(+), 57 deletions(-) diff --git a/bridges/NextgovBridge.php b/bridges/NextgovBridge.php index 1e096d6c..bc92d306 100644 --- a/bridges/NextgovBridge.php +++ b/bridges/NextgovBridge.php @@ -27,29 +27,30 @@ class NextgovBridge extends FeedExpander public function collectData() { $url = self::URI . 'rss/' . $this->getInput('category') . '/'; - $this->collectExpandableDatas($url, 10); + $limit = 10; + $this->collectExpandableDatas($url, $limit); } - protected function parseItem($newsItem) + protected function parseItem($item) { - $item = parent::parseItem($newsItem); + $item = parent::parseItem($item); $article_thumbnail = 'https://cdn.nextgov.com/nextgov/images/logo.png'; $item['content'] = '

' . $item['content'] . '

'; - $namespaces = $newsItem->getNamespaces(true); - if (isset($namespaces['media'])) { - $media = $newsItem->children($namespaces['media']); - if (isset($media->content)) { - $attributes = $media->content->attributes(); - $item['content'] = '

' . $item['content']; - $article_thumbnail = str_replace( - 'large.jpg', - 'small.jpg', - strval($attributes['url']) - ); - } - } +// $namespaces = $newsItem->getNamespaces(true); +// if (isset($namespaces['media'])) { +// $media = $newsItem->children($namespaces['media']); +// if (isset($media->content)) { +// $attributes = $media->content->attributes(); +// $item['content'] = '

' . $item['content']; +// $article_thumbnail = str_replace( +// 'large.jpg', +// 'small.jpg', +// strval($attributes['url']) +// ); +// } +// } $item['enclosures'] = [$article_thumbnail]; $item['content'] .= $this->extractContent($item['uri']); diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php index 3b5ad3ad..f7eea07f 100644 --- a/bridges/NyaaTorrentsBridge.php +++ b/bridges/NyaaTorrentsBridge.php @@ -1,6 +1,6 @@ collectExpandableDatas($this->getURI(), 20); - } + // Manually parsing because we need to acccess the nyaa namespace in the xml + $xml = simplexml_load_string(getContents($this->getURI())); + $channel = $xml->channel[0]; + $feed = []; + $feed['title'] = trim((string)$channel->title); + $feed['uri'] = trim((string)$channel->link); + if (!empty($channel->image)) { + $feed['icon'] = trim((string)$channel->image->url); + } + $items = $xml->channel[0]->item; + foreach ($items as $feedItem) { + $item = [ + 'title' => (string) $feedItem->title, + 'uri' => (string) $feedItem->link, + ]; - protected function parseItem($newsItem) - { - $item = parent::parseItem($newsItem); - $nyaaFields = (array)($newsItem->children('nyaa', true)); + $item['id'] = str_replace(['https://nyaa.si/download/', '.torrent'], '', $item['uri']); - $item['id'] = str_replace(['https://nyaa.si/download/', '.torrent'], '', $item['uri']); + $nyaaNamespace = (array)($feedItem->children('nyaa', true)); + $item = array_merge($item, $nyaaNamespace); - $item = array_merge($item, $nyaaFields); + // Convert URI from torrent file to web page + $item['uri'] = str_replace('/download/', '/view/', $item['uri']); + $item['uri'] = str_replace('.torrent', '', $item['uri']); - // Convert URI from torrent file to web page - $item['uri'] = str_replace('/download/', '/view/', $item['uri']); - $item['uri'] = str_replace('.torrent', '', $item['uri']); + $item_html = getSimpleHTMLDOMCached($item['uri']); + if ($item_html) { + // Retrieve full description from page contents + $item_desc = str_get_html( + markdownToHtml(html_entity_decode($item_html->find('#torrent-description', 0)->innertext)) + ); - $item_html = getSimpleHTMLDOMCached($item['uri']); - if ($item_html) { - // Retrieve full description from page contents - $item_desc = str_get_html( - markdownToHtml(html_entity_decode($item_html->find('#torrent-description', 0)->innertext)) - ); - - // Retrieve image for thumbnail or generic logo fallback - $item_image = $this->getURI() . 'static/img/avatar/default.png'; - foreach ($item_desc->find('img') as $img) { - if (strpos($img->src, 'prez') === false) { - $item_image = $img->src; - break; + // Retrieve image for thumbnail or generic logo fallback + $item_image = $this->getURI() . 'static/img/avatar/default.png'; + foreach ($item_desc->find('img') as $img) { + if (strpos($img->src, 'prez') === false) { + $item_image = $img->src; + break; + } } + + $item['enclosures'] = [$item_image]; + $item['content'] = $item_desc; } - $item['enclosures'] = [$item_image]; - $item['content'] = $item_desc; + $this->items[] = $item; + if (count($this->items) >= 10) { + break; + } } - - return $item; } public function getIcon() diff --git a/bridges/TapasBridge.php b/bridges/TapasBridge.php index 11a9551d..ddfbfb92 100644 --- a/bridges/TapasBridge.php +++ b/bridges/TapasBridge.php @@ -43,20 +43,20 @@ class TapasBridge extends FeedExpander $this->collectExpandableDatas($this->getURI()); } - protected function parseItem($feedItem) + protected function parseItem($item) { - $item = parent::parseItem($feedItem); + $item = parent::parseItem($item); - $namespaces = $feedItem->getNamespaces(true); - if (isset($namespaces['content'])) { - $description = $feedItem->children($namespaces['content']); - if (isset($description->encoded)) { - $item['content'] = (string)$description->encoded; - } - } +// $namespaces = $feedItem->getNamespaces(true); +// if (isset($namespaces['content'])) { +// $description = $feedItem->children($namespaces['content']); +// if (isset($description->encoded)) { +// $item['content'] = (string)$description->encoded; +// } +// } if ($this->getInput('extend_content')) { - $html = getSimpleHTMLDOM($item['uri']) or returnServerError('Could not request ' . $this->getURI()); + $html = getSimpleHTMLDOM($item['uri']); if (!$item['content']) { $item['content'] = ''; } @@ -79,8 +79,6 @@ class TapasBridge extends FeedExpander { if ($this->id) { return self::URI . 'rss/series/' . $this->id; - } else { - return self::URI . 'series/' . $this->getInput('title') . '/info/'; } return self::URI; } diff --git a/lib/FeedParser.php b/lib/FeedParser.php index a9aabde0..04452e7d 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -6,7 +6,10 @@ final class FeedParser { public function parseFeed(string $xmlString): array { + libxml_use_internal_errors(true); $xml = simplexml_load_string(trim($xmlString)); + $xmlErrors = libxml_get_errors(); + libxml_use_internal_errors(false); if ($xml === false) { throw new \Exception('Unable to parse xml'); } From 2880524dfc7685985fde8429c1dcb85387f4ba14 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 01:59:05 +0200 Subject: [PATCH 168/716] refactor: remove parent calls to parseItem (#3747) --- bridges/AcrimedBridge.php | 4 +- bridges/ArsTechnicaBridge.php | 4 +- bridges/BleepingComputerBridge.php | 4 +- bridges/CNETFranceBridge.php | 4 +- bridges/CaschyBridge.php | 4 +- bridges/CommonDreamsBridge.php | 3 +- bridges/CourrierInternationalBridge.php | 4 +- bridges/DarkReadingBridge.php | 4 +- bridges/DauphineLibereBridge.php | 3 +- bridges/DeutscheWelleBridge.php | 4 +- bridges/DeveloppezDotComBridge.php | 4 +- bridges/EconomistBridge.php | 3 +- bridges/EngadgetBridge.php | 4 +- bridges/EsquerdaNetBridge.php | 4 +- bridges/FeedExpanderTestBridge.php | 6 +- bridges/FilterBridge.php | 4 +- bridges/FolhaDeSaoPauloBridge.php | 4 +- bridges/ForGifsBridge.php | 4 +- bridges/FreeCodeCampBridge.php | 4 +- bridges/FuturaSciencesBridge.php | 4 +- bridges/GizmodoBridge.php | 4 +- bridges/GolemBridge.php | 3 +- bridges/HardwareInfoBridge.php | 4 +- bridges/HeiseBridge.php | 4 +- bridges/IGNBridge.php | 4 +- bridges/KoreusBridge.php | 4 +- bridges/LeMondeInformatiqueBridge.php | 4 +- bridges/ListverseBridge.php | 3 +- bridges/MediapartBridge.php | 4 +- bridges/MsnMondeBridge.php | 4 +- bridges/NYTBridge.php | 4 +- bridges/NextInpactBridge.php | 4 +- bridges/NextgovBridge.php | 4 +- bridges/NiceMatinBridge.php | 4 +- bridges/OnVaSortirBridge.php | 3 +- bridges/PhoronixBridge.php | 4 +- bridges/QwantzBridge.php | 6 +- bridges/RaceDepartmentBridge.php | 4 +- bridges/ScribbleHubBridge.php | 4 +- bridges/SplCenterBridge.php | 4 +- bridges/TapasBridge.php | 4 +- bridges/TheGuardianBridge.php | 4 +- bridges/TwitterEngineeringBridge.php | 4 +- bridges/VarietyBridge.php | 4 +- bridges/ViceBridge.php | 4 +- bridges/WeLiveSecurityBridge.php | 4 +- bridges/WiredBridge.php | 4 +- bridges/WordPressBridge.php | 4 +- bridges/WorldOfTanksBridge.php | 3 +- bridges/ZDNetBridge.php | 4 +- bridges/ZeitBridge.php | 4 +- docs/05_Bridge_API/03_FeedExpander.md | 90 ++++++------------------- docs/05_Bridge_API/index.md | 2 +- lib/FeedExpander.php | 88 +++++------------------- lib/FeedParser.php | 8 +-- 55 files changed, 96 insertions(+), 293 deletions(-) diff --git a/bridges/AcrimedBridge.php b/bridges/AcrimedBridge.php index 93890f35..f7bbd58e 100644 --- a/bridges/AcrimedBridge.php +++ b/bridges/AcrimedBridge.php @@ -25,10 +25,8 @@ class AcrimedBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOM($item['uri']); $article = sanitize($articlePage->find('article.article1', 0)->innertext); $article = defaultLinkTo($article, static::URI); diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index 98e5566b..d15cfb4f 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -33,10 +33,8 @@ class ArsTechnicaBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $item_html = getSimpleHTMLDOMCached($item['uri'] . '&'); $item_html = defaultLinkTo($item_html, self::URI); $item['content'] = $item_html->find('.amp-wp-article-content', 0); diff --git a/bridges/BleepingComputerBridge.php b/bridges/BleepingComputerBridge.php index bad78561..79d84176 100644 --- a/bridges/BleepingComputerBridge.php +++ b/bridges/BleepingComputerBridge.php @@ -13,10 +13,8 @@ class BleepingComputerBridge extends FeedExpander $this->collectExpandableDatas($feed); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $article_html = getSimpleHTMLDOMCached($item['uri']); if (!$article_html) { $item['content'] .= '

Could not request ' . $this->getName() . ': ' . $item['uri'] . '

'; diff --git a/bridges/CNETFranceBridge.php b/bridges/CNETFranceBridge.php index d6a766de..35e92daf 100644 --- a/bridges/CNETFranceBridge.php +++ b/bridges/CNETFranceBridge.php @@ -43,10 +43,8 @@ class CNETFranceBridge extends FeedExpander $this->collectExpandableDatas('https://www.cnetfrance.fr/feeds/rss/news/'); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - foreach ($this->bannedTitle as $term) { if (preg_match('/' . $term . '/mi', $item['title']) === 1) { return null; diff --git a/bridges/CaschyBridge.php b/bridges/CaschyBridge.php index 7d632bf6..0e3a07bc 100644 --- a/bridges/CaschyBridge.php +++ b/bridges/CaschyBridge.php @@ -34,10 +34,8 @@ class CaschyBridge extends FeedExpander ); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - if (strpos($item['uri'], 'https://stadt-bremerhaven.de/') !== 0) { return $item; } diff --git a/bridges/CommonDreamsBridge.php b/bridges/CommonDreamsBridge.php index e1a185de..34532284 100644 --- a/bridges/CommonDreamsBridge.php +++ b/bridges/CommonDreamsBridge.php @@ -12,9 +12,8 @@ class CommonDreamsBridge extends FeedExpander $this->collectExpandableDatas('http://www.commondreams.org/rss.xml', 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); $item['content'] = $this->extractContent($item['uri']); return $item; } diff --git a/bridges/CourrierInternationalBridge.php b/bridges/CourrierInternationalBridge.php index 9e30fd51..3d9889b0 100644 --- a/bridges/CourrierInternationalBridge.php +++ b/bridges/CourrierInternationalBridge.php @@ -13,10 +13,8 @@ class CourrierInternationalBridge extends FeedExpander $this->collectExpandableDatas(static::URI . 'feed/all/rss.xml', 20); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOMCached($item['uri']); $content = $articlePage->find('.article-text, depeche-text', 0); if (!$content) { diff --git a/bridges/DarkReadingBridge.php b/bridges/DarkReadingBridge.php index aca30490..4f1622e3 100644 --- a/bridges/DarkReadingBridge.php +++ b/bridges/DarkReadingBridge.php @@ -56,10 +56,8 @@ class DarkReadingBridge extends FeedExpander $this->collectExpandableDatas($feed_url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $article = getSimpleHTMLDOMCached($item['uri']); $item['content'] = $this->extractArticleContent($article); $item['enclosures'] = []; //remove author profile picture diff --git a/bridges/DauphineLibereBridge.php b/bridges/DauphineLibereBridge.php index 0ab808cd..05748a5d 100644 --- a/bridges/DauphineLibereBridge.php +++ b/bridges/DauphineLibereBridge.php @@ -43,9 +43,8 @@ class DauphineLibereBridge extends FeedExpander $this->collectExpandableDatas($url, 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); $item['content'] = $this->extractContent($item['uri']); return $item; } diff --git a/bridges/DeutscheWelleBridge.php b/bridges/DeutscheWelleBridge.php index 2e10d670..29b478b9 100644 --- a/bridges/DeutscheWelleBridge.php +++ b/bridges/DeutscheWelleBridge.php @@ -71,10 +71,8 @@ class DeutscheWelleBridge extends FeedExpander $this->collectExpandableDatas($this->getInput('feed')); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $parsedUrl = parse_url($item['uri']); unset($parsedUrl['query']); $url = $this->unparseUrl($parsedUrl); diff --git a/bridges/DeveloppezDotComBridge.php b/bridges/DeveloppezDotComBridge.php index 9dcbc31a..d9583fed 100644 --- a/bridges/DeveloppezDotComBridge.php +++ b/bridges/DeveloppezDotComBridge.php @@ -176,10 +176,8 @@ class DeveloppezDotComBridge extends FeedExpander * Parse the content of every RSS item. And will try to get the full article * pointed by the item URL intead of the default abstract. */ - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - if (count($this->items) >= $this->getInput('limit')) { return null; } diff --git a/bridges/EconomistBridge.php b/bridges/EconomistBridge.php index 0572ab8f..aad72275 100644 --- a/bridges/EconomistBridge.php +++ b/bridges/EconomistBridge.php @@ -97,9 +97,8 @@ class EconomistBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); $dom = getSimpleHTMLDOM($item['uri']); $article = $dom->find('#new-article-template', 0); diff --git a/bridges/EngadgetBridge.php b/bridges/EngadgetBridge.php index 3253cc2e..b9861b40 100644 --- a/bridges/EngadgetBridge.php +++ b/bridges/EngadgetBridge.php @@ -15,10 +15,8 @@ class EngadgetBridge extends FeedExpander $this->collectExpandableDatas($url, $max); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $itemUrl = trim($item['uri']); if (!$itemUrl) { return $item; diff --git a/bridges/EsquerdaNetBridge.php b/bridges/EsquerdaNetBridge.php index 64a6949f..aa92aa38 100644 --- a/bridges/EsquerdaNetBridge.php +++ b/bridges/EsquerdaNetBridge.php @@ -31,10 +31,8 @@ class EsquerdaNetBridge extends FeedExpander parent::collectExpandableDatas($this->getURI()); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $html = getSimpleHTMLDOMCached($item['uri']); $content = $html->find('div#content div.content', 0); ## Fix author diff --git a/bridges/FeedExpanderTestBridge.php b/bridges/FeedExpanderTestBridge.php index 9a6e7bb7..65b3db89 100644 --- a/bridges/FeedExpanderTestBridge.php +++ b/bridges/FeedExpanderTestBridge.php @@ -14,9 +14,9 @@ class FeedExpanderTestBridge extends FeedExpander public function collectData() { $url = 'http://static.userland.com/gems/backend/sampleRss.xml'; // rss 0.91 - //$url = 'http://feeds.nature.com/nature/rss/current?format=xml'; // rss 1.0 - //$url = 'https://dvikan.no/feed.xml'; // rss 2.0 - //$url = 'https://nedlasting.geonorge.no/geonorge/Tjenestefeed.xml'; // atom + $url = 'http://feeds.nature.com/nature/rss/current?format=xml'; // rss 1.0 + $url = 'https://dvikan.no/feed.xml'; // rss 2.0 + $url = 'https://nedlasting.geonorge.no/geonorge/Tjenestefeed.xml'; // atom $this->collectExpandableDatas($url); } diff --git a/bridges/FilterBridge.php b/bridges/FilterBridge.php index 3e3e812d..1add47f4 100644 --- a/bridges/FilterBridge.php +++ b/bridges/FilterBridge.php @@ -82,10 +82,8 @@ class FilterBridge extends FeedExpander $this->collectExpandableDatas($this->getURI()); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - // Generate title from first 50 characters of content? if ($this->getInput('title_from_content') && array_key_exists('content', $item)) { $content = str_get_html($item['content']); diff --git a/bridges/FolhaDeSaoPauloBridge.php b/bridges/FolhaDeSaoPauloBridge.php index d8d93c4f..dba86c52 100644 --- a/bridges/FolhaDeSaoPauloBridge.php +++ b/bridges/FolhaDeSaoPauloBridge.php @@ -29,10 +29,8 @@ class FolhaDeSaoPauloBridge extends FeedExpander ] ]; - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - if ($this->getInput('deep_crawl')) { $articleHTMLContent = getSimpleHTMLDOMCached($item['uri']); if ($articleHTMLContent) { diff --git a/bridges/ForGifsBridge.php b/bridges/ForGifsBridge.php index e210124a..0a054930 100644 --- a/bridges/ForGifsBridge.php +++ b/bridges/ForGifsBridge.php @@ -12,10 +12,8 @@ class ForGifsBridge extends FeedExpander $this->collectExpandableDatas('https://forgifs.com/gallery/srss/7'); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $dom = str_get_html($item['content']); $img = $dom->find('img', 0); $poster = $img->src; diff --git a/bridges/FreeCodeCampBridge.php b/bridges/FreeCodeCampBridge.php index 141746d2..aaeaf7bd 100644 --- a/bridges/FreeCodeCampBridge.php +++ b/bridges/FreeCodeCampBridge.php @@ -14,10 +14,8 @@ class FreeCodeCampBridge extends FeedExpander $this->collectExpandableDatas('https://www.freecodecamp.org/news/rss/', 15); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $dom = getSimpleHTMLDOM($item['uri']); // figure contain's the main article image diff --git a/bridges/FuturaSciencesBridge.php b/bridges/FuturaSciencesBridge.php index cfb2d711..6420f319 100644 --- a/bridges/FuturaSciencesBridge.php +++ b/bridges/FuturaSciencesBridge.php @@ -85,10 +85,8 @@ class FuturaSciencesBridge extends FeedExpander $this->collectExpandableDatas($url, 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $item['uri'] = str_replace('#xtor%3DRSS-8', '', $item['uri']); $dom = getSimpleHTMLDOMCached($item['uri']); $item['content'] = $this->extractArticleContent($dom); diff --git a/bridges/GizmodoBridge.php b/bridges/GizmodoBridge.php index 8ed30704..52812d33 100644 --- a/bridges/GizmodoBridge.php +++ b/bridges/GizmodoBridge.php @@ -8,10 +8,8 @@ class GizmodoBridge extends FeedExpander const CACHE_TIMEOUT = 1800; // 30min const DESCRIPTION = 'Returns the newest posts from Gizmodo.'; - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $html = getSimpleHTMLDOMCached($item['uri']); $html = defaultLinkTo($html, $this->getURI()); diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index 96fa4506..6699e433 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -63,9 +63,8 @@ class GolemBridge extends FeedExpander ); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); $item['content'] ??= ''; $uri = $item['uri']; diff --git a/bridges/HardwareInfoBridge.php b/bridges/HardwareInfoBridge.php index 6a47df66..5970ecd0 100644 --- a/bridges/HardwareInfoBridge.php +++ b/bridges/HardwareInfoBridge.php @@ -12,10 +12,8 @@ class HardwareInfoBridge extends FeedExpander $this->collectExpandableDatas('https://nl.hardware.info/updates/all.rss', 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $itemUrl = $item['uri']; $articlePage = getSimpleHTMLDOMCached($itemUrl); diff --git a/bridges/HeiseBridge.php b/bridges/HeiseBridge.php index 434e7514..f89594ee 100644 --- a/bridges/HeiseBridge.php +++ b/bridges/HeiseBridge.php @@ -125,10 +125,8 @@ class HeiseBridge extends FeedExpander ); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $sessioncookie = $this->getInput('sessioncookie'); // strip rss parameter diff --git a/bridges/IGNBridge.php b/bridges/IGNBridge.php index c0260cbd..e063dbbb 100644 --- a/bridges/IGNBridge.php +++ b/bridges/IGNBridge.php @@ -15,10 +15,8 @@ class IGNBridge extends FeedExpander // IGNs feed is both hidden and incomplete. This bridge tries to fix this. - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOM($item['uri']); // List of BS elements diff --git a/bridges/KoreusBridge.php b/bridges/KoreusBridge.php index 874c2c92..5ef5e11f 100644 --- a/bridges/KoreusBridge.php +++ b/bridges/KoreusBridge.php @@ -7,10 +7,8 @@ class KoreusBridge extends FeedExpander const URI = 'https://www.koreus.com/'; const DESCRIPTION = 'Returns the newest posts from Koreus (full text)'; - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $html = getSimpleHTMLDOMCached($item['uri']); $text = $html->find('p.itemText', 0)->innertext; $item['content'] = utf8_encode($text); diff --git a/bridges/LeMondeInformatiqueBridge.php b/bridges/LeMondeInformatiqueBridge.php index c91a0437..860c0887 100644 --- a/bridges/LeMondeInformatiqueBridge.php +++ b/bridges/LeMondeInformatiqueBridge.php @@ -12,10 +12,8 @@ class LeMondeInformatiqueBridge extends FeedExpander $this->collectExpandableDatas(self::URI . 'rss/rss.xml', 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $article_html = getSimpleHTMLDOMCached($item['uri']); //Deduce thumbnail URL from article image URL diff --git a/bridges/ListverseBridge.php b/bridges/ListverseBridge.php index b7acbdd0..bffa4cda 100644 --- a/bridges/ListverseBridge.php +++ b/bridges/ListverseBridge.php @@ -13,9 +13,8 @@ class ListverseBridge extends FeedExpander $this->collectExpandableDatas('https://listverse.com/feed/', 15); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); $dom = getSimpleHTMLDOM($item['uri']); $article = $dom->find('#articlecontentonly', 0); $item['content'] = $article; diff --git a/bridges/MediapartBridge.php b/bridges/MediapartBridge.php index c4deda61..aa10e159 100644 --- a/bridges/MediapartBridge.php +++ b/bridges/MediapartBridge.php @@ -29,10 +29,8 @@ class MediapartBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $itemUrl = $item['uri']; // Mediapart provide multiple type of contents. diff --git a/bridges/MsnMondeBridge.php b/bridges/MsnMondeBridge.php index 9b308b99..a2592702 100644 --- a/bridges/MsnMondeBridge.php +++ b/bridges/MsnMondeBridge.php @@ -25,10 +25,8 @@ class MsnMondeBridge extends FeedExpander $this->collectExpandableDatas(self::FEED_URL, 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - if (!preg_match('#fr-fr/actualite.*/ar-(?[\w]*)\?#', $item['uri'], $matches)) { return null; } diff --git a/bridges/NYTBridge.php b/bridges/NYTBridge.php index 57c3e2af..a9942e2d 100644 --- a/bridges/NYTBridge.php +++ b/bridges/NYTBridge.php @@ -14,10 +14,8 @@ class NYTBridge extends FeedExpander $this->collectExpandableDatas($url, 40); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $article = ''; try { diff --git a/bridges/NextInpactBridge.php b/bridges/NextInpactBridge.php index 0260da14..6982c104 100644 --- a/bridges/NextInpactBridge.php +++ b/bridges/NextInpactBridge.php @@ -88,10 +88,8 @@ class NextInpactBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $item['content'] = $this->extractContent($item, $item['uri']); if (is_null($item['content'])) { return null; //Filtered article diff --git a/bridges/NextgovBridge.php b/bridges/NextgovBridge.php index bc92d306..7fe7130a 100644 --- a/bridges/NextgovBridge.php +++ b/bridges/NextgovBridge.php @@ -31,10 +31,8 @@ class NextgovBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $article_thumbnail = 'https://cdn.nextgov.com/nextgov/images/logo.png'; $item['content'] = '

' . $item['content'] . '

'; diff --git a/bridges/NiceMatinBridge.php b/bridges/NiceMatinBridge.php index bcebbbbb..dd90dbfe 100644 --- a/bridges/NiceMatinBridge.php +++ b/bridges/NiceMatinBridge.php @@ -12,10 +12,8 @@ class NiceMatinBridge extends FeedExpander $this->collectExpandableDatas(self::URI . 'derniere-minute/rss', 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $item['content'] = $this->extractContent($item['uri']); return $item; } diff --git a/bridges/OnVaSortirBridge.php b/bridges/OnVaSortirBridge.php index 9f9a750c..f8c395c1 100644 --- a/bridges/OnVaSortirBridge.php +++ b/bridges/OnVaSortirBridge.php @@ -123,9 +123,8 @@ class OnVaSortirBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); $dom = getSimpleHTMLDOMCached($item['uri']); $text = $dom->find('div.corpsMax', 0)->innertext; $item['content'] = utf8_encode($text); diff --git a/bridges/PhoronixBridge.php b/bridges/PhoronixBridge.php index fc0d78e5..227685e0 100644 --- a/bridges/PhoronixBridge.php +++ b/bridges/PhoronixBridge.php @@ -29,10 +29,8 @@ but some RSS readers don\'t support this. "img" tag are supported by most browse $this->collectExpandableDatas('https://www.phoronix.com/rss.php', $this->getInput('n')); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $itemUrl = $item['uri']; $articlePage = getSimpleHTMLDOM($itemUrl); diff --git a/bridges/QwantzBridge.php b/bridges/QwantzBridge.php index 2117c33c..b975bd43 100644 --- a/bridges/QwantzBridge.php +++ b/bridges/QwantzBridge.php @@ -11,14 +11,12 @@ class QwantzBridge extends FeedExpander $this->collectExpandableDatas(self::URI . 'rssfeed.php'); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $item['author'] = 'Ryan North'; preg_match('/title="(.*?)"/', $item['content'], $matches); - $title = $matches[1]; + $title = $matches[1] ?? ''; $content = str_get_html(html_entity_decode($item['content'])); $comicURL = $content->find('img')[0]->{'src'}; diff --git a/bridges/RaceDepartmentBridge.php b/bridges/RaceDepartmentBridge.php index 3783b53e..7390761f 100644 --- a/bridges/RaceDepartmentBridge.php +++ b/bridges/RaceDepartmentBridge.php @@ -12,10 +12,8 @@ class RaceDepartmentBridge extends FeedExpander $this->collectExpandableDatas('https://www.racedepartment.com/ams/index.rss', 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOMCached($item['uri']); $coverImage = $articlePage->find('img.js-articleCoverImage', 0); diff --git a/bridges/ScribbleHubBridge.php b/bridges/ScribbleHubBridge.php index 8f52d461..e7cdf337 100644 --- a/bridges/ScribbleHubBridge.php +++ b/bridges/ScribbleHubBridge.php @@ -42,10 +42,8 @@ class ScribbleHubBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - //For series, filter out other series from 'All' feed if ( $this->queriedContext === 'Series' diff --git a/bridges/SplCenterBridge.php b/bridges/SplCenterBridge.php index ca764846..af25ec48 100644 --- a/bridges/SplCenterBridge.php +++ b/bridges/SplCenterBridge.php @@ -27,10 +27,8 @@ class SplCenterBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $articleHtml = getSimpleHTMLDOMCached($item['uri']); foreach ($articleHtml->find('.file') as $index => $media) { diff --git a/bridges/TapasBridge.php b/bridges/TapasBridge.php index ddfbfb92..19995a23 100644 --- a/bridges/TapasBridge.php +++ b/bridges/TapasBridge.php @@ -43,10 +43,8 @@ class TapasBridge extends FeedExpander $this->collectExpandableDatas($this->getURI()); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - // $namespaces = $feedItem->getNamespaces(true); // if (isset($namespaces['content'])) { // $description = $feedItem->children($namespaces['content']); diff --git a/bridges/TheGuardianBridge.php b/bridges/TheGuardianBridge.php index 98e56506..ac8a9661 100644 --- a/bridges/TheGuardianBridge.php +++ b/bridges/TheGuardianBridge.php @@ -56,10 +56,8 @@ class TheGuardianBridge extends FeedExpander $this->collectExpandableDatas($url, 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOM($item['uri']); // figure contain's the main article image $article = $articlePage->find('figure', 0); diff --git a/bridges/TwitterEngineeringBridge.php b/bridges/TwitterEngineeringBridge.php index b98cfb87..96319c97 100644 --- a/bridges/TwitterEngineeringBridge.php +++ b/bridges/TwitterEngineeringBridge.php @@ -14,10 +14,8 @@ class TwitterEngineeringBridge extends FeedExpander $this->collectExpandableDatas($url); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $dom = getSimpleHTMLDOMCached($item['uri']); if (!$dom) { $item['content'] .= '

Could not request ' . $this->getName() . ': ' . $item['uri'] . '

'; diff --git a/bridges/VarietyBridge.php b/bridges/VarietyBridge.php index 6625dca2..a49ea353 100644 --- a/bridges/VarietyBridge.php +++ b/bridges/VarietyBridge.php @@ -13,10 +13,8 @@ class VarietyBridge extends FeedExpander $this->collectExpandableDatas('https://feeds.feedburner.com/variety/headlines', 15); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - // $articlePage gets the entire page's contents $articlePage = getSimpleHTMLDOM($item['uri']); // Remove Script tags diff --git a/bridges/ViceBridge.php b/bridges/ViceBridge.php index c7ecec33..dd81c559 100644 --- a/bridges/ViceBridge.php +++ b/bridges/ViceBridge.php @@ -32,10 +32,8 @@ class ViceBridge extends FeedExpander $this->collectExpandableDatas($feedURL, 10); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $articlePage = getSimpleHTMLDOM($item['uri']); // text and embedded content $article = $articlePage->find('.article__body', 0); diff --git a/bridges/WeLiveSecurityBridge.php b/bridges/WeLiveSecurityBridge.php index f54f6b29..151484c4 100644 --- a/bridges/WeLiveSecurityBridge.php +++ b/bridges/WeLiveSecurityBridge.php @@ -12,10 +12,8 @@ class WeLiveSecurityBridge extends FeedExpander ], ]; - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $html = getSimpleHTMLDOMCached($item['uri']); if (!$html) { $item['content'] .= '

Could not request ' . $this->getName() . ': ' . $item['uri'] . '

'; diff --git a/bridges/WiredBridge.php b/bridges/WiredBridge.php index 7f7f6051..f7da288c 100644 --- a/bridges/WiredBridge.php +++ b/bridges/WiredBridge.php @@ -50,10 +50,8 @@ class WiredBridge extends FeedExpander $this->collectExpandableDatas($feed_url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $originalContent = $item['content']; $article = getSimpleHTMLDOMCached($item['uri']); diff --git a/bridges/WordPressBridge.php b/bridges/WordPressBridge.php index fb27eb31..84a6b8ab 100644 --- a/bridges/WordPressBridge.php +++ b/bridges/WordPressBridge.php @@ -34,10 +34,8 @@ class WordPressBridge extends FeedExpander } } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $dom = getSimpleHTMLDOMCached($item['uri']); // Find article body diff --git a/bridges/WorldOfTanksBridge.php b/bridges/WorldOfTanksBridge.php index 52691025..7bf20015 100644 --- a/bridges/WorldOfTanksBridge.php +++ b/bridges/WorldOfTanksBridge.php @@ -30,9 +30,8 @@ class WorldOfTanksBridge extends FeedExpander $this->collectExpandableDatas(sprintf('https://worldoftanks.eu/%s/rss/news/', $this->getInput('lang'))); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); $item['content'] = $this->loadFullArticle($item['uri']); return $item; } diff --git a/bridges/ZDNetBridge.php b/bridges/ZDNetBridge.php index 00b272ce..e3b659a8 100644 --- a/bridges/ZDNetBridge.php +++ b/bridges/ZDNetBridge.php @@ -174,10 +174,8 @@ class ZDNetBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $article = getSimpleHTMLDOMCached($item['uri']); if (!$article) { $this->logger->info('Unable to parse the dom from ' . $item['uri']); diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php index 7d7e89aa..0ed9276b 100644 --- a/bridges/ZeitBridge.php +++ b/bridges/ZeitBridge.php @@ -59,10 +59,8 @@ class ZeitBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $item['enclosures'] = []; $headers = [ diff --git a/docs/05_Bridge_API/03_FeedExpander.md b/docs/05_Bridge_API/03_FeedExpander.md index 910d1abb..62356fc9 100644 --- a/docs/05_Bridge_API/03_FeedExpander.md +++ b/docs/05_Bridge_API/03_FeedExpander.md @@ -1,85 +1,35 @@ -`FeedExpander` extends [`BridgeAbstract`](./02_BridgeAbstract.md) and adds functions to collect data from existing feeds. - **Usage example**: _You have discovered a site that provides feeds which are hidden and inaccessible by normal means. You want your bridge to directly read the feeds and provide them via **RSS-Bridge**_ -To create a new Bridge extending `FeedExpander` you must implement all required functions of [`BridgeAbstract`](./02_BridgeAbstract.md). `FeedExpander` additionally provides following functions: - -* [`parseItem`](#the-parseitem-function) -* [`getName`](#the-getname-function) -* [`getURI`](#the-geturi-function) -* [`getDescription`](#the-getdescription-function) - Find a [template](#template) at the end of this file. **Notice:** For a standard feed only `collectData` need to be implemented. `collectData` should call `$this->collectExpandableDatas('your URI here');` to automatically load feed items and header data (will subsequently call `parseItem` for each item in the feed). You can limit the number of items to fetch by specifying an additional parameter for: `$this->collectExpandableDatas('your URI here', 10)` (limited to 10 items). -## The `parseItem` function +## The `parseItem` method -This function receives one item from the current feed and should return one **RSS-Bridge** item. +This method receives one item from the current feed and should return one **RSS-Bridge** item. The default function does all the work to get the item data from the feed, whether it is RSS 1.0, -RSS 2.0 or Atom 1.0. If you have to redefine this function in your **RSS-Bridge** for whatever reason, -you should first call the parent function to initialize the item, then apply the changes that you require. +RSS 2.0 or Atom 1.0. **Notice:** The following code sample is just an example. Implementation depends on your requirements! ```PHP -protected function parseItem($feedItem){ - $item = parent::parseItem($feedItem); - $item['content'] = str_replace('rssbridge','RSS-Bridge',$feedItem->content); - +protected function parseItem(array $item) +{ + $item['content'] = str_replace('rssbridge','RSS-Bridge',$item['content']); return $item; } ``` -### Helper functions +### Feed parsing -The `FeedExpander` already provides a set of functions to parse RSS or Atom items based on the specifications. Where possible make use of these functions: - -Function | Description ----------|------------ -`parseATOMItem` | Parses an Atom 1.0 feed item -`parseRSS_0_9_1_Item` | Parses an RSS 0.91 feed item -`parseRSS_1_0_Item` | Parses an RSS 1.0 feed item -`parseRSS_2_0_Item` | Parses an RSS 2.0 feed item - -In the following list you'll find the feed tags assigned to the the **RSS-Bridge** item keys: +How rss-bridge processes xml feeds: Function | uri | title | timestamp | author | content ---------|-----|-------|-----------|--------|-------- -`parseATOMItem` | id | title | updated | author | content -`parseRSS_0_9_1_Item` | link | title | | | description -`parseRSS_1_0_Item` | link | title | dc:date | dc:creator | description -`parseRSS_2_0_Item` | link, guid | title | pubDate, dc:date | author, dc:creator | description - -## The `getName` function - -Returns the name of the current feed. - -```PHP -return $this->name; -``` - -**Notice:** Only implement this function if you require different behavior! - -## The `getURI` function - -Return the uri for the current feed. - -```PHP -return $this->uri; -``` - -**Notice:** Only implement this function if you require different behavior! - -## The `getDescription` function - -Returns the description for the current bridge. - -```PHP -return $this->description; -``` - -**Notice:** Only implement this function if you require different behavior! +`atom` | id | title | updated | author | content +`rss 0.91` | link | title | | | description +`rss 1.0` | link | title | dc:date | dc:creator | description +`rss 2.0` | link, guid | title | pubDate, dc:date | author, dc:creator | description # Template @@ -87,19 +37,19 @@ This is the template for a new bridge: ```PHP collectExpandableDatas('your feed URI'); } } -// Imaginary empty line! ``` \ No newline at end of file diff --git a/docs/05_Bridge_API/index.md b/docs/05_Bridge_API/index.md index e49e47be..06445246 100644 --- a/docs/05_Bridge_API/index.md +++ b/docs/05_Bridge_API/index.md @@ -7,7 +7,7 @@ and extends one of the base classes of **RSS-Bridge**: Base class | Description -----------|------------ [`BridgeAbstract`](./02_BridgeAbstract.md) | This class is intended for standard _Bridges_ that need to filter HTML pages for content. -[`FeedExpander`](./03_FeedExpander.md) | This class is an extension of `HttpCachingBridgeAbstract`, designed to load existing feeds into **RSS-Bridge** +[`FeedExpander`](./03_FeedExpander.md) | Expand/modify existing feed urls [`XPathAbstract`](./04_XPathAbstract.md) | This class is meant as an alternative base class for bridge implementations. It offers preliminary functionality for generating feeds based on _XPath expressions_. For more information about how to create a new _Bridge_, read [How to create a new Bridge?](./01_How_to_create_a_new_bridge.md) \ No newline at end of file diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index 70c4560d..f9cff900 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -5,111 +5,57 @@ */ abstract class FeedExpander extends BridgeAbstract { - const FEED_TYPE_RSS_1_0 = 'RSS_1_0'; - const FEED_TYPE_RSS_2_0 = 'RSS_2_0'; - const FEED_TYPE_ATOM_1_0 = 'ATOM_1_0'; - - private string $feedType; - private FeedParser $feedParser; - private array $parsedFeed; - - public function __construct(CacheInterface $cache, Logger $logger) - { - parent::__construct($cache, $logger); - $this->feedParser = new FeedParser(); - } + private array $feed; public function collectExpandableDatas(string $url, $maxItems = -1) { if (!$url) { throw new \Exception('There is no $url for this RSS expander'); } + $maxItems = (int) $maxItems; if ($maxItems === -1) { $maxItems = 999; } $accept = [MrssFormat::MIME_TYPE, AtomFormat::MIME_TYPE, '*/*']; $httpHeaders = ['Accept: ' . implode(', ', $accept)]; - // Notice we do not use cache here on purpose. We want a fresh view of the RSS stream each time $xmlString = getContents($url, $httpHeaders); if ($xmlString === '') { throw new \Exception(sprintf('Unable to parse xml from `%s` because we got the empty string', $url), 10); } - // Maybe move this call earlier up the stack frames - // Disable triggering of the php error-handler and handle errors manually instead - libxml_use_internal_errors(true); - // Consider replacing libxml with https://www.php.net/domdocument - // Intentionally not using the silencing operator (@) because it has no effect here - $xml = simplexml_load_string(trim($xmlString)); - if ($xml === false) { - $xmlErrors = libxml_get_errors(); - foreach ($xmlErrors as $xmlError) { - Debug::log(trim($xmlError->message)); - } - if ($xmlErrors) { - // Render only the first error into exception message - $firstXmlErrorMessage = $xmlErrors[0]->message; - } - throw new \Exception(sprintf('Unable to parse xml from `%s` %s', $url, $firstXmlErrorMessage ?? ''), 11); - } - // Restore previous behaviour in case other code relies on it being off - libxml_use_internal_errors(false); - - // Currently only feed metadata (not items) are plucked out - $this->parsedFeed = $this->feedParser->parseFeed($xmlString); - - if (isset($xml->item[0])) { - $this->feedType = self::FEED_TYPE_RSS_1_0; - $items = $xml->item; - } elseif (isset($xml->channel[0])) { - $this->feedType = self::FEED_TYPE_RSS_2_0; - $items = $xml->channel[0]->item; - } elseif (isset($xml->entry[0])) { - $this->feedType = self::FEED_TYPE_ATOM_1_0; - $items = $xml->entry; - } else { - throw new \Exception(sprintf('Unable to detect feed format from `%s`', $url)); - } + $feedParser = new FeedParser(); + $this->feed = $feedParser->parseFeed($xmlString); + $items = array_slice($this->feed['items'], 0, $maxItems); foreach ($items as $item) { - $parsedItem = $this->parseItem($item); - if ($parsedItem) { - $this->items[] = $parsedItem; - } - if (count($this->items) >= $maxItems) { - break; + // Give bridges a chance to modify the item + $item = $this->parseItem($item); + if ($item) { + $this->items[] = $item; } } - return $this; } /** - * @param \SimpleXMLElement $item The feed item to be parsed + * This method is overidden by bridges + * + * @return array */ - protected function parseItem($item) + protected function parseItem(array $item) { - switch ($this->feedType) { - case self::FEED_TYPE_RSS_1_0: - return $this->feedParser->parseRss1Item($item); - case self::FEED_TYPE_RSS_2_0: - return $this->feedParser->parseRss2Item($item); - case self::FEED_TYPE_ATOM_1_0: - return $this->feedParser->parseAtomItem($item); - default: - throw new \Exception(sprintf('Unknown version %s!', $this->getInput('version'))); - } + return $item; } public function getURI() { - return $this->parsedFeed['uri'] ?? parent::getURI(); + return $this->feed['uri'] ?? parent::getURI(); } public function getName() { - return $this->parsedFeed['title'] ?? parent::getName(); + return $this->feed['title'] ?? parent::getName(); } public function getIcon() { - return $this->parsedFeed['icon'] ?? parent::getIcon(); + return $this->feed['icon'] ?? parent::getIcon(); } } diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 04452e7d..0a5b4679 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -14,10 +14,10 @@ final class FeedParser throw new \Exception('Unable to parse xml'); } $feed = [ - 'title' => null, - 'url' => null, - 'icon' => null, - 'items' => [], + 'title' => null, + 'uri' => null, + 'icon' => null, + 'items' => [], ]; if (isset($xml->item[0])) { // rss 1.0 From 49d9dafaecdb1e63ba1fe966f0e73c7f228fa5c3 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 02:31:09 +0200 Subject: [PATCH 169/716] refactor: more feed parsing tweaks (#3748) --- bridges/TapasBridge.php | 7 +++---- lib/FeedExpander.php | 5 +++++ lib/FeedParser.php | 15 ++++++++++++--- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/bridges/TapasBridge.php b/bridges/TapasBridge.php index 19995a23..ea6a7ff6 100644 --- a/bridges/TapasBridge.php +++ b/bridges/TapasBridge.php @@ -40,7 +40,7 @@ class TapasBridge extends FeedExpander $this->id = $html->find('meta[property$=":url"]', 0)->content; $this->id = str_ireplace(['tapastic://series/', '/info'], '', $this->id); } - $this->collectExpandableDatas($this->getURI()); + $this->collectExpandableDatas($this->getURI(), 10); } protected function parseItem(array $item) @@ -55,9 +55,8 @@ class TapasBridge extends FeedExpander if ($this->getInput('extend_content')) { $html = getSimpleHTMLDOM($item['uri']); - if (!$item['content']) { - $item['content'] = ''; - } + $item['content'] = $item['content'] ?? ''; + if ($html->find('article.main__body', 0)) { foreach ($html->find('article', 0)->find('img') as $line) { $item['content'] .= ''; diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index f9cff900..361df4d9 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -22,6 +22,11 @@ abstract class FeedExpander extends BridgeAbstract if ($xmlString === '') { throw new \Exception(sprintf('Unable to parse xml from `%s` because we got the empty string', $url), 10); } + // prepare/massage the xml to make it more acceptable + $badStrings = [ + '»', + ]; + $xmlString = str_replace($badStrings, '', $xmlString); $feedParser = new FeedParser(); $this->feed = $feedParser->parseFeed($xmlString); $items = array_slice($this->feed['items'], 0, $maxItems); diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 0a5b4679..7c8a5232 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -11,7 +11,10 @@ final class FeedParser $xmlErrors = libxml_get_errors(); libxml_use_internal_errors(false); if ($xml === false) { - throw new \Exception('Unable to parse xml'); + if ($xmlErrors) { + $firstXmlErrorMessage = $xmlErrors[0]->message; + } + throw new \Exception(sprintf('Unable to parse xml: %s', $firstXmlErrorMessage ?? '')); } $feed = [ 'title' => null, @@ -123,7 +126,6 @@ final class FeedParser { // Primary data is compatible to 0.91 with some additional data $item = $this->parseRss091Item($feedItem); - $namespaces = $feedItem->getNamespaces(true); if (isset($namespaces['dc'])) { $dc = $feedItem->children($namespaces['dc']); @@ -192,7 +194,14 @@ final class FeedParser public function parseRss091Item(\SimpleXMLElement $feedItem): array { - $item = []; + $item = [ + 'uri' => null, + 'title' => null, + 'content' => null, + 'timestamp' => null, + 'author' => null, + 'enclosures' => [], + ]; if (isset($feedItem->link)) { // todo: trim uri $item['uri'] = (string)$feedItem->link; From 920d00480dcfa669229ad6a420525e2e670a38bf Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 11:24:22 +0200 Subject: [PATCH 170/716] fix(senscritique) (#3750) --- bridges/SensCritiqueBridge.php | 38 +++++++--------------------------- bridges/VkBridge.php | 4 ++-- lib/FeedParser.php | 9 ++++++++ lib/FormatAbstract.php | 5 ++++- lib/url.php | 7 +++++-- 5 files changed, 28 insertions(+), 35 deletions(-) diff --git a/bridges/SensCritiqueBridge.php b/bridges/SensCritiqueBridge.php index 9e42d6a6..b823b55c 100644 --- a/bridges/SensCritiqueBridge.php +++ b/bridges/SensCritiqueBridge.php @@ -56,7 +56,8 @@ class SensCritiqueBridge extends BridgeAbstract break; } $html = getSimpleHTMLDOM($uri); - $list = $html->find('ul.elpr-list', 0); + // This selector name looks like it's automatically generated + $list = $html->find('div.Universes__WrapperProducts-sc-1qa2w66-0.eVdcAv', 0); $this->extractDataFromList($list); } @@ -68,36 +69,13 @@ class SensCritiqueBridge extends BridgeAbstract if ($list === null) { returnClientError('Cannot extract data from list'); } - - foreach ($list->find('li') as $movie) { + foreach ($list->find('div[data-testid="product-list-item"]') as $movie) { $item = []; - $item['author'] = htmlspecialchars_decode($movie->find('.elco-title a', 0)->plaintext, ENT_QUOTES) - . ' ' - . $movie->find('.elco-date', 0)->plaintext; - - $item['title'] = $movie->find('.elco-title a', 0)->plaintext - . ' ' - . $movie->find('.elco-date', 0)->plaintext; - - $item['content'] = ''; - $originalTitle = $movie->find('.elco-original-title', 0); - $description = $movie->find('.elco-description', 0); - - if ($originalTitle) { - $item['content'] = '' . $originalTitle->plaintext . '

'; - } - - $item['content'] .= $movie->find('.elco-baseline', 0)->plaintext - . '
' - . $movie->find('.elco-baseline', 1)->plaintext - . '

' - . ($description ? $description->plaintext : '') - . '

' - . trim($movie->find('.erra-ratings .erra-global', 0)->plaintext) - . ' / 10'; - - $item['id'] = $this->getURI() . ltrim($movie->find('.elco-title a', 0)->href, '/'); - $item['uri'] = $this->getURI() . ltrim($movie->find('.elco-title a', 0)->href, '/'); + $item['title'] = $movie->find('h2 a', 0)->plaintext; + // todo: fix image + $item['content'] = $movie->innertext; + $item['id'] = $this->getURI() . ltrim($movie->find('a', 0)->href, '/'); + $item['uri'] = $this->getURI() . ltrim($movie->find('a', 0)->href, '/'); $this->items[] = $item; } } diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index d86b9cf9..0d47692d 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -158,8 +158,8 @@ class VkBridge extends BridgeAbstract $article_author_selector = 'div.article_snippet__author'; $article_thumb_selector = 'div.article_snippet__image'; } - $article_title = $article->find($article_title_selector, 0)->innertext; - $article_author = $article->find($article_author_selector, 0)->innertext; + $article_title = $article->find($article_title_selector, 0)->innertext ?? ''; + $article_author = $article->find($article_author_selector, 0)->innertext ?? ''; $article_link = $article->getAttribute('href'); $article_img_element_style = $article->find($article_thumb_selector, 0)->getAttribute('style'); preg_match('/background-image: url\((.*)\)/', $article_img_element_style, $matches); diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 7c8a5232..64a3587d 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -2,6 +2,13 @@ declare(strict_types=1); +/** + * Very basic and naive feed parser that srapes out rss 0.91, 1.0, 2.0 and atom 1.0. + * + * Emit arrays meant to be used inside rss-bridge. + * + * The feed item structure is identical to that of FeedItem + */ final class FeedParser { public function parseFeed(string $xmlString): array @@ -200,6 +207,8 @@ final class FeedParser 'content' => null, 'timestamp' => null, 'author' => null, + 'uid' => null, + 'categories' => [], 'enclosures' => [], ]; if (isset($feedItem->link)) { diff --git a/lib/FormatAbstract.php b/lib/FormatAbstract.php index 0304f627..b05a5764 100644 --- a/lib/FormatAbstract.php +++ b/lib/FormatAbstract.php @@ -31,7 +31,10 @@ abstract class FormatAbstract $this->lastModified = $lastModified; } - public function setItems(array $items) + /** + * @param FeedItem[] $items + */ + public function setItems(array $items): void { $this->items = $items; } diff --git a/lib/url.php b/lib/url.php index 2dcbbba5..993fef96 100644 --- a/lib/url.php +++ b/lib/url.php @@ -7,7 +7,9 @@ final class UrlException extends \Exception } /** - * Intentionally restrictive url parser + * Intentionally restrictive url parser. + * + * Only absolute http/https urls. */ final class Url { @@ -29,7 +31,7 @@ final class Url $parts = parse_url($url); if ($parts === false) { - throw new UrlException(sprintf('Invalid url %s', $url)); + throw new UrlException(sprintf('Failed to parse_url(): %s', $url)); } return (new self()) @@ -38,6 +40,7 @@ final class Url ->withPort($parts['port'] ?? 80) ->withPath($parts['path'] ?? '/') ->withQueryString($parts['query'] ?? null); + // todo: add fragment } public static function validate(string $url): bool From fd52b9b9a487cd6693455ed6df36d6b805adb1f7 Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 13 Oct 2023 19:27:33 +0200 Subject: [PATCH 171/716] [CssSelectorFeedExpander] Fix ArgumentCountError (#3739) (#3751) * [CssSelectorFeedExpander] Fix ArgumentCountError (#3739) Fix ArgumentCountError (#3739) using new FeedParser class (#3740) Implement default value for feed name / url if missing * [CssSelectorFeedExpander] Skip empty fields in source feed Fix empty feed properties being passed down from source feed rssbridge.DEBUG lib/FeedItem.php(177): Author must be a string! rssbridge.DEBUG lib/FeedItem.php(267): Unique id must be a string! If "don't expand metadata" is checked, then source feed is passed down verbatim (only content is expanded) so the debug messages will persist, but the issue is in source feed, not in the bridge. --- bridges/CssSelectorFeedExpanderBridge.php | 40 ++++++++++++----------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/bridges/CssSelectorFeedExpanderBridge.php b/bridges/CssSelectorFeedExpanderBridge.php index 008921b1..9f332fb9 100644 --- a/bridges/CssSelectorFeedExpanderBridge.php +++ b/bridges/CssSelectorFeedExpanderBridge.php @@ -1,16 +1,5 @@ getInput('discard_thumbnail'); $limit = $this->getInput('limit'); - //$xmlString = getContents($url); - //$feed = (new FeedParser())->parseFeed($xmlString); - //$items = $feed['items']; + $source_feed = (new FeedParser())->parseFeed(getContents($url)); + $items = $source_feed['items']; - $feed_expander = new CssSelectorFeedExpanderBridgeInternal(); - $items = $feed_expander->collectExpandableDatas($url)->getItems(); + // Map Homepage URL (Default: Root page) + if (isset($source_feed['uri'])) { + $this->homepageUrl = $source_feed['uri']; + } else { + $this->homepageUrl = urljoin($url, '/'); + } - $this->homepageUrl = urljoin($url, '/'); - $this->feedName = $feed_expander->getName(); + // Map Feed Name (Default: Domain name) + if (isset($source_feed['title'])) { + $this->feedName = $source_feed['title']; + } else { + $this->feedName = explode('/', urljoin($url, '/'))[2]; + } + // Apply item limit (Default: Global limit) + if ($limit > 0) { + $items = array_slice($items, 0, $limit); + } + + // Expand feed items (CssSelectorBridge) foreach ($items as $item_from_feed) { $item_expanded = $this->expandEntryWithSelector( $item_from_feed['uri'], @@ -86,7 +88,7 @@ class CssSelectorFeedExpanderBridge extends CssSelectorBridge } else { // Take expanded item, but give priority to metadata already in source item foreach ($item_from_feed as $field => $val) { - if ($field !== 'content') { + if ($field !== 'content' && !empty($val)) { $item_expanded[$field] = $val; } } From 5f37c72be0a994257b01a8c43c4e488611f092f2 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 20:48:08 +0200 Subject: [PATCH 172/716] fix(binance): plus some other tweaks (#3753) --- bridges/BinanceBridge.php | 59 +++++++++------------------- bridges/CssSelectorComplexBridge.php | 4 +- bridges/FallGuysBridge.php | 2 +- bridges/InstagramBridge.php | 11 ++++-- bridges/MastodonBridge.php | 2 +- bridges/PokemonTVBridge.php | 9 ++++- lib/FeedExpander.php | 1 + lib/FeedItem.php | 10 +++-- lib/logger.php | 1 + 9 files changed, 46 insertions(+), 53 deletions(-) diff --git a/bridges/BinanceBridge.php b/bridges/BinanceBridge.php index 03449fb1..5aec0604 100644 --- a/bridges/BinanceBridge.php +++ b/bridges/BinanceBridge.php @@ -8,48 +8,27 @@ class BinanceBridge extends BridgeAbstract const MAINTAINER = 'thefranke'; const CACHE_TIMEOUT = 3600; // 1h + public function collectData() + { + $url = 'https://www.binance.com/bapi/composite/v1/public/content/blog/list?category=&tag=&page=1&size=12'; + $json = getContents($url); + $data = Json::decode($json, false); + foreach ($data->data->blogList as $post) { + $item = []; + $item['title'] = $post->title; + // Url slug not in json + //$item['uri'] = $uri; + $item['timestamp'] = $post->postTimeUTC / 1000; + $item['author'] = 'Binance'; + $item['content'] = $post->brief; + //$item['categories'] = $category; + $item['uid'] = $post->idStr; + $this->items[] = $item; + } + } + public function getIcon() { return 'https://bin.bnbstatic.com/static/images/common/favicon.ico'; } - - public function collectData() - { - $html = getSimpleHTMLDOM(self::URI) - or returnServerError('Could not fetch Binance blog data.'); - - $appData = $html->find('script[id="__APP_DATA"]'); - $appDataJson = json_decode($appData[0]->innertext); - $allposts = $appDataJson->routeProps->f3ac->blogListRes->list; - - foreach ($allposts as $element) { - $date = $element->releasedTime; - $title = $element->title; - $category = $element->category->name; - - $suburl = strtolower($category); - $suburl = str_replace(' ', '_', $suburl); - - $uri = self::URI . '/' . $suburl . '/' . $element->idStr; - - $contentHTML = getSimpleHTMLDOMCached($uri); - $contentAppData = $contentHTML->find('script[id="__APP_DATA"]'); - $contentAppDataJson = json_decode($contentAppData[0]->innertext); - $content = $contentAppDataJson->routeProps->a106->blogDetail->content; - - $item = []; - $item['title'] = $title; - $item['uri'] = $uri; - $item['timestamp'] = substr($date, 0, -3); - $item['author'] = 'Binance'; - $item['content'] = $content; - $item['categories'] = $category; - - $this->items[] = $item; - - if (count($this->items) >= 10) { - break; - } - } - } } diff --git a/bridges/CssSelectorComplexBridge.php b/bridges/CssSelectorComplexBridge.php index c09db74a..e661fe18 100644 --- a/bridges/CssSelectorComplexBridge.php +++ b/bridges/CssSelectorComplexBridge.php @@ -224,7 +224,7 @@ class CssSelectorComplexBridge extends BridgeAbstract { if (!empty($url_pattern)) { $url_pattern = '/' . str_replace('/', '\/', $url_pattern) . '/'; - $links = array_filter($links, function ($url) { + $links = array_filter($links, function ($url) use ($url_pattern) { return preg_match($url_pattern, $url) === 1; }); } @@ -359,7 +359,7 @@ class CssSelectorComplexBridge extends BridgeAbstract $article_content = $entry_html->find($content_selector, 0); if (is_null($article_content)) { - returnClientError('Could not article content at URL: ' . $entry_url); + returnClientError('Could not get article content at URL: ' . $entry_url); } $article_content = defaultLinkTo($article_content, $entry_url); diff --git a/bridges/FallGuysBridge.php b/bridges/FallGuysBridge.php index dbb34792..1e527484 100644 --- a/bridges/FallGuysBridge.php +++ b/bridges/FallGuysBridge.php @@ -85,7 +85,7 @@ class FallGuysBridge extends BridgeAbstract for ($i = 0; $i < count($mediaOptions); $i++) { if (property_exists($mediaOptions[$i], 'youtubeVideo')) { $videoUrl = 'https://youtu.be/' . $mediaOptions[$i]->youtubeVideo->contentId; - $image = $mainContentOptions[$i]->image->src; + $image = $mainContentOptions[$i]->image->src ?? ''; $content .= '

'; diff --git a/bridges/InstagramBridge.php b/bridges/InstagramBridge.php index 1714a691..633d6080 100644 --- a/bridges/InstagramBridge.php +++ b/bridges/InstagramBridge.php @@ -121,6 +121,9 @@ class InstagramBridge extends BridgeAbstract $directLink = !is_null($this->getInput('direct_links')) && $this->getInput('direct_links'); $data = $this->getInstagramJSON($this->getURI()); + if (!$data) { + return; + } if (!is_null($this->getInput('u'))) { $userMedia = $data->data->user->edge_owner_to_timeline_media->edges; @@ -286,9 +289,11 @@ class InstagramBridge extends BridgeAbstract $html = getContents($uri); $scriptRegex = '/window\._sharedData = (.*);<\/script>/'; - preg_match($scriptRegex, $html, $matches, PREG_OFFSET_CAPTURE, 0); - - return json_decode($matches[1][0]); + $ret = preg_match($scriptRegex, $html, $matches, PREG_OFFSET_CAPTURE); + if ($ret) { + return json_decode($matches[1][0]); + } + return null; } } diff --git a/bridges/MastodonBridge.php b/bridges/MastodonBridge.php index 01d425b4..e673bf14 100644 --- a/bridges/MastodonBridge.php +++ b/bridges/MastodonBridge.php @@ -193,7 +193,7 @@ class MastodonBridge extends BridgeAbstract && preg_match('/^http(s|):\/\//', $attachment['url'], $match) ) { $item['content'] = $item['content'] . '
', $attachment['url']); diff --git a/bridges/PokemonTVBridge.php b/bridges/PokemonTVBridge.php index 2c6b8bdc..a4c0a4be 100644 --- a/bridges/PokemonTVBridge.php +++ b/bridges/PokemonTVBridge.php @@ -64,15 +64,20 @@ class PokemonTVBridge extends BridgeAbstract continue; } } - switch ($element->{'media_type'}) { + switch ($element->media_type) { case 'movie': - $itemtitle = $element->{'channel_name'}; + case 'junior': + case 'original': + case 'non-animation': + $itemtitle = $element->channel_name; break; case 'episode': $season = str_pad($mediaelement->{'season'}, 2, '0', STR_PAD_LEFT); $episode = str_pad($mediaelement->{'episode'}, 2, '0', STR_PAD_LEFT); $itemtitle = $element->{'channel_name'} . ' - S' . $season . 'E' . $episode; break; + default: + $itemtitle = ''; } $streamurl = 'https://watch.pokemon.com/' . $this->getCountryCode() . '/#/player?id=' . $mediaelement->{'id'}; $item = []; diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index 361df4d9..056578e9 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -24,6 +24,7 @@ abstract class FeedExpander extends BridgeAbstract } // prepare/massage the xml to make it more acceptable $badStrings = [ + ' ', '»', ]; $xmlString = str_replace($badStrings, '', $xmlString); diff --git a/lib/FeedItem.php b/lib/FeedItem.php index 0bc22ee2..bd37f119 100644 --- a/lib/FeedItem.php +++ b/lib/FeedItem.php @@ -260,13 +260,15 @@ class FeedItem return $this->uid; } - public function setUid($uid) + public function setUid($uid): void { $this->uid = null; if (!is_string($uid)) { - Debug::log('Unique id must be a string!'); - } elseif (preg_match('/^[a-f0-9]{40}$/', $uid)) { - // keep id if it already is SHA-1 hash + Debug::log(sprintf('uid must be string: %s (%s)', (string) $uid, var_export($uid, true))); + return; + } + if (preg_match('/^[a-f0-9]{40}$/', $uid)) { + // Preserve sha1 hash $this->uid = $uid; } else { $this->uid = sha1($uid); diff --git a/lib/logger.php b/lib/logger.php index e41de34b..7a902b5b 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -149,6 +149,7 @@ final class StreamHandler ); error_log($text); if ($record['level'] < Logger::ERROR && Debug::isEnabled()) { + // Not a good idea to print here because http headers might not have been sent print sprintf("

%s
\n", e($text)); } //$bytes = file_put_contents('/tmp/rss-bridge.log', $text, FILE_APPEND | LOCK_EX); From daef240cd2fac0473113da456e11172d7d24c7a4 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 13 Oct 2023 23:14:08 +0200 Subject: [PATCH 173/716] test: add test for FeedParser (#3754) --- bridges/ArsTechnicaBridge.php | 2 +- bridges/UrlebirdBridge.php | 62 ++++++++++------ lib/FeedParser.php | 7 +- tests/FeedParserTest.php | 128 ++++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 25 deletions(-) create mode 100644 tests/FeedParserTest.php diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index d15cfb4f..5b3283b5 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -30,7 +30,7 @@ class ArsTechnicaBridge extends FeedExpander public function collectData() { $url = 'https://feeds.arstechnica.com/arstechnica/' . $this->getInput('section'); - $this->collectExpandableDatas($url); + $this->collectExpandableDatas($url, 10); } protected function parseItem(array $item) diff --git a/bridges/UrlebirdBridge.php b/bridges/UrlebirdBridge.php index 429e93f5..38f73249 100644 --- a/bridges/UrlebirdBridge.php +++ b/bridges/UrlebirdBridge.php @@ -6,7 +6,7 @@ class UrlebirdBridge extends BridgeAbstract const NAME = 'urlebird.com'; const URI = 'https://urlebird.com/'; const DESCRIPTION = 'Bridge for urlebird.com'; - const CACHE_TIMEOUT = 10; + const CACHE_TIMEOUT = 60 * 5; const PARAMETERS = [ [ 'query' => [ @@ -21,50 +21,70 @@ class UrlebirdBridge extends BridgeAbstract private $title; - private function fixURI($uri) - { - $path = parse_url($uri, PHP_URL_PATH); - $encoded_path = array_map('urlencode', explode('/', $path)); - return str_replace($path, implode('/', $encoded_path), $uri); - } - public function collectData() { switch ($this->getInput('query')[0]) { - default: - returnServerError('Please, enter valid username or hashtag!'); - break; case '@': $url = 'https://urlebird.com/user/' . substr($this->getInput('query'), 1) . '/'; break; case '#': $url = 'https://urlebird.com/hash/' . substr($this->getInput('query'), 1) . '/'; break; + default: + returnServerError('Please, enter valid username or hashtag!'); + break; } $html = getSimpleHTMLDOM($url); + $limit = 10; + $this->title = $html->find('title', 0)->innertext; $articles = $html->find('div.thumb'); + $articles = array_slice($articles, 0, $limit); foreach ($articles as $article) { $item = []; - $item['uri'] = $this->fixURI($article->find('a', 2)->href); - $article_content = getSimpleHTMLDOM($item['uri']); - $item['author'] = $article->find('img', 0)->alt . ' (' . - $article_content->find('a.user-video', 1)->innertext . ')'; - $item['title'] = $article_content->find('title', 0)->innertext; - $item['enclosures'][] = $article_content->find('video', 0)->poster; - $video = $article_content->find('video', 0); + $itemUrl = $article->find('a', 2)->href; + $item['uri'] = $this->encodePathSegments($itemUrl); + + $dom = getSimpleHTMLDOM($item['uri']); + $videoDiv = $dom->find('div.video', 0); + + // timestamp + $timestampH6 = $videoDiv->find('h6', 0); + $datetimeString = str_replace('Posted ', '', $timestampH6->plaintext); + $item['timestamp'] = $datetimeString; + + $innertext = $dom->find('a.user-video', 1)->innertext; + $alt = $article->find('img', 0)->alt; + $item['author'] = $alt . ' (' . $innertext . ')'; + + $item['title'] = $dom->find('title', 0)->innertext; + $item['enclosures'][] = $dom->find('video', 0)->poster; + + $video = $dom->find('video', 0); $video->autoplay = null; + $item['content'] = $video->outertext . '
' . - $article_content->find('div.music', 0) . '
' . - $article_content->find('div.info2', 0)->innertext . - '

find('video', 0)->src . '">Direct video link

Post link

'; + $this->items[] = $item; } } + private function encodePathSegments($url) + { + $path = parse_url($url, PHP_URL_PATH); + $pathSegments = explode('/', $path); + $encodedPathSegments = array_map('urlencode', $pathSegments); + $encodedPath = implode('/', $encodedPathSegments); + $result = str_replace($path, $encodedPath, $url); + return $result; + } + public function getName() { return $this->title ?: parent::getName(); diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 64a3587d..1393f5f5 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -142,6 +142,7 @@ final class FeedParser } if (isset($feedItem->guid)) { + // Pluck out a url from guid foreach ($feedItem->guid->attributes() as $attribute => $value) { if ( $attribute === 'isPermaLink' @@ -207,9 +208,9 @@ final class FeedParser 'content' => null, 'timestamp' => null, 'author' => null, - 'uid' => null, - 'categories' => [], - 'enclosures' => [], + //'uid' => null, + //'categories' => [], + //'enclosures' => [], ]; if (isset($feedItem->link)) { // todo: trim uri diff --git a/tests/FeedParserTest.php b/tests/FeedParserTest.php new file mode 100644 index 00000000..acd93e52 --- /dev/null +++ b/tests/FeedParserTest.php @@ -0,0 +1,128 @@ + + + + hello feed + http://meerkat.oreillynet.com + Meerkat: An Open Wire Service + + + + + + + + + + XML: A Disruptive Technology + http://c.moreover.com/click/here.pl?r123 + desc + + + XML; + + $sut = new \FeedParser(); + $feed = $sut->parseFeed($xml); + + $this->assertSame('hello feed', $feed['title']); + $this->assertSame('http://meerkat.oreillynet.com', $feed['uri']); + $this->assertSame(null, $feed['icon']); + + $item = $feed['items'][0]; + $this->assertSame('XML: A Disruptive Technology', $item['title']); + $this->assertSame('http://c.moreover.com/click/here.pl?r123', $item['uri']); + $this->assertSame('desc', $item['content']); + } + + public function testRss2() + { + $xml = << + + + hello feed + https://example.com/ + + https://example.com/2.ico + + + + hello world + https://example.com/1 + desc2 + Tue, 26 Apr 2022 00:00:00 +0200 + root + + + + + XML; + + $sut = new \FeedParser(); + $feed = $sut->parseFeed($xml); + + $this->assertSame('hello feed', $feed['title']); + $this->assertSame('https://example.com/', $feed['uri']); + $this->assertSame('https://example.com/2.ico', $feed['icon']); + + $item = $feed['items'][0]; + $this->assertSame('hello world', $item['title']); + $this->assertSame('https://example.com/1', $item['uri']); + $this->assertSame(1650924000, $item['timestamp']); + $this->assertSame('root', $item['author']); + $this->assertSame('desc2', $item['content']); + $this->assertSame(['https://example.com/1.png'], $item['enclosures']); + } + + public function testAtom() + { + $xml = << + + hello feed + + https://example.com/2.ico + + + hello world + + + root + + html + 2015-11-05T14:38:49+01:00 + + + XML; + + $sut = new \FeedParser(); + $feed = $sut->parseFeed($xml); + + $this->assertSame('hello feed', $feed['title']); + $this->assertSame('https://example.com/1', $feed['uri']); + $this->assertSame('https://example.com/2.ico', $feed['icon']); + + $item = $feed['items'][0]; + $this->assertSame('hello world', $item['title']); + $this->assertSame('https://example.com/1', $item['uri']); + $this->assertSame(1446730729, $item['timestamp']); + $this->assertSame('root', $item['author']); + $this->assertSame('html', $item['content']); + } +} From cf9558648efc86dce7a10136c53bd93688198755 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 15 Oct 2023 00:08:18 +0200 Subject: [PATCH 174/716] refactor: YoutubeBridge (#3755) --- bridges/YoutubeBridge.php | 378 ++++++++++++++++++-------------------- caches/FileCache.php | 6 +- 2 files changed, 180 insertions(+), 204 deletions(-) diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 418e715e..40a8f6a9 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -11,7 +11,7 @@ class YoutubeBridge extends BridgeAbstract { const NAME = 'YouTube Bridge'; const URI = 'https://www.youtube.com'; - const CACHE_TIMEOUT = 10800; // 3h + const CACHE_TIMEOUT = 60 * 60 * 3; const DESCRIPTION = 'Returns the 10 newest videos by username/channel/playlist or search'; const PARAMETERS = [ @@ -78,116 +78,6 @@ class YoutubeBridge extends BridgeAbstract // This took from repo BetterVideoRss of VerifiedJoseph. const URI_REGEX = '/(https?:\/\/(?:www\.)?(?:[a-zA-Z0-9-.]{2,256}\.[a-z]{2,20})(\:[0-9]{2 ,4})?(?:\/[a-zA-Z0-9@:%_\+.,~#"\'!?&\/\/=\-*]+|\/)?)/ims'; //phpcs:ignore - private function collectDataInternal() - { - $xml = ''; - $html = ''; - $url_feed = ''; - $url_listing = ''; - - if ($this->getInput('u')) { - /* User and Channel modes */ - $request = $this->getInput('u'); - $url_feed = self::URI . '/feeds/videos.xml?user=' . urlencode($request); - $url_listing = self::URI . '/user/' . urlencode($request) . '/videos'; - } elseif ($this->getInput('c')) { - $request = $this->getInput('c'); - $url_feed = self::URI . '/feeds/videos.xml?channel_id=' . urlencode($request); - $url_listing = self::URI . '/channel/' . urlencode($request) . '/videos'; - } elseif ($this->getInput('custom')) { - $request = $this->getInput('custom'); - $url_listing = self::URI . '/' . urlencode($request) . '/videos'; - } - - if (!empty($url_feed) || !empty($url_listing)) { - $this->feeduri = $url_listing; - if (!empty($this->getInput('custom'))) { - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl; - $this->feedIconUrl = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; - } - if (!$this->skipFeeds()) { - $html = $this->ytGetSimpleHTMLDOM($url_feed); - $this->ytBridgeParseXmlFeed($html); - } else { - if (empty($this->getInput('custom'))) { - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - } - $channel_id = ''; - if (isset($jsonData->contents)) { - $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId; - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1]; - $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents; - // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; - $this->parseJSONListing($jsonData); - } else { - returnServerError('Unable to get data from YouTube. Username/Channel: ' . $request); - } - } - $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); - } elseif ($this->getInput('p')) { - /* playlist mode */ - // TODO: this mode makes a lot of excess video query requests. - // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration" - // This cache will be used to find out, which videos to fetch - // to make feed of 15 items or more, if there a lot of videos published on that date. - $request = $this->getInput('p'); - $url_feed = self::URI . '/feeds/videos.xml?playlist_id=' . urlencode($request); - $url_listing = self::URI . '/playlist?list=' . urlencode($request); - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - // TODO: this method returns only first 100 video items - // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0]; - $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; - $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; - $item_count = count($jsonData); - - if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) { - $this->ytBridgeParseXmlFeed($xml); - } else { - $this->parseJSONListing($jsonData); - } - $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); - usort($this->items, function ($item1, $item2) { - if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) { - $item1['timestamp'] = strtotime($item1['timestamp']); - $item2['timestamp'] = strtotime($item2['timestamp']); - } - return $item2['timestamp'] - $item1['timestamp']; - }); - } elseif ($this->getInput('s')) { - /* search mode */ - $request = $this->getInput('s'); - $url_listing = self::URI - . '/results?search_query=' - . urlencode($request) - . '&sp=CAI%253D'; - - $html = $this->ytGetSimpleHTMLDOM($url_listing); - - $jsonData = $this->getJSONData($html); - $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; - $jsonData = $jsonData->sectionListRenderer->contents; - foreach ($jsonData as $data) { - // Search result includes some ads, have to filter them - if (isset($data->itemSectionRenderer->contents[0]->videoRenderer)) { - $jsonData = $data->itemSectionRenderer->contents; - break; - } - } - $this->parseJSONListing($jsonData); - $this->feeduri = $url_listing; - $this->feedName = 'Search: ' . $request; - } else { - /* no valid mode */ - returnClientError("You must either specify either:\n - YouTube - username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); - } - } - public function collectData() { $cacheKey = 'youtube_rate_limit'; @@ -204,9 +94,133 @@ class YoutubeBridge extends BridgeAbstract } } - private function ytBridgeQueryVideoInfo($vid, &$author, &$desc, &$time) + private function collectDataInternal() { - $html = $this->ytGetSimpleHTMLDOM(self::URI . "/watch?v=$vid", true); + $xml = ''; + $html = ''; + $url_feed = ''; + $url_listing = ''; + + $username = $this->getInput('u'); + $channel = $this->getInput('c'); + $custom = $this->getInput('custom'); + + if ($username) { + // user and channel + $request = $username; + $url_feed = self::URI . '/feeds/videos.xml?user=' . urlencode($request); + $url_listing = self::URI . '/user/' . urlencode($request) . '/videos'; + } elseif ($channel) { + $request = $channel; + $url_feed = self::URI . '/feeds/videos.xml?channel_id=' . urlencode($request); + $url_listing = self::URI . '/channel/' . urlencode($request) . '/videos'; + } elseif ($custom) { + $request = $custom; + $url_listing = self::URI . '/' . urlencode($request) . '/videos'; + } + + $playlist = $this->getInput('p'); + $search = $this->getInput('s'); + + $durationMin = $this->getInput('duration_min'); + $durationMax = $this->getInput('duration_max'); + + // Whether to discriminate videos by duration + $filterByDuration = $durationMin || $durationMax; + + if ($url_feed || $url_listing) { + // user, channel or custom + $this->feeduri = $url_listing; + if ($custom) { + // Extract the feed url for the custom name + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + // Pluck out the rss feed url + $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl; + $this->feedIconUrl = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; + } + if ($filterByDuration) { + if (!$custom) { + // Fetch the html page + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + } + $channel_id = ''; + if (isset($jsonData->contents)) { + $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId; + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1]; + $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents; + // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; + $this->fetchItemsFromFromJsonData($jsonData); + } else { + returnServerError('Unable to get data from YouTube. Username/Channel: ' . $request); + } + } else { + // Fetch the xml feed + $html = $this->fetch($url_feed); + $this->extractItemsFromXmlFeed($html); + } + $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); + } elseif ($playlist) { + // playlist + // TODO: this mode makes a lot of excess video query requests. + // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration" + // This cache will be used to find out, which videos to fetch + // to make feed of 15 items or more, if there a lot of videos published on that date. + $request = $playlist; + $url_feed = self::URI . '/feeds/videos.xml?playlist_id=' . urlencode($request); + $url_listing = self::URI . '/playlist?list=' . urlencode($request); + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + // TODO: this method returns only first 100 video items + // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0]; + $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; + $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; + $item_count = count($jsonData); + + if ($item_count > 15 || $filterByDuration) { + $this->fetchItemsFromFromJsonData($jsonData); + } else { + $xml = $this->fetch($url_feed); + $this->extractItemsFromXmlFeed($xml); + } + $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); + usort($this->items, function ($item1, $item2) { + if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) { + $item1['timestamp'] = strtotime($item1['timestamp']); + $item2['timestamp'] = strtotime($item2['timestamp']); + } + return $item2['timestamp'] - $item1['timestamp']; + }); + } elseif ($search) { + // search + $request = $search; + $url_listing = self::URI . '/results?search_query=' . urlencode($request) . '&sp=CAI%253D'; + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; + $jsonData = $jsonData->sectionListRenderer->contents; + foreach ($jsonData as $data) { + // Search result includes some ads, have to filter them + if (isset($data->itemSectionRenderer->contents[0]->videoRenderer)) { + $jsonData = $data->itemSectionRenderer->contents; + break; + } + } + $this->fetchItemsFromFromJsonData($jsonData); + $this->feeduri = $url_listing; + $this->feedName = 'Search: ' . $request; + } else { + returnClientError("You must either specify either:\n - YouTube + username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); + } + } + + private function fetchVideoDetails($vid, &$author, &$desc, &$time) + { + $url = self::URI . "/watch?v=$vid"; + $html = $this->fetch($url, true); // Skip unavailable videos if (strpos($html->innertext, 'IS_UNAVAILABLE_PAGE') !== false) { @@ -223,7 +237,7 @@ class YoutubeBridge extends BridgeAbstract $time = strtotime($elDatePublished->getAttribute('content')); } - $jsonData = $this->getJSONData($html); + $jsonData = $this->extractJsonFromHtml($html); if (!isset($jsonData->contents)) { return; } @@ -370,7 +384,7 @@ class YoutubeBridge extends BridgeAbstract if ($commandUrl['path'] === '/redirect') { parse_str($commandUrl['query'], $commandUrlQuery); $enhancement['url'] = urldecode($commandUrlQuery['q']); - } else if (isset($commandUrl['host'])) { + } elseif (isset($commandUrl['host'])) { $enhancement['url'] = $commandMetadata->url; } else { $enhancement['url'] = $baseUrl . $commandMetadata->url; @@ -388,94 +402,37 @@ class YoutubeBridge extends BridgeAbstract return array_reverse($enhancements); } - private function ytBridgeAddItem($vid, $title, $author, $desc, $time, $thumbnail = '') + private function extractItemsFromXmlFeed($xml) { - $item = []; - $item['id'] = $vid; - $item['title'] = $title; - $item['author'] = $author; - $item['timestamp'] = $time; - $item['uri'] = self::URI . '/watch?v=' . $vid; - if (!$thumbnail) { - // Fallback to default thumbnail if there aren't any provided. - $thumbnail = '0'; - } - $thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $vid . '/' . $thumbnail . '.jpg'; - $item['content'] = '
' . $desc; - $this->items[] = $item; - } + $this->feedName = $this->decodeTitle($xml->find('feed > title', 0)->plaintext); - private function ytBridgeParseXmlFeed($xml) - { foreach ($xml->find('entry') as $element) { - $title = $this->ytBridgeFixTitle($element->find('title', 0)->plaintext); + $videoId = str_replace('yt:video:', '', $element->find('id', 0)->plaintext); + if (strpos($videoId, 'googleads') !== false) { + continue; + } + $title = $this->decodeTitle($element->find('title', 0)->plaintext); $author = $element->find('name', 0)->plaintext; $desc = $element->find('media:description', 0)->innertext; - - // Make sure the description is easy on the eye :) $desc = htmlspecialchars($desc); $desc = nl2br($desc); - $desc = preg_replace( - self::URI_REGEX, - '$1 ', - $desc - ); - - $vid = str_replace('yt:video:', '', $element->find('id', 0)->plaintext); + $desc = preg_replace(self::URI_REGEX, '$1 ', $desc); $time = strtotime($element->find('published', 0)->plaintext); - if (strpos($vid, 'googleads') === false) { - $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); - } + $this->addItem($videoId, $title, $author, $desc, $time); } - $this->feedName = $this->ytBridgeFixTitle($xml->find('feed > title', 0)->plaintext); // feedName will be used by getName() } - private function ytBridgeFixTitle($title) + private function fetch($url, bool $cache = false) { - // convert both Ӓ and " to UTF-8 - return html_entity_decode($title, ENT_QUOTES, 'UTF-8'); - } - - private function ytGetSimpleHTMLDOM($url, $cached = false) - { - $header = [ - 'Accept-Language: en-US' - ]; - $opts = []; - $lowercase = true; - $forceTagsClosed = true; - $target_charset = DEFAULT_TARGET_CHARSET; - $stripRN = false; - $defaultBRText = DEFAULT_BR_TEXT; - $defaultSpanText = DEFAULT_SPAN_TEXT; - if ($cached) { - return getSimpleHTMLDOMCached( - $url, - 86400, - $header, - $opts, - $lowercase, - $forceTagsClosed, - $target_charset, - $stripRN, - $defaultBRText, - $defaultSpanText - ); + $header = ['Accept-Language: en-US']; + if ($cache) { + $ttl = 86400; + return getSimpleHTMLDOMCached($url, $ttl, $header); } - return getSimpleHTMLDOM( - $url, - $header, - $opts, - $lowercase, - $forceTagsClosed, - $target_charset, - $stripRN, - $defaultBRText, - $defaultSpanText - ); + return getSimpleHTMLDOM($url, $header); } - private function getJSONData($html) + private function extractJsonFromHtml($html) { $scriptRegex = '/var ytInitialData = (.*?);<\/script>/'; $result = preg_match($scriptRegex, $html, $matches); @@ -483,10 +440,11 @@ class YoutubeBridge extends BridgeAbstract $this->logger->debug('Could not find ytInitialData'); return null; } - return json_decode($matches[1]); + $data = json_decode($matches[1]); + return $data; } - private function parseJSONListing($jsonData) + private function fetchItemsFromFromJsonData($jsonData) { $duration_min = $this->getInput('duration_min') ?: -1; $duration_min = $duration_min * 60; @@ -497,9 +455,6 @@ class YoutubeBridge extends BridgeAbstract if ($duration_max < $duration_min) { returnClientError('Max duration must be greater than min duration!'); } - - // $vid_list = ''; - foreach ($jsonData as $item) { $wrapper = null; if (isset($item->gridVideoRenderer)) { @@ -513,10 +468,8 @@ class YoutubeBridge extends BridgeAbstract } else { continue; } - - $vid = $wrapper->videoId; + $videoId = $wrapper->videoId; $title = $wrapper->title->runs[0]->text; - $author = ''; $desc = ''; $time = ''; @@ -535,7 +488,6 @@ class YoutubeBridge extends BridgeAbstract } } } - if (is_string($durationText)) { if (preg_match('/([\d]{1,2})\:([\d]{1,2})\:([\d]{2})/', $durationText)) { $durationText = preg_replace('/([\d]{1,2})\:([\d]{1,2})\:([\d]{2})/', '$1:$2:$3', $durationText); @@ -549,15 +501,37 @@ class YoutubeBridge extends BridgeAbstract } } - // $vid_list .= $vid . ','; - $this->ytBridgeQueryVideoInfo($vid, $author, $desc, $time); - $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); + //$durationSeconds = (int) $wrapper->lengthSeconds; + if ($duration < $duration_min || $duration > $duration_max) { + continue; + } + $this->fetchVideoDetails($videoId, $author, $desc, $time); + $this->addItem($videoId, $title, $author, $desc, $time); } } - private function skipFeeds() + private function addItem($videoId, $title, $author, $desc, $time, $thumbnail = '') { - return ($this->getInput('duration_min') || $this->getInput('duration_max')); + $item = []; + // This should probably be uid? + $item['id'] = $videoId; + $item['title'] = $title; + $item['author'] = $author; + $item['timestamp'] = $time; + $item['uri'] = self::URI . '/watch?v=' . $videoId; + if (!$thumbnail) { + // Fallback to default thumbnail if there aren't any provided. + $thumbnail = '0'; + } + $thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $videoId . '/' . $thumbnail . '.jpg'; + $item['content'] = sprintf('
%s', $item['uri'], $thumbnailUri, $desc); + $this->items[] = $item; + } + + private function decodeTitle($title) + { + // convert both Ӓ and " to UTF-8 + return html_entity_decode($title, ENT_QUOTES, 'UTF-8'); } public function getURI() diff --git a/caches/FileCache.php b/caches/FileCache.php index 1ae88704..2f4b3ad5 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -30,7 +30,8 @@ class FileCache implements CacheInterface if (!file_exists($cacheFile)) { return $default; } - $item = unserialize(file_get_contents($cacheFile)); + $data = file_get_contents($cacheFile); + $item = unserialize($data); if ($item === false) { $this->logger->warning(sprintf('Failed to unserialize: %s', $cacheFile)); $this->delete($key); @@ -87,7 +88,8 @@ class FileCache implements CacheInterface if (isset($excluded[$filename]) || !is_file($cacheFile)) { continue; } - $item = unserialize(file_get_contents($cacheFile)); + $data = file_get_contents($cacheFile); + $item = unserialize($data); if ($item === false) { unlink($cacheFile); continue; From 2aa52aa99ae8866aa7cd320a88aa1b9a1ca49c5b Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 15 Oct 2023 01:13:17 +0200 Subject: [PATCH 175/716] fix(youtube): bug in prior refactor (#3756) --- bridges/YoutubeBridge.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 40a8f6a9..5bcdc00f 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -425,11 +425,12 @@ class YoutubeBridge extends BridgeAbstract private function fetch($url, bool $cache = false) { $header = ['Accept-Language: en-US']; + $ttl = 86400; + $stripNewlines = false; if ($cache) { - $ttl = 86400; - return getSimpleHTMLDOMCached($url, $ttl, $header); + return getSimpleHTMLDOMCached($url, $ttl, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines); } - return getSimpleHTMLDOM($url, $header); + return getSimpleHTMLDOM($url, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines); } private function extractJsonFromHtml($html) From 611fabe46c6118339e8e6f5a119b2dc5850266a7 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 15 Oct 2023 03:15:47 +0200 Subject: [PATCH 176/716] fix(youtube): reduce excessive network calls (#3757) --- bridges/YoutubeBridge.php | 110 ++++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 5bcdc00f..9b5dd44e 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -104,6 +104,14 @@ class YoutubeBridge extends BridgeAbstract $username = $this->getInput('u'); $channel = $this->getInput('c'); $custom = $this->getInput('custom'); + $playlist = $this->getInput('p'); + $search = $this->getInput('s'); + + $durationMin = $this->getInput('duration_min'); + $durationMax = $this->getInput('duration_max'); + + // Whether to discriminate videos by duration + $filterByDuration = $durationMin || $durationMax; if ($username) { // user and channel @@ -119,15 +127,6 @@ class YoutubeBridge extends BridgeAbstract $url_listing = self::URI . '/' . urlencode($request) . '/videos'; } - $playlist = $this->getInput('p'); - $search = $this->getInput('s'); - - $durationMin = $this->getInput('duration_min'); - $durationMax = $this->getInput('duration_max'); - - // Whether to discriminate videos by duration - $filterByDuration = $durationMin || $durationMax; - if ($url_feed || $url_listing) { // user, channel or custom $this->feeduri = $url_listing; @@ -217,9 +216,9 @@ class YoutubeBridge extends BridgeAbstract } } - private function fetchVideoDetails($vid, &$author, &$desc, &$time) + private function fetchVideoDetails($videoId, &$author, &$description, &$timestamp) { - $url = self::URI . "/watch?v=$vid"; + $url = self::URI . "/watch?v=$videoId"; $html = $this->fetch($url, true); // Skip unavailable videos @@ -234,7 +233,7 @@ class YoutubeBridge extends BridgeAbstract $elDatePublished = $html->find('meta[itemprop=datePublished]', 0); if (!is_null($elDatePublished)) { - $time = strtotime($elDatePublished->getAttribute('content')); + $timestamp = strtotime($elDatePublished->getAttribute('content')); } $jsonData = $this->extractJsonFromHtml($html); @@ -254,30 +253,28 @@ class YoutubeBridge extends BridgeAbstract } } if (!$videoSecondaryInfo) { - returnServerError('Could not find videoSecondaryInfoRenderer. Error at: ' . $vid); + returnServerError('Could not find videoSecondaryInfoRenderer. Error at: ' . $videoId); } - $desc = $videoSecondaryInfo->attributedDescription->content ?? ''; + $description = $videoSecondaryInfo->attributedDescription->content ?? ''; // Default whitespace chars used by trim + non-breaking spaces (https://en.wikipedia.org/wiki/Non-breaking_space) $whitespaceChars = " \t\n\r\0\x0B\u{A0}\u{2060}\u{202F}\u{2007}"; - $descEnhancements = $this->ytBridgeGetVideoDescriptionEnhancements($videoSecondaryInfo, $desc, self::URI, $whitespaceChars); + $descEnhancements = $this->ytBridgeGetVideoDescriptionEnhancements($videoSecondaryInfo, $description, self::URI, $whitespaceChars); foreach ($descEnhancements as $descEnhancement) { if (isset($descEnhancement['url'])) { - $descBefore = mb_substr($desc, 0, $descEnhancement['pos']); - $descValue = mb_substr($desc, $descEnhancement['pos'], $descEnhancement['len']); - $descAfter = mb_substr($desc, $descEnhancement['pos'] + $descEnhancement['len'], null); + $descBefore = mb_substr($description, 0, $descEnhancement['pos']); + $descValue = mb_substr($description, $descEnhancement['pos'], $descEnhancement['len']); + $descAfter = mb_substr($description, $descEnhancement['pos'] + $descEnhancement['len'], null); // Extended trim for the display value of internal links, e.g.: // FAVICON • Video Name // FAVICON / @ChannelName $descValue = trim($descValue, $whitespaceChars . '•/'); - $desc = sprintf('%s%s%s', $descBefore, $descEnhancement['url'], $descValue, $descAfter); + $description = sprintf('%s%s%s', $descBefore, $descEnhancement['url'], $descValue, $descAfter); } } - - $desc = nl2br($desc); } private function ytBridgeGetVideoDescriptionEnhancements( @@ -425,7 +422,7 @@ class YoutubeBridge extends BridgeAbstract private function fetch($url, bool $cache = false) { $header = ['Accept-Language: en-US']; - $ttl = 86400; + $ttl = 86400 * 3; // 3d $stripNewlines = false; if ($cache) { return getSimpleHTMLDOMCached($url, $ttl, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines); @@ -447,15 +444,9 @@ class YoutubeBridge extends BridgeAbstract private function fetchItemsFromFromJsonData($jsonData) { - $duration_min = $this->getInput('duration_min') ?: -1; - $duration_min = $duration_min * 60; + $minimumDurationSeconds = ($this->getInput('duration_min') ?: -1) * 60; + $maximumDurationSeconds = ($this->getInput('duration_max') ?: INF) * 60; - $duration_max = $this->getInput('duration_max') ?: INF; - $duration_max = $duration_max * 60; - - if ($duration_max < $duration_min) { - returnClientError('Max duration must be greater than min duration!'); - } foreach ($jsonData as $item) { $wrapper = null; if (isset($item->gridVideoRenderer)) { @@ -469,18 +460,33 @@ class YoutubeBridge extends BridgeAbstract } else { continue; } - $videoId = $wrapper->videoId; - $title = $wrapper->title->runs[0]->text; - $author = ''; - $desc = ''; - $time = ''; - // The duration comes in one of the formats: - // hh:mm:ss / mm:ss / m:ss - // 01:03:30 / 15:06 / 1:24 + // 01:03:30 | 15:06 | 1:24 + $lengthText = $wrapper->lengthText->simpleText ?? null; + // 6,875 views + $viewCount = $wrapper->viewCountText->simpleText ?? null; + // Dc645M8Het8 + $videoId = $wrapper->videoId; + // Jumbo frames - transfer more data faster! + $title = $wrapper->title->runs[0]->text ?? $wrapper->title->accessibility->accessibilityData->label ?? null; + $author = null; + $description = $wrapper->descriptionSnippet->runs[0]->text ?? null; + // 5 days ago | 1 month ago + $publishedTimeText = $wrapper->publishedTimeText->simpleText ?? $wrapper->videoInfo->runs[2]->text ?? null; + $timestamp = null; + if ($publishedTimeText) { + try { + $publicationDate = new \DateTimeImmutable($publishedTimeText); + // Hard-code hour, minute and second + $publicationDate = $publicationDate->setTime(0, 0, 0); + $timestamp = $publicationDate->getTimestamp(); + } catch (\Exception $e) { + } + } + $durationText = 0; - if (isset($wrapper->lengthText)) { - $durationText = $wrapper->lengthText->simpleText; + if ($lengthText) { + $durationText = $lengthText; } else { foreach ($wrapper->thumbnailOverlays as $overlay) { if (isset($overlay->thumbnailOverlayTimeStatusRenderer)) { @@ -497,35 +503,37 @@ class YoutubeBridge extends BridgeAbstract } sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds); $duration = $hours * 3600 + $minutes * 60 + $seconds; - if ($duration < $duration_min || $duration > $duration_max) { + if ($duration < $minimumDurationSeconds || $duration > $maximumDurationSeconds) { continue; } } - - //$durationSeconds = (int) $wrapper->lengthSeconds; - if ($duration < $duration_min || $duration > $duration_max) { - continue; + if (!$description || !$timestamp) { + $this->fetchVideoDetails($videoId, $author, $description, $timestamp); + } + $this->addItem($videoId, $title, $author, $description, $timestamp); + if (count($this->items) >= 99) { + break; } - $this->fetchVideoDetails($videoId, $author, $desc, $time); - $this->addItem($videoId, $title, $author, $desc, $time); } } - private function addItem($videoId, $title, $author, $desc, $time, $thumbnail = '') + private function addItem($videoId, $title, $author, $description, $timestamp, $thumbnail = '') { + $description = nl2br($description); + $item = []; // This should probably be uid? $item['id'] = $videoId; $item['title'] = $title; - $item['author'] = $author; - $item['timestamp'] = $time; + $item['author'] = $author ?? ''; + $item['timestamp'] = $timestamp; $item['uri'] = self::URI . '/watch?v=' . $videoId; if (!$thumbnail) { // Fallback to default thumbnail if there aren't any provided. $thumbnail = '0'; } $thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $videoId . '/' . $thumbnail . '.jpg'; - $item['content'] = sprintf('
%s', $item['uri'], $thumbnailUri, $desc); + $item['content'] = sprintf('
%s', $item['uri'], $thumbnailUri, $description); $this->items[] = $item; } From f7f3ca0126ed42ded38c9d299a12cb6c8dfccba0 Mon Sep 17 00:00:00 2001 From: Dag Date: Sun, 15 Oct 2023 03:37:50 +0200 Subject: [PATCH 177/716] fix(tapas): bug in prior refactor (#3758) --- bridges/TapasBridge.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bridges/TapasBridge.php b/bridges/TapasBridge.php index ea6a7ff6..8009e64d 100644 --- a/bridges/TapasBridge.php +++ b/bridges/TapasBridge.php @@ -35,7 +35,7 @@ class TapasBridge extends FeedExpander if (preg_match('/^[\d]+$/', $this->getInput('title'))) { $this->id = $this->getInput('title'); } - if ($this->getInput('force_title') or !$this->id) { + if ($this->getInput('force_title') || !$this->id) { $html = getSimpleHTMLDOM($this->getURI()) or returnServerError('Could not request ' . $this->getURI()); $this->id = $html->find('meta[property$=":url"]', 0)->content; $this->id = str_ireplace(['tapastic://series/', '/info'], '', $this->id); @@ -53,6 +53,7 @@ class TapasBridge extends FeedExpander // } // } + $item['content'] ??= ''; if ($this->getInput('extend_content')) { $html = getSimpleHTMLDOM($item['uri']); $item['content'] = $item['content'] ?? ''; @@ -77,6 +78,6 @@ class TapasBridge extends FeedExpander if ($this->id) { return self::URI . 'rss/series/' . $this->id; } - return self::URI; + return self::URI . 'series/' . $this->getInput('title') . '/info/'; } } From 408c2e5e918dd94716281dfce4bb5b6882cc829e Mon Sep 17 00:00:00 2001 From: Ololbu Date: Sun, 15 Oct 2023 18:24:07 +0500 Subject: [PATCH 178/716] [FicbookBridge] Fix timestamp (#3760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete a year word after date digits: `DD m YYYY г., HH:MM` to `DD m YYYY, HH:MM` --- bridges/FicbookBridge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/bridges/FicbookBridge.php b/bridges/FicbookBridge.php index 98288514..d11015ad 100644 --- a/bridges/FicbookBridge.php +++ b/bridges/FicbookBridge.php @@ -184,6 +184,7 @@ class FicbookBridge extends BridgeAbstract ]; $fixed_date = str_replace($ru_month, $en_month, $date); + $fixed_date = str_replace(' г.', '', $fixed_date); if ($fixed_date === $date) { Debug::log('Unable to fix date: ' . $date); From ef5bd83bd0d8645b1d7ae4201e7a167f82e3eaee Mon Sep 17 00:00:00 2001 From: Dag Date: Mon, 16 Oct 2023 02:58:03 +0200 Subject: [PATCH 179/716] feat: preserve and reproduce podcast feeds (itunes rss module) (#3759) --- bridges/CssSelectorFeedExpanderBridge.php | 4 +- bridges/NyaaTorrentsBridge.php | 43 ++------- composer.json | 4 +- formats/AtomFormat.php | 18 +++- formats/MrssFormat.php | 30 ++++-- lib/FeedParser.php | 110 ++++++++++++++-------- lib/FormatAbstract.php | 2 + 7 files changed, 126 insertions(+), 85 deletions(-) diff --git a/bridges/CssSelectorFeedExpanderBridge.php b/bridges/CssSelectorFeedExpanderBridge.php index 9f332fb9..49bbd473 100644 --- a/bridges/CssSelectorFeedExpanderBridge.php +++ b/bridges/CssSelectorFeedExpanderBridge.php @@ -50,7 +50,9 @@ class CssSelectorFeedExpanderBridge extends CssSelectorBridge $discard_thumbnail = $this->getInput('discard_thumbnail'); $limit = $this->getInput('limit'); - $source_feed = (new FeedParser())->parseFeed(getContents($url)); + $feedParser = new FeedParser(); + $xml = getContents($url); + $source_feed = $feedParser->parseFeed($xml); $items = $source_feed['items']; // Map Homepage URL (Default: Root page) diff --git a/bridges/NyaaTorrentsBridge.php b/bridges/NyaaTorrentsBridge.php index f7eea07f..fcf2b197 100644 --- a/bridges/NyaaTorrentsBridge.php +++ b/bridges/NyaaTorrentsBridge.php @@ -62,52 +62,27 @@ class NyaaTorrentsBridge extends BridgeAbstract public function collectData() { - // Manually parsing because we need to acccess the nyaa namespace in the xml - $xml = simplexml_load_string(getContents($this->getURI())); - $channel = $xml->channel[0]; - $feed = []; - $feed['title'] = trim((string)$channel->title); - $feed['uri'] = trim((string)$channel->link); - if (!empty($channel->image)) { - $feed['icon'] = trim((string)$channel->image->url); - } - $items = $xml->channel[0]->item; - foreach ($items as $feedItem) { - $item = [ - 'title' => (string) $feedItem->title, - 'uri' => (string) $feedItem->link, - ]; - + $feedParser = new FeedParser(); + $feed = $feedParser->parseFeed(getContents($this->getURI())); + foreach ($feed['items'] as $item) { $item['id'] = str_replace(['https://nyaa.si/download/', '.torrent'], '', $item['uri']); - - $nyaaNamespace = (array)($feedItem->children('nyaa', true)); - $item = array_merge($item, $nyaaNamespace); - - // Convert URI from torrent file to web page $item['uri'] = str_replace('/download/', '/view/', $item['uri']); $item['uri'] = str_replace('.torrent', '', $item['uri']); - - $item_html = getSimpleHTMLDOMCached($item['uri']); - if ($item_html) { - // Retrieve full description from page contents - $item_desc = str_get_html( - markdownToHtml(html_entity_decode($item_html->find('#torrent-description', 0)->innertext)) - ); - - // Retrieve image for thumbnail or generic logo fallback + $dom = getSimpleHTMLDOMCached($item['uri']); + if ($dom) { + $description = $dom->find('#torrent-description', 0)->innertext ?? ''; + $itemDom = str_get_html(markdownToHtml(html_entity_decode($description))); $item_image = $this->getURI() . 'static/img/avatar/default.png'; - foreach ($item_desc->find('img') as $img) { + foreach ($itemDom->find('img') as $img) { if (strpos($img->src, 'prez') === false) { $item_image = $img->src; break; } } - $item['enclosures'] = [$item_image]; - $item['content'] = $item_desc; + $item['content'] = (string) $itemDom; } - $this->items[] = $item; if (count($this->items) >= 10) { break; diff --git a/composer.json b/composer.json index 31e31d74..0e7abb84 100644 --- a/composer.json +++ b/composer.json @@ -28,6 +28,7 @@ "ext-openssl": "*", "ext-libxml": "*", "ext-simplexml": "*", + "ext-dom": "*", "ext-json": "*" }, "require-dev": { @@ -38,8 +39,7 @@ "ext-memcached": "Allows to use memcached as cache type", "ext-sqlite3": "Allows to use an SQLite database for caching", "ext-zip": "Required for FDroidRepoBridge", - "ext-intl": "Required for OLXBridge", - "ext-dom": "Allows to use some bridges based on XPath expressions" + "ext-intl": "Required for OLXBridge" }, "autoload-dev": { "psr-4": { diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index 9886e4b7..d59e42fe 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -16,6 +16,8 @@ class AtomFormat extends FormatAbstract public function stringify() { + $document = new \DomDocument('1.0', $this->getCharset()); + $feedUrl = get_current_url(); $extraInfos = $this->getExtraInfos(); @@ -25,7 +27,6 @@ class AtomFormat extends FormatAbstract $uri = $extraInfos['uri']; } - $document = new \DomDocument('1.0', $this->getCharset()); $document->formatOutput = true; $feed = $document->createElementNS(self::ATOM_NS, 'feed'); $document->appendChild($feed); @@ -81,6 +82,7 @@ class AtomFormat extends FormatAbstract $linkSelf->setAttribute('href', $feedUrl); foreach ($this->getItems() as $item) { + $itemArray = $item->toArray(); $entryTimestamp = $item->getTimestamp(); $entryTitle = $item->getTitle(); $entryContent = $item->getContent(); @@ -138,7 +140,19 @@ class AtomFormat extends FormatAbstract $entry->appendChild($id); $id->appendChild($document->createTextNode($entryID)); - if (!empty($entryUri)) { + if (isset($itemArray['itunes'])) { + $feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:itunes', self::ITUNES_NS); + foreach ($itemArray['itunes'] as $itunesKey => $itunesValue) { + $itunesProperty = $document->createElementNS(self::ITUNES_NS, $itunesKey); + $entry->appendChild($itunesProperty); + $itunesProperty->appendChild($document->createTextNode($itunesValue)); + } + $itunesEnclosure = $document->createElement('enclosure'); + $entry->appendChild($itunesEnclosure); + $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); + $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); + $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + } elseif (!empty($entryUri)) { $entryLinkAlternate = $document->createElement('link'); $entry->appendChild($entryLinkAlternate); $entryLinkAlternate->setAttribute('rel', 'alternate'); diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 984611c7..4fd06439 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -34,6 +34,8 @@ class MrssFormat extends FormatAbstract public function stringify() { + $document = new \DomDocument('1.0', $this->getCharset()); + $feedUrl = get_current_url(); $extraInfos = $this->getExtraInfos(); if (empty($extraInfos['uri'])) { @@ -42,7 +44,6 @@ class MrssFormat extends FormatAbstract $uri = $extraInfos['uri']; } - $document = new \DomDocument('1.0', $this->getCharset()); $document->formatOutput = true; $feed = $document->createElement('rss'); $document->appendChild($feed); @@ -99,22 +100,23 @@ class MrssFormat extends FormatAbstract $linkSelf->setAttribute('href', $feedUrl); foreach ($this->getItems() as $item) { + $itemArray = $item->toArray(); $itemTimestamp = $item->getTimestamp(); $itemTitle = $item->getTitle(); $itemUri = $item->getURI(); $itemContent = $item->getContent() ? break_annoying_html_tags($item->getContent()) : ''; - $entryID = $item->getUid(); + $itemUid = $item->getUid(); $isPermaLink = 'false'; - if (empty($entryID) && !empty($itemUri)) { + if (empty($itemUid) && !empty($itemUri)) { // Fallback to provided URI - $entryID = $itemUri; + $itemUid = $itemUri; $isPermaLink = 'true'; } - if (empty($entryID)) { + if (empty($itemUid)) { // Fallback to title and content - $entryID = hash('sha1', $itemTitle . $itemContent); + $itemUid = hash('sha1', $itemTitle . $itemContent); } $entry = $document->createElement('item'); @@ -126,7 +128,19 @@ class MrssFormat extends FormatAbstract $entryTitle->appendChild($document->createTextNode($itemTitle)); } - if (!empty($itemUri)) { + if (isset($itemArray['itunes'])) { + $feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:itunes', self::ITUNES_NS); + foreach ($itemArray['itunes'] as $itunesKey => $itunesValue) { + $itunesProperty = $document->createElementNS(self::ITUNES_NS, $itunesKey); + $entry->appendChild($itunesProperty); + $itunesProperty->appendChild($document->createTextNode($itunesValue)); + } + $itunesEnclosure = $document->createElement('enclosure'); + $entry->appendChild($itunesEnclosure); + $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); + $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); + $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + } if (!empty($itemUri)) { $entryLink = $document->createElement('link'); $entry->appendChild($entryLink); $entryLink->appendChild($document->createTextNode($itemUri)); @@ -135,7 +149,7 @@ class MrssFormat extends FormatAbstract $entryGuid = $document->createElement('guid'); $entryGuid->setAttribute('isPermaLink', $isPermaLink); $entry->appendChild($entryGuid); - $entryGuid->appendChild($document->createTextNode($entryID)); + $entryGuid->appendChild($document->createTextNode($itemUid)); if (!empty($itemTimestamp)) { $entryPublished = $document->createElement('pubDate'); diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 1393f5f5..2d982de1 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -3,11 +3,13 @@ declare(strict_types=1); /** - * Very basic and naive feed parser that srapes out rss 0.91, 1.0, 2.0 and atom 1.0. + * Very basic and naive feed parser. * - * Emit arrays meant to be used inside rss-bridge. + * Scrapes out rss 0.91, 1.0, 2.0 and atom 1.0. * - * The feed item structure is identical to that of FeedItem + * Produce arrays meant to be used inside rss-bridge. + * + * The item structure is tweaked so that works with FeedItem */ final class FeedParser { @@ -85,9 +87,7 @@ final class FeedParser public function parseAtomItem(\SimpleXMLElement $feedItem): array { - // Some ATOM entries also contain RSS 2.0 fields $item = $this->parseRss2Item($feedItem); - if (isset($feedItem->id)) { $item['uri'] = (string)$feedItem->id; } @@ -131,8 +131,35 @@ final class FeedParser public function parseRss2Item(\SimpleXMLElement $feedItem): array { - // Primary data is compatible to 0.91 with some additional data - $item = $this->parseRss091Item($feedItem); + $item = [ + 'uri' => '', + 'title' => '', + 'content' => '', + 'timestamp' => '', + 'author' => '', + //'uid' => null, + //'categories' => [], + //'enclosures' => [], + ]; + + foreach ($feedItem as $k => $v) { + $hasChildren = count($v) !== 0; + if (!$hasChildren) { + $item[$k] = (string) $v; + } + } + + if (isset($feedItem->link)) { + // todo: trim uri + $item['uri'] = (string)$feedItem->link; + } + if (isset($feedItem->title)) { + $item['title'] = html_entity_decode((string)$feedItem->title); + } + if (isset($feedItem->description)) { + $item['content'] = (string)$feedItem->description; + } + $namespaces = $feedItem->getNamespaces(true); if (isset($namespaces['dc'])) { $dc = $feedItem->children($namespaces['dc']); @@ -140,7 +167,24 @@ final class FeedParser if (isset($namespaces['media'])) { $media = $feedItem->children($namespaces['media']); } - + foreach ($namespaces as $namespaceName => $namespaceUrl) { + if (in_array($namespaceName, ['', 'content', 'media'])) { + continue; + } + $module = $feedItem->children($namespaceUrl); + $item[$namespaceName] = []; + foreach ($module as $moduleKey => $moduleValue) { + $item[$namespaceName][$moduleKey] = (string) $moduleValue; + } + } + if (isset($namespaces['itunes'])) { + $enclosure = $feedItem->enclosure; + $item['enclosure'] = [ + 'url' => (string) $enclosure['url'], + 'length' => (string) $enclosure['length'], + 'type' => (string) $enclosure['type'], + ]; + } if (isset($feedItem->guid)) { // Pluck out a url from guid foreach ($feedItem->guid->attributes() as $attribute => $value) { @@ -185,8 +229,26 @@ final class FeedParser public function parseRss1Item(\SimpleXMLElement $feedItem): array { - // 1.0 adds optional elements around the 0.91 standard - $item = $this->parseRss091Item($feedItem); + $item = [ + 'uri' => '', + 'title' => '', + 'content' => '', + 'timestamp' => '', + 'author' => '', + //'uid' => null, + //'categories' => [], + //'enclosures' => [], + ]; + if (isset($feedItem->link)) { + // todo: trim uri + $item['uri'] = (string)$feedItem->link; + } + if (isset($feedItem->title)) { + $item['title'] = html_entity_decode((string)$feedItem->title); + } + if (isset($feedItem->description)) { + $item['content'] = (string)$feedItem->description; + } $namespaces = $feedItem->getNamespaces(true); if (isset($namespaces['dc'])) { $dc = $feedItem->children($namespaces['dc']); @@ -199,32 +261,4 @@ final class FeedParser } return $item; } - - public function parseRss091Item(\SimpleXMLElement $feedItem): array - { - $item = [ - 'uri' => null, - 'title' => null, - 'content' => null, - 'timestamp' => null, - 'author' => null, - //'uid' => null, - //'categories' => [], - //'enclosures' => [], - ]; - if (isset($feedItem->link)) { - // todo: trim uri - $item['uri'] = (string)$feedItem->link; - } - if (isset($feedItem->title)) { - $item['title'] = html_entity_decode((string)$feedItem->title); - } - // rss 0.91 doesn't support timestamps - // rss 0.91 doesn't support authors - // rss 0.91 doesn't support enclosures - if (isset($feedItem->description)) { - $item['content'] = (string)$feedItem->description; - } - return $item; - } } diff --git a/lib/FormatAbstract.php b/lib/FormatAbstract.php index b05a5764..c76d1e42 100644 --- a/lib/FormatAbstract.php +++ b/lib/FormatAbstract.php @@ -2,6 +2,8 @@ abstract class FormatAbstract { + public const ITUNES_NS = 'http://www.itunes.com/dtds/podcast-1.0.dtd'; + const MIME_TYPE = 'text/plain'; protected string $charset = 'UTF-8'; From 563c2a345b632a0245f888a925151e4d23b1a776 Mon Sep 17 00:00:00 2001 From: Dag Date: Mon, 16 Oct 2023 03:43:18 +0200 Subject: [PATCH 180/716] refactor (#3763) --- bridges/ARDAudiothekBridge.php | 7 ++++++- bridges/YoutubeBridge.php | 33 +++++++++++---------------------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/bridges/ARDAudiothekBridge.php b/bridges/ARDAudiothekBridge.php index 07a3cc68..2c1958f3 100644 --- a/bridges/ARDAudiothekBridge.php +++ b/bridges/ARDAudiothekBridge.php @@ -113,7 +113,12 @@ class ARDAudiothekBridge extends BridgeAbstract $item['timestamp'] = $audio->publicationStartDateAndTime; $item['uid'] = $audio->id; $item['author'] = $audio->programSet->publicationService->title; - $item['categories'] = [ $audio->programSet->editorialCategories->title ]; + + $category = $audio->programSet->editorialCategories->title ?? null; + if ($category) { + $item['categories'] = [$category]; + } + $this->items[] = $item; } } diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 9b5dd44e..993f8c90 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -96,7 +96,6 @@ class YoutubeBridge extends BridgeAbstract private function collectDataInternal() { - $xml = ''; $html = ''; $url_feed = ''; $url_listing = ''; @@ -115,16 +114,13 @@ class YoutubeBridge extends BridgeAbstract if ($username) { // user and channel - $request = $username; - $url_feed = self::URI . '/feeds/videos.xml?user=' . urlencode($request); - $url_listing = self::URI . '/user/' . urlencode($request) . '/videos'; + $url_feed = self::URI . '/feeds/videos.xml?user=' . urlencode($username); + $url_listing = self::URI . '/user/' . urlencode($username) . '/videos'; } elseif ($channel) { - $request = $channel; - $url_feed = self::URI . '/feeds/videos.xml?channel_id=' . urlencode($request); - $url_listing = self::URI . '/channel/' . urlencode($request) . '/videos'; + $url_feed = self::URI . '/feeds/videos.xml?channel_id=' . urlencode($channel); + $url_listing = self::URI . '/channel/' . urlencode($channel) . '/videos'; } elseif ($custom) { - $request = $custom; - $url_listing = self::URI . '/' . urlencode($request) . '/videos'; + $url_listing = self::URI . '/' . urlencode($custom) . '/videos'; } if ($url_feed || $url_listing) { @@ -152,7 +148,7 @@ class YoutubeBridge extends BridgeAbstract // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; $this->fetchItemsFromFromJsonData($jsonData); } else { - returnServerError('Unable to get data from YouTube. Username/Channel: ' . $request); + returnServerError('Unable to get data from YouTube'); } } else { // Fetch the xml feed @@ -162,13 +158,8 @@ class YoutubeBridge extends BridgeAbstract $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); } elseif ($playlist) { // playlist - // TODO: this mode makes a lot of excess video query requests. - // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration" - // This cache will be used to find out, which videos to fetch - // to make feed of 15 items or more, if there a lot of videos published on that date. - $request = $playlist; - $url_feed = self::URI . '/feeds/videos.xml?playlist_id=' . urlencode($request); - $url_listing = self::URI . '/playlist?list=' . urlencode($request); + $url_feed = self::URI . '/feeds/videos.xml?playlist_id=' . urlencode($playlist); + $url_listing = self::URI . '/playlist?list=' . urlencode($playlist); $html = $this->fetch($url_listing); $jsonData = $this->extractJsonFromHtml($html); // TODO: this method returns only first 100 video items @@ -194,8 +185,7 @@ class YoutubeBridge extends BridgeAbstract }); } elseif ($search) { // search - $request = $search; - $url_listing = self::URI . '/results?search_query=' . urlencode($request) . '&sp=CAI%253D'; + $url_listing = self::URI . '/results?search_query=' . urlencode($search) . '&sp=CAI%253D'; $html = $this->fetch($url_listing); $jsonData = $this->extractJsonFromHtml($html); $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; @@ -209,10 +199,9 @@ class YoutubeBridge extends BridgeAbstract } $this->fetchItemsFromFromJsonData($jsonData); $this->feeduri = $url_listing; - $this->feedName = 'Search: ' . $request; + $this->feedName = 'Search: ' . $search; } else { - returnClientError("You must either specify either:\n - YouTube - username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); + returnClientError("You must either specify either:\n - YouTube username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); } } From 8203196145c30587aa83e0d22c9d1c915356e0c9 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Wed, 18 Oct 2023 02:33:29 +0200 Subject: [PATCH 181/716] [ImgsedBridge] More robust data parsing (#3766) Date Interval with the article "an" or "a" are now handled in a generic way : every "article" is replaced by the number "1" instead of a handling of multiple special case --- bridges/ImgsedBridge.php | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/bridges/ImgsedBridge.php b/bridges/ImgsedBridge.php index e605cf4f..12466c6b 100644 --- a/bridges/ImgsedBridge.php +++ b/bridges/ImgsedBridge.php @@ -206,14 +206,12 @@ HTML, { // Parse date, and transform the date into a timetamp, even in a case of a relative date $date = date_create(); - $dateString = str_replace(' ago', '', $content); - // Special case : 'a day' is not a valid interval in PHP, so replace it with it's PHP equivalenbt : '1 day' - if ($dateString == 'a day') { - $dateString = '1 day'; - } - if ($dateString === 'an hour') { - $dateString = '1 hour'; - } + + // Content trimmed to be sure that the "article" is at the beginning of the string and remove "ago" to make it a valid PHP date interval + $dateString = trim(str_replace(' ago', '', $content)); + + // Replace the article "an" or "a" by the number "1" to be a valid PHP date interval + $dateString = preg_replace('/^((an|a) )/m', '1 ', $dateString); $relativeDate = date_interval_create_from_date_string($dateString); if ($relativeDate) { From a41bb088f816454d9fbd07011e1365d0845fde3d Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 18 Oct 2023 19:10:52 +0200 Subject: [PATCH 182/716] [CssSelectorBridge] Add more metadata tags (#3768) Add og: variants for published/updated time and author --- bridges/CssSelectorBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/CssSelectorBridge.php b/bridges/CssSelectorBridge.php index f6ab8d15..8fba5285 100644 --- a/bridges/CssSelectorBridge.php +++ b/bridges/CssSelectorBridge.php @@ -336,9 +336,11 @@ class CssSelectorBridge extends BridgeAbstract ], 'timestamp' => [ 'article:published_time', + 'og:article:published_time', 'releaseDate', 'releasedate', 'article:modified_time', + 'og:article:modified_time', 'lastModified', 'lastmodified' ], @@ -351,8 +353,9 @@ class CssSelectorBridge extends BridgeAbstract 'thumbnailimg' ], 'author' => [ - 'author', 'article:author', + 'og:article:author', + 'author', 'article:author:username', 'profile:first_name', 'profile:last_name', From 7533ef12e3348779b1be2a1aa1704f972d3176a3 Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 18 Oct 2023 19:12:19 +0200 Subject: [PATCH 183/716] [html] improve srcset attribute parsing (#3769) Fix commas not being used for splitting, resulting in broken src URL in some cases: srcset="url1.jpg, url2.jpg 2x" would give src="url1.jpg," --- lib/html.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/html.php b/lib/html.php index 505221fc..ba8067a6 100644 --- a/lib/html.php +++ b/lib/html.php @@ -244,16 +244,26 @@ function convertLazyLoading($dom) $dom = str_get_html($dom); } + // Retrieve image URL from srcset attribute + // https://developer.mozilla.org/en-US/docs/Web/API/HTMLImageElement/srcset + // Example: convert "header640.png 640w, header960.png 960w, header1024.png 1024w" to "header1024.png" + $srcset_to_src = function ($srcset) { + $sources = explode(',', $srcset); + $last_entry = trim(end($sources)); + $url = explode(' ', $last_entry)[0]; + return $url; + }; + // Process standalone images, embeds and picture sources foreach ($dom->find('img, iframe, source') as $img) { if (!empty($img->getAttribute('data-src'))) { $img->src = $img->getAttribute('data-src'); } elseif (!empty($img->getAttribute('data-srcset'))) { - $img->src = explode(' ', $img->getAttribute('data-srcset'))[0]; + $img->src = $srcset_to_src($img->getAttribute('data-srcset')); } elseif (!empty($img->getAttribute('data-lazy-src'))) { $img->src = $img->getAttribute('data-lazy-src'); } elseif (!empty($img->getAttribute('srcset'))) { - $img->src = explode(' ', $img->getAttribute('srcset'))[0]; + $img->src = $srcset_to_src($img->getAttribute('srcset')); } else { continue; // Proceed to next element without removing attributes } From 9056106c2d52991931ba7fc1c41494697396d477 Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 18 Oct 2023 19:13:33 +0200 Subject: [PATCH 184/716] [CNet] Rewrite bridge (#3764) (#3770) Bridge was broken. Full bridge rewrite using Sitemap as source. --- bridges/CNETBridge.php | 168 +++++++++++++++++++------------------- bridges/SitemapBridge.php | 6 +- 2 files changed, 87 insertions(+), 87 deletions(-) diff --git a/bridges/CNETBridge.php b/bridges/CNETBridge.php index 34442abd..4a63c847 100644 --- a/bridges/CNETBridge.php +++ b/bridges/CNETBridge.php @@ -1,6 +1,6 @@ 'list', 'values' => [ 'All articles' => '', - 'Apple' => 'apple', - 'Google' => 'google', - 'Microsoft' => 'tags-microsoft', - 'Computers' => 'topics-computers', - 'Mobile' => 'topics-mobile', - 'Sci-Tech' => 'topics-sci-tech', - 'Security' => 'topics-security', - 'Internet' => 'topics-internet', - 'Tech Industry' => 'topics-tech-industry' + 'Tech' => 'tech', + 'Money' => 'personal-finance', + 'Home' => 'home', + 'Wellness' => 'health', + 'Energy' => 'home/energy-and-utilities', + 'Deals' => 'deals', + 'Computing' => 'tech/computing', + 'Mobile' => 'tech/mobile', + 'Science' => 'science', + 'Services' => 'tech/services-and-software' ] - ] + ], + 'limit' => self::LIMIT ] ]; - private function cleanArticle($article_html) - { - $offset_p = strpos($article_html, '

'); - $offset_figure = strpos($article_html, '', '', $article_html); - $article_html = str_replace('', '', $article_html); - $article_html = StripWithDelimiters($article_html, ''); - $article_html = stripWithDelimiters($article_html, 'find('div.originalImage', 0); - } - if (empty($article_thumbnail)) { - $article_thumbnail = $article_html->find('span.imageContainer', 0); - } - if (is_object($article_thumbnail)) { - $article_thumbnail = $article_thumbnail->find('img', 0)->src; - } - - $article_content .= trim( - $this->cleanArticle( - extractFromDelimiters( - $article_html, - 'find('script[type=application/ld+json]') as $ldjson) { + $datePublished = extractFromDelimiters($ldjson->innertext, '"datePublished":"', '"'); + if ($datePublished !== false) { + $date = strtotime($datePublished); + } + $imageObject = extractFromDelimiters($ldjson->innertext, 'ImageObject","url":"', '"'); + if ($imageObject !== false) { + $enclosure = $imageObject; } - - $item = []; - $item['uri'] = $article_uri; - $item['title'] = $article_title; - $item['author'] = $article_author; - $item['timestamp'] = $article_timestamp; - $item['enclosures'] = [$article_thumbnail]; - $item['content'] = $article_content; - $this->items[] = $item; } + + foreach ($content->find('div.c-shortcodeGallery') as $cleanup) { + $cleanup->outertext = ''; + } + + foreach ($content->find('figure') as $figure) { + $img = $figure->find('img', 0); + if ($img) { + $figure->outertext = $img->outertext; + } + } + + $content = $content->innertext; + + if ($enclosure) { + $content = "

" . $content; + } + + if ($headline) { + $content = '

' . $headline->plaintext . '


' . $content; + } + + $item = []; + $item['uri'] = $article_uri; + $item['title'] = $title; + $item['author'] = $author; + $item['content'] = $content; + + if (!is_null($date)) { + $item['timestamp'] = $date; + } + + if (!is_null($enclosure)) { + $item['enclosures'] = [$enclosure]; + } + + $this->items[] = $item; } } } diff --git a/bridges/SitemapBridge.php b/bridges/SitemapBridge.php index bdf662ee..bbbb3e16 100644 --- a/bridges/SitemapBridge.php +++ b/bridges/SitemapBridge.php @@ -131,7 +131,7 @@ class SitemapBridge extends CssSelectorBridge foreach ($sitemap->find('sitemap') as $nested_sitemap) { $url = $nested_sitemap->find('loc'); if (!empty($url)) { - $url = $url[0]->plaintext; + $url = trim($url[0]->plaintext); if (str_ends_with(strtolower($url), '.xml')) { $nested_sitemap_xml = $this->getSitemapXml($url, true); $nested_sitemap_links = $this->sitemapXmlToList($nested_sitemap_xml, $url_pattern, null, true); @@ -148,8 +148,8 @@ class SitemapBridge extends CssSelectorBridge $url = $item->find('loc'); $lastmod = $item->find('lastmod'); if (!empty($url) && !empty($lastmod)) { - $url = $url[0]->plaintext; - $lastmod = $lastmod[0]->plaintext; + $url = trim($url[0]->plaintext); + $lastmod = trim($lastmod[0]->plaintext); $timestamp = strtotime($lastmod); if (empty($url_pattern) || preg_match('/' . $url_pattern . '/', $url) === 1) { $links[$url] = $timestamp; From 658391263ebdbd8614cefeb63eb43e3ea521e5b6 Mon Sep 17 00:00:00 2001 From: Teemu Ikonen Date: Thu, 19 Oct 2023 18:02:53 +0300 Subject: [PATCH 185/716] Add 'itunes:duration' tag for items with duration (#3774) * [{Atom,Mrss}Format] Allow itunes tags on items without enclosure * [Arte7Bridge] Add $item['itunes']['duration'] value --- bridges/Arte7Bridge.php | 4 ++++ formats/AtomFormat.php | 12 +++++++----- formats/MrssFormat.php | 12 +++++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/bridges/Arte7Bridge.php b/bridges/Arte7Bridge.php index 239fc6ad..5898e881 100644 --- a/bridges/Arte7Bridge.php +++ b/bridges/Arte7Bridge.php @@ -156,6 +156,10 @@ class Arte7Bridge extends BridgeAbstract . $element['mainImage']['url'] . '" />
'; + $item['itunes'] = [ + 'duration' => $durationSeconds, + ]; + $this->items[] = $item; } } diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index d59e42fe..07ca7272 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -147,11 +147,13 @@ class AtomFormat extends FormatAbstract $entry->appendChild($itunesProperty); $itunesProperty->appendChild($document->createTextNode($itunesValue)); } - $itunesEnclosure = $document->createElement('enclosure'); - $entry->appendChild($itunesEnclosure); - $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); - $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); - $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + if (isset($itemArray['enclosure'])) { + $itunesEnclosure = $document->createElement('enclosure'); + $entry->appendChild($itunesEnclosure); + $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); + $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); + $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + } } elseif (!empty($entryUri)) { $entryLinkAlternate = $document->createElement('link'); $entry->appendChild($entryLinkAlternate); diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 4fd06439..5b96a6a7 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -135,11 +135,13 @@ class MrssFormat extends FormatAbstract $entry->appendChild($itunesProperty); $itunesProperty->appendChild($document->createTextNode($itunesValue)); } - $itunesEnclosure = $document->createElement('enclosure'); - $entry->appendChild($itunesEnclosure); - $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); - $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); - $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + if (isset($itemArray['enclosure'])) { + $itunesEnclosure = $document->createElement('enclosure'); + $entry->appendChild($itunesEnclosure); + $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']); + $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); + $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); + } } if (!empty($itemUri)) { $entryLink = $document->createElement('link'); $entry->appendChild($entryLink); From 8ff39f64f7554de110cb4296e199789d9fa7f5a8 Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 20 Oct 2023 13:31:52 +0200 Subject: [PATCH 186/716] [html] add data-orig-file tag (#3777) Add support for data-orig-file tag in convertLazyLoading() Remplace end() with array_key_last() as discussed in #3769 Fix typo in comment --- lib/html.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/html.php b/lib/html.php index ba8067a6..1de581d9 100644 --- a/lib/html.php +++ b/lib/html.php @@ -249,7 +249,7 @@ function convertLazyLoading($dom) // Example: convert "header640.png 640w, header960.png 960w, header1024.png 1024w" to "header1024.png" $srcset_to_src = function ($srcset) { $sources = explode(',', $srcset); - $last_entry = trim(end($sources)); + $last_entry = trim($sources[array_key_last($sources)]); $url = explode(' ', $last_entry)[0]; return $url; }; @@ -262,12 +262,15 @@ function convertLazyLoading($dom) $img->src = $srcset_to_src($img->getAttribute('data-srcset')); } elseif (!empty($img->getAttribute('data-lazy-src'))) { $img->src = $img->getAttribute('data-lazy-src'); + } elseif (!empty($img->getAttribute('data-orig-file'))) { + $img->src = $img->getAttribute('data-orig-file'); } elseif (!empty($img->getAttribute('srcset'))) { $img->src = $srcset_to_src($img->getAttribute('srcset')); } else { continue; // Proceed to next element without removing attributes } - foreach (['loading', 'decoding', 'srcset', 'data-src', 'data-srcset'] as $attr) { + // Remove attributes that may be processed by the client (data-* are not) + foreach (['loading', 'decoding', 'srcset'] as $attr) { if ($img->hasAttribute($attr)) { $img->removeAttribute($attr); } @@ -284,7 +287,7 @@ function convertLazyLoading($dom) $img->tag = 'img'; } // Adding/removing node would change its position inside the parent element, - // So instead we rewrite the node in-place though the outertext attribute + // So instead we rewrite the node in-place through the outertext attribute $picture->outertext = $img->outertext; } } From 4f7451895bf888e465a9879c466d4067a8c4a17f Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 20 Oct 2023 13:33:07 +0200 Subject: [PATCH 187/716] Fix: content.php: last-modified/if-unmodified-since (#3771) (#3772) * Fix: content.php: last-modified/if-unmodified-since (#3771) Fix exception if server sent invalid Last-Modified header Add support for Unix time instead of standard date string Send back standard RFC7231 date string instead of Unix time * Fix: content.php: if-unmodified-since: cURL API Use getTimestamp() as cURL expects that and will format the If-Modified-Since header appropriately. --- lib/contents.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/contents.php b/lib/contents.php index a3830ca7..055d6bf3 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -63,8 +63,13 @@ function getContents( if ($cachedResponse) { $cachedLastModified = $cachedResponse->getHeader('last-modified'); if ($cachedLastModified) { - $cachedLastModified = new \DateTimeImmutable($cachedLastModified); - $config['if_not_modified_since'] = $cachedLastModified->getTimestamp(); + try { + // Some servers send Unix timestamp instead of RFC7231 date. Prepend it with @ to allow parsing as DateTime + $cachedLastModified = new \DateTimeImmutable((is_numeric($cachedLastModified) ? '@' : '') . $cachedLastModified); + $config['if_not_modified_since'] = $cachedLastModified->getTimestamp(); + } catch (Exception $dateTimeParseFailue) { + // Ignore invalid 'Last-Modified' HTTP header value + } } } From 4722201281f3e62c3f8067d2721e4b771a6ba5e0 Mon Sep 17 00:00:00 2001 From: Manu <3916435+m3nu@users.noreply.github.com> Date: Fri, 20 Oct 2023 19:29:28 +0100 Subject: [PATCH 188/716] Add one-click install to PikaPods (#3778) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7037095e..570fb87d 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ Browse http://localhost:3000/ [![Deploy on Scalingo](https://cdn.scalingo.com/deploy/button.svg)](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge) [![Deploy to Heroku](https://www.herokucdn.com/deploy/button.svg)](https://heroku.com/deploy) [![Deploy to Cloudron](https://cloudron.io/img/button.svg)](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html) +[![Run on PikaPods](https://www.pikapods.com/static/run-button.svg)](https://www.pikapods.com/pods?run=rssbridge) The Heroku quick deploy currently does not work. It might possibly work if you fork this repo and modify the `repository` in `scalingo.json`. See https://github.com/RSS-Bridge/rss-bridge/issues/2688 From a6a450220991a42f34b3dfa3219b0cafbaba11eb Mon Sep 17 00:00:00 2001 From: mruac Date: Sat, 21 Oct 2023 20:24:50 +1030 Subject: [PATCH 189/716] [Itaku] extend the number of images shown in a post (#3780) * minor fixes - extended itaku post if post does not have all images * phpcbf * . * resolve deprecated explode param yay null coalesces --- bridges/ItakuBridge.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index 62a130ff..149757f5 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -201,7 +201,7 @@ class ItakuBridge extends BridgeAbstract 'rating_e' => $this->getInput('rating_e') ]; - $tag_arr = explode(' ', $this->getInput('tags')); + $tag_arr = explode(' ', $this->getInput('tags') ?? ''); foreach ($tag_arr as $str) { switch ($str[0]) { case '-': @@ -446,6 +446,9 @@ class ItakuBridge extends BridgeAbstract private function getPost($id, array $metadata = null) { + if (isset($metadata) && sizeof($metadata['gallery_images']) < $metadata['num_images']) { + $metadata = null; //force re-fetch of metadata + } $uri = self::URI . '/posts/' . $id; $url = self::URI . '/api/posts/' . $id . '/?format=json'; $data = $metadata ?? $this->getData($url, true, true) From f134808a268065e5000ef694149f62bb0f263b16 Mon Sep 17 00:00:00 2001 From: Park0 Date: Sun, 22 Oct 2023 17:36:36 +0200 Subject: [PATCH 190/716] Marktplaats categories added (#3761) * Update MarktplaatsBridge.php * Update MarktplaatsBridge.php only main categories As the whole list is too big only main categories are used for now. * Renamed parameter 2 to sc Renamed unused method to better reflect it usage * Update MarktplaatsBridge.php Several fixed Categories completed Added a default empty one Check if the input is not empty before using Added helper methods to generate the categorylist * Update MarktplaatsBridge.php Set the methods to private for the CI --- bridges/MarktplaatsBridge.php | 143 +++++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 4 deletions(-) diff --git a/bridges/MarktplaatsBridge.php b/bridges/MarktplaatsBridge.php index 70a369d9..6ba993e7 100644 --- a/bridges/MarktplaatsBridge.php +++ b/bridges/MarktplaatsBridge.php @@ -14,6 +14,51 @@ class MarktplaatsBridge extends BridgeAbstract 'required' => true, 'title' => 'The search string for marktplaats', ], + 'c' => [ + 'name' => 'Category', + 'type' => 'list', + 'values' => [ + 'Select a category' => '', + 'Antiek en Kunst' => '1', + 'Audio, Tv en Foto' => '31', + 'Auto's' => '91', + 'Auto-onderdelen' => '2600', + 'Auto diversen' => '48', + 'Boeken' => '201', + 'Caravans en Kamperen' => '289', + 'Cd's en Dvd's' => '1744', + 'Computers en Software' => '322', + 'Contacten en Berichten' => '378', + 'Diensten en Vakmensen' => '1098', + 'Dieren en Toebehoren' => '395', + 'Doe-het-zelf en Verbouw' => '239', + 'Fietsen en Brommers' => '445', + 'Hobby en Vrije tijd' => '1099', + 'Huis en Inrichting' => '504', + 'Huizen en Kamers' => '1032', + 'Kinderen en Baby's' => '565', + 'Kleding | Dames' => '621', + 'Kleding | Heren' => '1776', + 'Motoren' => '678', + 'Muziek en Instrumenten' => '728', + 'Postzegels en Munten' => '1784', + 'Sieraden, Tassen en Uiterlijk' => '1826', + 'Spelcomputers en Games' => '356', + 'Sport en Fitness' => '784', + 'Telecommunicatie' => '820', + 'Tickets en Kaartjes' => '1984', + 'Tuin en Terras' => '1847', + 'Vacatures' => '167', + 'Vakantie' => '856', + 'Verzamelen' => '895', + 'Watersport en Boten' => '976', + 'Witgoed en Apparatuur' => '537', + 'Zakelijke goederen' => '1085', + 'Diversen' => '428', + ], + 'required' => false, + 'title' => 'The category to search in', + ], 'z' => [ 'name' => 'zipcode', 'type' => 'text', @@ -57,7 +102,15 @@ class MarktplaatsBridge extends BridgeAbstract 'type' => 'checkbox', 'required' => false, 'title' => 'Include the raw data behind the content', - ] + ], + 'sc' => [ + 'name' => 'Sub category', + 'type' => 'number', + 'required' => false, + 'exampleValue' => '12345', + 'title' => 'Sub category has to be given by id as the list is too big to show here. + Only use subcategories that belong to the main category. Both have to be correct', + ], ] ]; const CACHE_TIMEOUT = 900; @@ -80,6 +133,12 @@ class MarktplaatsBridge extends BridgeAbstract $excludeGlobal = true; } } + if (!empty($this->getInput('c'))) { + $query .= '&l1CategoryId=' . $this->getInput('c'); + } + if (!is_null($this->getInput('sc'))) { + $query .= '&l2CategoryId=' . $this->getInput('sc'); + } $url = 'https://www.marktplaats.nl/lrp/api/search?query=' . urlencode($this->getInput('q')) . $query; $jsonString = getSimpleHTMLDOM($url); $jsonObj = json_decode($jsonString); @@ -97,15 +156,15 @@ class MarktplaatsBridge extends BridgeAbstract $item['enclosures'] = $listing->imageUrls; if (is_array($listing->imageUrls)) { foreach ($listing->imageUrls as $imgurl) { - $item['content'] .= "
\n"; + $item['content'] .= "
\n"; } } else { - $item['content'] .= "
\n"; + $item['content'] .= "
\n"; } } if (!is_null($this->getInput('r'))) { if ($this->getInput('r')) { - $item['content'] .= "
\n
\n
\n" . json_encode($listing); + $item['content'] .= "
\n
\n
\n" . json_encode($listing) . "
$url"; } } $item['content'] .= "
\n
\nPrice: " . $listing->priceInfo->priceCents / 100; @@ -130,4 +189,80 @@ class MarktplaatsBridge extends BridgeAbstract } return parent::getName(); } + + /** + * Method can be used to scrape the subcategories from marktplaats + */ + private static function scrapeSubCategories() + { + $main = []; + $main['Select a category'] = ''; + $marktplaatsHTML = file_get_html('https://www.marktplaats.nl'); + foreach ($marktplaatsHTML->find('select[id=categoryId] option') as $opt) { + if (!str_contains($opt->innertext, 'categorie')) { + $main[$opt->innertext] = $opt->value; + $ids[] = $opt->value; + } + } + + $result = []; + foreach ($ids as $id) { + $url = 'https://www.marktplaats.nl/lrp/api/search?l1CategoryId=' . $id; + $jsonstring = getContents($url); + $jsondata = json_decode((string)$jsonstring); + if (isset($jsondata->searchCategoryOptions)) { + $categories = $jsondata->searchCategoryOptions; + if (isset($jsondata->categoriesById->$id)) { + $maincategory = $jsondata->categoriesById->$id; + $array = []; + foreach ($categories as $categorie) { + $array[$categorie->fullName] = $categorie->id; + } + $result[$maincategory->fullName] = $array; + } + } else { + print($jsonstring); + } + } + $combinedResult = [ + 'main' => $main, + 'sub' => $result + ]; + return $combinedResult; + } + + /** + * Helper method to construct the array that could be used for categories + * + * @param $array + * @param $indent + * @return void + */ + private static function printArrayAsCode($array, $indent = 0) + { + foreach ($array as $key => $value) { + if (is_array($value)) { + echo str_repeat(' ', $indent) . "'$key' => [" . PHP_EOL; + self::printArrayAsCode($value, $indent + 1); + echo str_repeat(' ', $indent) . '],' . PHP_EOL; + } else { + $value = str_replace('\'', '\\\'', $value); + $key = str_replace('\'', '\\\'', $key); + echo str_repeat(' ', $indent) . "'$key' => '$value'," . PHP_EOL; + } + } + } + + private static function printScrapeArray() + { + $array = (MarktplaatsBridge::scrapeSubCategories()); + + echo '$myArray = [' . PHP_EOL; + self::printArrayAsCode($array['main'], 1); + echo '];' . PHP_EOL; + + echo '$myArray = [' . PHP_EOL; + self::printArrayAsCode($array['sub'], 1); + echo '];' . PHP_EOL; + } } From d4e4c3e89ac148d4943c8c1669322649e1d0ae7b Mon Sep 17 00:00:00 2001 From: Ryan Stafford Date: Mon, 23 Oct 2023 17:12:05 -0400 Subject: [PATCH 191/716] [FarsideNitterBridge] New twitter bridge (#3781) * [FarsideNitterBridge] New twitter bridge * example value * lint fix --- bridges/FarsideNitterBridge.php | 103 ++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 bridges/FarsideNitterBridge.php diff --git a/bridges/FarsideNitterBridge.php b/bridges/FarsideNitterBridge.php new file mode 100644 index 00000000..b167347a --- /dev/null +++ b/bridges/FarsideNitterBridge.php @@ -0,0 +1,103 @@ + [ + 'name' => 'username', + 'required' => true, + 'exampleValue' => 'NASA' + ], + 'noreply' => [ + 'name' => 'Without replies', + 'type' => 'checkbox', + 'title' => 'Only return initial tweets' + ], + 'noretweet' => [ + 'name' => 'Without retweets', + 'required' => false, + 'type' => 'checkbox', + 'title' => 'Hide retweets' + ], + 'linkbacktotwitter' => [ + 'name' => 'Link back to twitter', + 'required' => false, + 'type' => 'checkbox', + 'title' => 'Rewrite links back to twitter.com' + ] + ], + ]; + + public function detectParameters($url) + { + if (preg_match('/^(https?:\/\/)?(www\.)?(nitter\.net|twitter\.com)\/([^\/?\n]+)/', $url, $matches) > 0) { + return [ + 'username' => $matches[4], + 'noreply' => true, + 'noretweet' => true, + 'linkbacktotwitter' => true + ]; + } + return null; + } + + public function collectData() + { + $this->getRSS(); + } + + private function getRSS($attempt = 0) + { + try { + $this->collectExpandableDatas(self::URI . $this->getInput('username') . '/rss'); + } catch (\Exception $e) { + if ($attempt >= self::MAX_RETRIES) { + throw $e; + } else { + $this->getRSS($attempt++); + } + } + } + + protected function parseItem(array $item) + { + if ($this->getInput('noreply') && substr($item['title'], 0, 5) == 'R to ') { + return; + } + if ($this->getInput('noretweet') && substr($item['title'], 0, 6) == 'RT by ') { + return; + } + $item['title'] = truncate($item['title']); + if (preg_match('/(\/status\/.+)/', $item['uri'], $matches) > 0) { + if ($this->getInput('linkbacktotwitter')) { + $item['uri'] = self::HOST . $this->getInput('username') . $matches[1]; + } else { + $item['uri'] = self::URI . $this->getInput('username') . $matches[1]; + } + } + return $item; + } + + public function getName() + { + if (preg_match('/(.+) \//', parent::getName(), $matches) > 0) { + return $matches[1]; + } + return parent::getName(); + } + + public function getURI() + { + if ($this->getInput('linkbacktotwitter')) { + return self::HOST . $this->getInput('username'); + } else { + return self::URI . $this->getInput('username'); + } + } +} From cee25d862d71a1f2484135f31b0bbbd5017fcd8d Mon Sep 17 00:00:00 2001 From: ORelio Date: Tue, 24 Oct 2023 19:57:25 +0200 Subject: [PATCH 192/716] [html] clean data attributes (#3782) Some feed readers had difficulties with attributes containing html tags --- lib/html.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/html.php b/lib/html.php index 1de581d9..d65d1b20 100644 --- a/lib/html.php +++ b/lib/html.php @@ -269,7 +269,15 @@ function convertLazyLoading($dom) } else { continue; // Proceed to next element without removing attributes } - // Remove attributes that may be processed by the client (data-* are not) + + // Remove data attributes, no longer necessary + foreach ($img->getAllAttributes() as $attr => $val) { + if (str_starts_with($attr, 'data-')) { + $img->removeAttribute($attr); + } + } + + // Remove other attributes that may be processed by the client foreach (['loading', 'decoding', 'srcset'] as $attr) { if ($img->hasAttribute($attr)) { $img->removeAttribute($attr); From 1dabd10e25ca6ac1ab7cd19742707380104d2642 Mon Sep 17 00:00:00 2001 From: Niehztog Date: Mon, 30 Oct 2023 11:47:25 +0100 Subject: [PATCH 193/716] [NintendoBridge] Add new bridge (#3784) * Adds new NintendoBridge * fix item uids, fix feed title * fix feed icon, adds item categories * fix feed source uri * make currentCatgory property nullable * fix linter errors * fix linter errors * attempt to fix unit tests by assigning default category --- bridges/NintendoBridge.php | 466 +++++++++++++++++++++++++++++++++++++ 1 file changed, 466 insertions(+) create mode 100644 bridges/NintendoBridge.php diff --git a/bridges/NintendoBridge.php b/bridges/NintendoBridge.php new file mode 100644 index 00000000..34550737 --- /dev/null +++ b/bridges/NintendoBridge.php @@ -0,0 +1,466 @@ + [ + 'category' => [ + 'name' => 'Category', + 'type' => 'list', + 'values' => [ + 'All' => 'all', + 'Mario Kart 8 Deluxe' => 'mk8d', + 'Splatoon 2' => 's2', + 'Super Mario 3D All-Stars' => 'sm3as', + 'Super Mario 3D World + Bowser’s Fury' => 'sm3wbf', + 'Super Mario Maker 2' => 'smm2', + 'Super Mario Odyssey' => 'smo', + 'Super Smash Bros. Ultimate' => 'ssbu', + 'Switch Firmware' => 'sf', + 'The Legend of Zelda: Link’s Awakening' => 'tlozla', + 'The Legend of Zelda: Skyward Sword HD' => 'tlozss', + 'The Legend of Zelda: Tears of the Kingdom' => 'tloztotk', + 'Xenoblade Chronicles 2' => 'xc2', + ], + 'defaultValue' => 'mk8d', + 'title' => 'Select category' + ], + 'country' => [ + 'name' => 'Country', + 'type' => 'list', + 'values' => [ + 'België' => 'be/nl', + 'Belgique' => 'be/fr', + 'Deutschland' => 'de', + 'España' => 'es', + 'France' => 'fr', + 'Italia' => 'it', + 'Nederland' => 'nl', + 'Österreich' => 'at', + 'Portugal' => 'pt', + 'Schweiz' => 'ch/de', + 'Suisse' => 'ch/fr', + 'Svizzera' => 'ch/it', + 'UK & Ireland' => 'co.uk', + 'South Africa' => 'co.za' + ], + 'defaultValue' => 'co.uk', + 'title' => 'Select your country' + ] + ] + ]; + + const CACHE_TIMEOUT = 3600; + + const FEED_SOURCE_URL = [ + 'mk8d' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Mario-Kart-8-Deluxe-1482895.html', + 's2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Splatoon-2-1482897.html', + 'sm3as' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-All-Stars-1844226.html', + 'sm3wbf' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-World-Bowser-s-Fury-1920668.html', + 'smm2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Maker-2-1586745.html', + 'smo' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Odyssey-1482901.html', + 'ssbu' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Smash-Bros-Ultimate-1484130.html', + 'sf' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/System-Updates/Nintendo-Switch-System-Updates-and-Change-History-1445507.html', + 'tlozla' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-The-Legend-of-Zelda-Link-s-Awakening-1666739.html', + 'tlozss' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-The-Legend-of-Zelda-Skyward-Sword-HD-2022801.html', + 'tloztotk' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-The-Legend-of-Zelda-Tears-of-the-Kingdom-2388231.html', + 'xc2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/Xenoblade-Chronicles-2-Update-History-1482911.html', + ]; + const XPATH_EXPRESSION_ITEM = '//div[@class="col-xs-12 content"]/div[starts-with(@id,"v") and @class="collapse"]'; + const XPATH_EXPRESSION_ITEM_FIRMWARE = '//div[@id="latest" and @class="collapse" and @rel="1"]'; + const XPATH_EXPRESSION_ITEM_TITLE = './/h2[1]/node()'; + const XPATH_EXPRESSION_ITEM_CONTENT = '.'; + const XPATH_EXPRESSION_ITEM_URI = '//link[@rel="canonical"]/@href'; + + //const XPATH_EXPRESSION_ITEM_AUTHOR = ''; + const XPATH_EXPRESSION_ITEM_TIMESTAMP_PART = 'substring-after(//a[@class="collapse_link collapsed" and @data-target="#{{id_here}}"]/text(), "{{label_here}}")'; + const XPATH_EXPRESSION_ITEM_TIMESTAMP = 'substring(' . self::XPATH_EXPRESSION_ITEM_TIMESTAMP_PART . ', 1, string-length(' + . self::XPATH_EXPRESSION_ITEM_TIMESTAMP_PART . ') - 1)'; + + //const XPATH_EXPRESSION_ITEM_ENCLOSURES = ''; + //const XPATH_EXPRESSION_ITEM_CATEGORIES = ''; + const SETTING_FIX_ENCODING = false; + const SETTING_USE_RAW_ITEM_CONTENT = true; + + private const GAME_COUNTRY_DATE_SUBSTRING_PART = [ + 'mk8d' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'ubblicata il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 's2' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'ubblicata il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'sm3as' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ubliée le ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'sm3wbf' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'smm2' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ubliée le ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'smo' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'ssbu' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'eleased ', + ], + 'sf' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ise en ligne le ', + 'it' => 'ubblicata il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada no dia ', + 'en' => 'istributed ', + ], + 'tlozla' => [ + 'de' => 'eröffentlicht ', + 'es' => 'ublicada el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgegeven op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'tlozss' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'ubblicata l\'', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'tloztotk' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'ubliée le ', + 'it' => 'ubblicata il ', + 'nl' => 'erschenen op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + 'xc2' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], + ]; + + private const GAME_COUNTRY_DATE_FORMAT = [ + 'mk8d' => [ + 'de' => 'd.m.y', + 'es' => 'd-m-y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + 's2' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-Y', + 'fr' => 'd/m/y', + 'it' => 'd/m/y', + 'nl' => 'd/m/y', + 'pt' => 'd/m/y', + 'en' => 'd F Y', + ], + 'sm3as' => [ + 'de' => 'j. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'j m Y', + 'it' => 'j m Y', + 'nl' => 'j m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], + 'sm3wbf' => [ + 'de' => 'd.m.y', + 'es' => 'd-m-y', + 'fr' => 'd/m/y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'F j, Y', + ], + 'smm2' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-Y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/Y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + 'smo' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-Y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + 'ssbu' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'j m Y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'd/m/Y', + 'en' => 'j F Y', + ], + 'sf' => [ + 'de' => 'd.m.Y', + 'es' => 'd-m-y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/Y', + 'nl' => 'd m Y', + 'pt' => 'd/m/Y', + 'en' => 'd/m/Y', + ], + 'tlozla' => [ + 'de' => 'd. m Y', + 'es' => 'j m \d\e Y', + 'fr' => 'd/m/y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F y', + ], + 'tlozss' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'd/m/y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], + 'tloztotk' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'j m Y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], + 'xc2' => [ + 'de' => 'd.m.y', + 'es' => 'd-m-y', + 'fr' => 'd/m/Y', + 'it' => 'd/m/y', + 'nl' => 'd m Y', + 'pt' => 'd/m/y', + 'en' => 'd/m/y', + ], + ]; + + private const FOREIGN_MONTH_NAMES = [ + 'nl' => ['01' => 'januari', '02' => 'februari', '03' => 'maart', '04' => 'april', '05' => 'mei', '06' => 'juni', '07' => 'juli', '08' => 'augustus', + '09' => 'september', '10' => 'oktober', '11' => 'november', '12' => 'december'], + 'fr' => ['01' => 'janvier', '02' => 'février', '03' => 'mars', '04' => 'avril', '05' => 'mai', '06' => 'juin', '07' => 'juillet', '08' => 'août', + '09' => 'septembre', '10' => 'octobre', '11' => 'novembre', '12' => 'décembre'], + 'de' => ['01' => 'Januar', '02' => 'Februar', '03' => 'März', '04' => 'April', '05' => 'Mai', '06' => 'Juni', '07' => 'Juli', '08' => 'August', + '09' => 'September', '10' => 'Oktober', '11' => 'November', '12' => 'Dezember'], + 'es' => ['01' => 'enero', '02' => 'febrero', '03' => 'marzo', '04' => 'abril', '05' => 'mayo', '06' => 'junio', '07' => 'julio', '08' => 'agosto', + '09' => 'septiembre', '10' => 'octubre', '11' => 'noviembre', '12' => 'diciembre'], + 'it' => ['01' => 'gennaio', '02' => 'febbraio', '03' => 'marzo', '04' => 'aprile', '05' => 'maggio', '06' => 'giugno', '07' => 'luglio', '08' => 'agosto', + '09' => 'settembre', '10' => 'ottobre', '11' => 'novembre', '12' => 'dicembre'], + 'pt' => ['01' => 'janeiro', '02' => 'fevereiro', '03' => 'março', '04' => 'abril', '05' => 'maio', '06' => 'junho', '07' => 'julho', '08' => 'agosto', + '09' => 'setembro', '10' => 'outubro', '11' => 'novembro', '12' => 'dezembro'], + ]; + const LANGUAGE_REWRITE = ['co.uk' => 'en', 'co.za' => 'en', 'at' => 'de']; + + private string $lastId = ''; + private ?string $currentCategory = ''; + + private function getCurrentCategory() + { + if (empty($this->currentCategory)) { + $category = $this->getInput('category'); + $this->currentCategory = empty($category) ? self::PARAMETERS['']['category']['defaultValue'] : $category; + } + return $this->currentCategory; + } + + public function getIcon() + { + return 'https://www.nintendo.co.uk/favicon.ico'; + } + + public function getURI() + { + $category = $this->getInput('category'); + return 'all' === $category ? self::URI : $this->getSourceUrl(); + } + + protected function provideFeedTitle(\DOMXPath $xpath) + { + $category = $this->getInput('category'); + $categoryName = array_search($category, self::PARAMETERS['']['category']['values']); + return 'all' === $category ? self::NAME : $categoryName . ' Software-Updates'; + } + + protected function getSourceUrl() + { + $country = $this->getInput('country'); + $category = $this->getCurrentCategory(); + return str_replace(self::PARAMETERS['']['country']['defaultValue'], $country, self::FEED_SOURCE_URL[$category]); + } + + protected function getExpressionItem() + { + $category = $this->getCurrentCategory(); + return 'sf' === $category ? self::XPATH_EXPRESSION_ITEM_FIRMWARE : self::XPATH_EXPRESSION_ITEM; + } + + protected function getExpressionItemTimestamp() + { + if (empty($this->lastId)) { + return null; + } + $country = $this->getInput('country'); + $category = $this->getCurrentCategory(); + $language = $this->getLanguageFromCountry($country); + return str_replace( + ['{{id_here}}', '{{label_here}}'], + [$this->lastId, static::GAME_COUNTRY_DATE_SUBSTRING_PART[$category][$language]], + static::XPATH_EXPRESSION_ITEM_TIMESTAMP + ); + } + + protected function getExpressionItemCategories() + { + $category = $this->getCurrentCategory(); + $categoryName = array_search($category, self::PARAMETERS['']['category']['values']); + return 'string("' . $categoryName . '")'; + } + + public function collectData() + { + $category = $this->getCurrentCategory(); + if ('all' === $category) { + $allItems = []; + foreach (self::PARAMETERS['']['category']['values'] as $catKey) { + if ('all' === $catKey) { + continue; + } + $this->currentCategory = $catKey; + $this->items = []; + parent::collectData(); + $allItems = [...$allItems, ...$this->items]; + } + $this->currentCategory = 'all'; + $this->items = $allItems; + } else { + parent::collectData(); + } + } + + protected function formatItemTitle($value) + { + if (false !== strpos($value, ' (')) { + $value = substr($value, 0, strpos($value, ' (')); + } + if ('all' === $this->getInput('category')) { + $category = $this->getCurrentCategory(); + $categoryName = array_search($category, self::PARAMETERS['']['category']['values']); + return $categoryName . ' ' . $value; + } + return $value; + } + + protected function formatItemContent($value) + { + $result = preg_match('~
(.*)
~', $value, $matches); + if (1 === $result) { + $this->lastId = $matches[1]; + return trim($matches[2]); + } + return $value; + } + + protected function formatItemTimestamp($value) + { + $country = $this->getInput('country'); + $category = $this->getCurrentCategory(); + $language = $this->getLanguageFromCountry($country); + + $aMonthNames = self::FOREIGN_MONTH_NAMES[$language] ?? null; + if (null !== $aMonthNames) { + $value = str_replace(array_values($aMonthNames), array_keys($aMonthNames), $value); + } + $value = str_replace('­', '-', $value); + $value = str_replace('--', '-', $value); + + $date = \DateTime::createFromFormat(self::GAME_COUNTRY_DATE_FORMAT[$category][$language], $value); + if (false === $date) { + $date = new \DateTime('now'); + } + return $date->getTimestamp(); + } + + protected function generateItemId(FeedItem $item) + { + return $this->getCurrentCategory() . '-' . $this->lastId; + } + + private function getLanguageFromCountry($country) + { + return (strpos($country, '/') !== false) ? substr($country, strpos($country, '/') + 1) : (self::LANGUAGE_REWRITE[$country] ?? $country); + } +} From 8d0ddb579fd4e15ec02ea17a11d4337b19fc2424 Mon Sep 17 00:00:00 2001 From: Evgeny <76707795+itsLameni@users.noreply.github.com> Date: Fri, 3 Nov 2023 16:42:34 +0000 Subject: [PATCH 194/716] Adding rss.m3wz.su instance to list of public hosts (#3789) --- docs/01_General/06_Public_Hosts.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index 9aa292a5..de538cf1 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -20,6 +20,8 @@ | ![](https://iplookup.flagfox.net/images/h16/NL.png) | https://feed.eugenemolotov.ru | ![](https://img.shields.io/website/https/feed.eugenemolotov.ru.svg) | [@em92](https://github.com/em92) | Hosted in Amsterdam, Netherlands | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss-bridge.mediani.de | ![](https://img.shields.io/website/https/rss-bridge.mediani.de.svg) | [@sokai](https://github.com/sokai) | Hosted with Netcup, Germany | | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | +| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud + ## Inactive instances From 84b5ffcc7c6ee42d323bb897f61e523127eb7595 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Tue, 7 Nov 2023 05:02:34 +0100 Subject: [PATCH 195/716] [PepperBridgeAbstract] Fix Deal Origin and Shipping cost (#3790) - Deal Origin was changed by the website : fixed the CSS class to get it - Shipping cost had an extra SVG image in the content : removed the whole HTML tags from the content --- bridges/PepperBridgeAbstract.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 5d2e552b..875ed8f6 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -356,11 +356,11 @@ HEREDOC; if ($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0) != null) { if ($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1) != null) { return '
' . $this->i8n('shipping') . ' : ' - . $deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1)->innertext + . strip_tags($deal->find('span[class*=space--ml-2 size--all-s overflow--wrap-off]', 0)->children(1)->innertext) . '
'; } else { return '
' . $this->i8n('shipping') . ' : ' - . $deal->find('span[class*=text--color-greyShade flex--inline]', 0)->innertext + . strip_tags($deal->find('span[class*=text--color-greyShade flex--inline]', 0)->innertext) . '
'; } } else { @@ -376,7 +376,7 @@ HEREDOC; { if (($origin = $deal->find('button[class*=text--color-greyShade]', 0)) != null) { $path = str_replace(' ', '/', trim(Json::decode($origin->{'data-cloak-link'})['path'])); - $text = $origin->find('span[class*=cept-merchant-name]', 0); + $text = $origin->find('span[class*=link]', 0); return '
' . $this->i8n('origin') . ' : ' . $text . '
'; } else { return ''; From a6310cff1ab77ebb9be4b9aadbf3f6a007f2b09c Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Tue, 7 Nov 2023 21:32:46 +0100 Subject: [PATCH 196/716] [GreatFonBridge] Add new Instagram Viewer Bridge (#3791) Add a new Instagram Bridge not using Cloudflare DDoS Protection --- bridges/GreatFonBridge.php | 140 +++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 bridges/GreatFonBridge.php diff --git a/bridges/GreatFonBridge.php b/bridges/GreatFonBridge.php new file mode 100644 index 00000000..2951634c --- /dev/null +++ b/bridges/GreatFonBridge.php @@ -0,0 +1,140 @@ + [ + 'u' => [ + 'name' => 'username', + 'type' => 'text', + 'title' => 'Instagram username you want to follow', + 'exampleValue' => 'aesoprockwins', + 'required' => true, + ], + ] + ]; + const TEST_DETECT_PARAMETERS = [ + 'https://www.instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], + 'https://instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], + 'https://greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], + 'https://www.greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], + ]; + + public function collectData() + { + $username = $this->getInput('u'); + $html = getSimpleHTMLDOMCached(self::URI . '/v/' . $username); + $html = defaultLinkTo($html, self::URI); + + foreach ($html->find('div[class*=content__item]') as $post) { + // Skip the ads + if (!str_contains($post->class, 'ads')) { + $url = $post->find('a[href^=https://greatfon.com/c/]', 0)->href; + $date = $this->parseDate($post->find('div[class=content__time-text]', 0)->plaintext); + $description = $post->find('img', 0)->alt; + $imageUrl = $post->find('img', 0)->src; + $author = $username; + $uid = $url; + $title = 'Post - ' . $username . ' - ' . $this->descriptionToTitle($description); + + // Checking post type + $isVideo = (bool) $post->find('div[class=content__camera]', 0); + $videoNote = $isVideo ? '

(video)

' : ''; + + $this->items[] = [ + 'uri' => $url, + 'author' => $author, + 'timestamp' => $date, + 'title' => $title, + 'thumbnail' => $imageUrl, + 'enclosures' => [$imageUrl], + 'content' => << + {$description} + +{$videoNote} +

{$description}

+HTML, + 'uid' => $uid + ]; + } + } + } + + private function parseDate($content) + { + // Parse date, and transform the date into a timetamp, even in a case of a relative date + $date = date_create(); + + // Content trimmed to be sure that the "article" is at the beginning of the string and remove "ago" to make it a valid PHP date interval + $dateString = trim(str_replace(' ago', '', $content)); + + // Replace the article "an" or "a" by the number "1" to be a valid PHP date interval + $dateString = preg_replace('/^((an|a) )/m', '1 ', $dateString); + + $relativeDate = date_interval_create_from_date_string($dateString); + if ($relativeDate) { + date_sub($date, $relativeDate); + // As the relative interval has the precision of a day for date older than 24 hours, we can remove the hour of the date, as it is not relevant + date_time_set($date, 0, 0, 0, 0); + } else { + $this->logger->info(sprintf('Unable to parse date string: %s', $dateString)); + } + return date_format($date, 'r'); + } + + public function getURI() + { + if (!is_null($this->getInput('u'))) { + return urljoin(self::URI, '/v/' . $this->getInput('u')); + } + + return parent::getURI(); + } + + public function getIcon() + { + return static::URI . '/images/favicon-hub-3ede543aa6d1225e8dc016ccff6879c8.ico?vsn=d'; + } + + private function descriptionToTitle($description) + { + return strlen($description) > 60 ? mb_substr($description, 0, 57) . '...' : $description; + } + + public function getName() + { + if (!is_null($this->getInput('u'))) { + return 'Username ' . $this->getInput('u') . ' - GreatFon Bridge'; + } + return parent::getName(); + } + + public function detectParameters($url) + { + $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})(\/reels\/|\/tagged\/|\/|)|(www\.|)(greatfon.com)\/v\/([a-zA-Z0-9_\.]{1,30}))/'; + if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'Username'; + // Extract detected domain using the regex + $domain = $matches[8] ?? $matches[4]; + if ($domain == 'greatfon.com') { + $params['u'] = $matches[9]; + return $params; + } elseif ($domain == 'instagram.com') { + $params['u'] = $matches[5]; + return $params; + } else { + return null; + } + } else { + return null; + } + } +} From 7a7fa876d2a07526f0c57610b3d83c261b99e0ed Mon Sep 17 00:00:00 2001 From: wpdevelopment11 <85058595+wpdevelopment11@users.noreply.github.com> Date: Wed, 8 Nov 2023 18:40:24 +0300 Subject: [PATCH 197/716] [VkBridge] Fix regex that extracts page name (#3793) Dot should be allowed in page names. Precise rules for page names are available here: https://vk.com/faq19715 (in Russian) --- bridges/VkBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 0d47692d..60e4315b 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -29,11 +29,12 @@ class VkBridge extends BridgeAbstract 'https://vk.com/groupname/anythingelse' => ['u' => 'groupname'], 'https://vk.com/groupname?w=somethingelse' => ['u' => 'groupname'], 'https://vk.com/with_underscore' => ['u' => 'with_underscore'], + 'https://vk.com/vk.cats' => ['u' => 'vk.cats'], ]; protected $pageName; protected $tz = 0; - private $urlRegex = '/vk\.com\/([\w]+)/'; + private $urlRegex = '/vk\.com\/([\w.]+)/'; public function getURI() { From 57b61c8787b7946c674a06e76699baff94931f7e Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Thu, 9 Nov 2023 10:16:34 +0100 Subject: [PATCH 198/716] [MydealsBridge] Fix keyword seatch (#3794) When no result were found using the keyword search, some random deals were displayed because the "not found" text has been modified : the text is now up to date. Some type in the textual name of the Bridge and texte about the website name was fixed --- bridges/MydealsBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 0ef9c201..de702583 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2,9 +2,9 @@ class MydealsBridge extends PepperBridgeAbstract { - const NAME = 'Mydeals bridge'; + const NAME = 'Mydealz bridge'; const URI = 'https://www.mydealz.de/'; - const DESCRIPTION = 'Zeigt die Deals von mydeals.de'; + const DESCRIPTION = 'Zeigt die Deals von mydealz.de'; const MAINTAINER = 'sysadminstory'; const PARAMETERS = [ 'Suche nach Stichworten' => [ @@ -2023,7 +2023,7 @@ class MydealsBridge extends PepperBridgeAbstract 'uri-deal' => 'deals/', 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', - 'no-results' => 'Ups, wir konnten keine Deals zu', + 'no-results' => 'Ups, wir konnten nichts', 'relative-date-indicator' => [ 'vor', 'seit' From e76b0601b3312d957b6bf6e46ddeb3df6cd70a01 Mon Sep 17 00:00:00 2001 From: SebLaus <97241865+SebLaus@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:55:56 +0100 Subject: [PATCH 199/716] [IdealoBridge] New Bridge to track prices on idealo.de (#3786) * [IdealoBridge] Created Checks the price of a given item on idealo.de. Can create an Alarm Message if a the price is lower than set or an Priceupdate if the price has changed. * Changed Exec and syntax * last fixes for remaining warning --- bridges/IdealoBridge.php | 180 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 bridges/IdealoBridge.php diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php new file mode 100644 index 00000000..89c5f87d --- /dev/null +++ b/bridges/IdealoBridge.php @@ -0,0 +1,180 @@ + [ + 'name' => 'Idealo.de Link to productpage', + 'required' => true, + 'exampleValue' => 'https://www.idealo.de/preisvergleich/OffersOfProduct/202007367_-s7-pro-ultra-roborock.html' + ], + 'ExcludeNew' => [ + 'name' => 'Priceupdate: Do not track new items', + 'type' => 'checkbox', + 'value' => 'c' + ], + 'ExcludeUsed' => [ + 'name' => 'Priceupdate: Do not track used items', + 'type' => 'checkbox', + 'value' => 'uc' + ], + 'MaxPriceNew' => [ + 'name' => 'Pricealarm: Maximum price for new Product', + 'type' => 'number' + ], + 'MaxPriceUsed' => [ + 'name' => 'Pricealarm: Maximum price for used Product', + 'type' => 'number' + ], + ] + ]; + + public function getIcon() + { + return 'https://cdn.idealo.com/storage/ids-assets/ico/favicon.ico'; + } + + public function collectData() + { + $link = $this->getInput('Link'); + $html = getSimpleHTMLDOM($link); + + // Get Productname + $titleobj = $html->find('.oopStage-title', 0); + $Productname = $titleobj->find('span', 0)->plaintext; + + // Create product specific Cache Keys with the link + $KeyNEW = $link; + $KeyNEW .= 'NEW'; + + $KeyUSED = $link; + $KeyUSED .= 'USED'; + + // Load previous Price + $OldPriceNew = $this->loadCacheValue($KeyNEW); + $OldPriceUsed = $this->loadCacheValue($KeyUSED); + + // First button is new. Found at oopStage-conditionButton-wrapper-text class (.) + $FirstButton = $html->find('.oopStage-conditionButton-wrapper-text', 0); + if ($FirstButton) { + $PriceNew = $FirstButton->find('strong', 0)->plaintext; + } + + // Second Button is used + $SecondButton = $html->find('.oopStage-conditionButton-wrapper-text', 1); + if ($SecondButton) { + $PriceUsed = $SecondButton->find('strong', 0)->plaintext; + } + + // Only continue if a price has changed + if ($PriceNew != $OldPriceNew || $PriceUsed != $OldPriceUsed) { + // Get Product Image + $image = $html->find('.datasheet-cover-image', 0)->src; + + // Generate Content + if ($PriceNew > 1) { + $content = "

Price New:
$PriceNew

"; + $content .= "

Price Newbefore:
$OldPriceNew

"; + } + + if ($this->getInput('MaxPriceNew') != '') { + $content .= sprintf('

Max Price Used:
%s,00 €

', $this->getInput('MaxPriceNew')); + } + + if ($PriceUsed > 1) { + $content .= "

Price Used:
$PriceUsed

"; + $content .= "

Price Used before:
$OldPriceUsed

"; + } + + if ($this->getInput('MaxPriceUsed') != '') { + $content .= sprintf('

Max Price Used:
%s,00 €

', $this->getInput('MaxPriceUsed')); + } + + $content .= ""; + + + $now = date('d.m.j H:m'); + + $Pricealarm = 'Pricealarm %s: %s %s %s'; + + // Currently under Max new price + if ($this->getInput('MaxPriceNew') != '') { + if ($PriceNew < $this->getInput('MaxPriceNew')) { + $title = sprintf($Pricealarm, 'Used', $PriceNew, $Productname, $now); + $item = [ + 'title' => $title, + 'uri' => $link, + 'content' => $content, + 'uid' => md5($title) + ]; + $this->items[] = $item; + } + } + + // Currently under Max used price + if ($this->getInput('MaxPriceUsed') != '') { + if ($PriceUsed < $this->getInput('MaxPriceUsed')) { + $title = sprintf($Pricealarm, 'Used', $PriceUsed, $Productname, $now); + $item = [ + 'title' => $title, + 'uri' => $link, + 'content' => $content, + 'uid' => md5($title) + ]; + $this->items[] = $item; + } + } + + // General Priceupdate + if ($this->getInput('MaxPriceUsed') == '' && $this->getInput('MaxPriceNew') == '') { + // check if a relevant pricechange happened + if ( + (!$this->getInput('ExcludeNew') && $PriceNew != $OldPriceNew ) || + (!$this->getInput('ExcludeUsed') && $PriceUsed != $OldPriceUsed ) + ) { + $title .= 'Priceupdate! '; + + if (!$this->getInput('ExcludeNew')) { + if ($PriceNew < $OldPriceNew) { + $title .= 'NEW:⬇ '; // Arrow Down Emoji + } + if ($PriceNew > $OldPriceNew) { + $title .= 'NEW:⬆ '; // Arrow Up Emoji + } + } + + + if (!$this->getInput('ExcludeUsed')) { + if ($PriceUsed < $OldPriceUsed) { + $title .= 'USED:⬇ '; // Arrow Down Emoji + } + if ($PriceUsed > $OldPriceUsed) { + $title .= 'USED:⬆ '; // Arrow Up Emoji + } + } + $title .= $Productname; + $title .= ' '; + $title .= $now; + + $item = [ + 'title' => $title, + 'uri' => $link, + 'content' => $content, + 'uid' => md5($title) + ]; + $this->items[] = $item; + } + } + } + + // Save current price + $this->saveCacheValue($KeyNEW, $PriceNew); + $this->saveCacheValue($KeyUSED, $PriceUsed); + } +} From b347a9268a7d6bc34442509e2cf7dcd793c39853 Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 10 Nov 2023 12:56:11 +0100 Subject: [PATCH 200/716] feat: new bridge MangaReader (#3795) --- bridges/MangaReaderBridge.php | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 bridges/MangaReaderBridge.php diff --git a/bridges/MangaReaderBridge.php b/bridges/MangaReaderBridge.php new file mode 100644 index 00000000..1fa0c62d --- /dev/null +++ b/bridges/MangaReaderBridge.php @@ -0,0 +1,44 @@ + [ + 'name' => 'Manga URL', + 'type' => 'text', + 'required' => true, + 'title' => 'The URL of the manga on MangaReader', + 'pattern' => '^https:\/\/mangareader\.to\/[^\/]+$', + 'exampleValue' => 'https://mangareader.to/bleach-1623', + ], + 'lang' => [ + 'name' => 'Chapter Language', + 'title' => 'two-letter language code (example "en", "jp", "fr")', + 'exampleValue' => 'en', + 'required' => true, + 'pattern' => '^[a-z][a-z]$', + ] + ] + ]; + + public function collectData() + { + $url = $this->getInput('url'); + $lang = $this->getInput('lang'); + $dom = getSimpleHTMLDOM($url); + $chapters = $dom->getElementById($lang . '-chapters'); + + foreach ($chapters->getElementsByTagName('li') as $chapter) { + $a = $chapter->getElementsByTagName('a')[0]; + $item = []; + $item['title'] = $a->getAttribute('title'); + $item['uri'] = self::URI . $a->getAttribute('href'); + $this->items[] = $item; + } + } +} From 4919c53c108a36a7969b027bc6e6653415096353 Mon Sep 17 00:00:00 2001 From: knrdl <35548889+knrdl@users.noreply.github.com> Date: Mon, 13 Nov 2023 00:11:19 +0100 Subject: [PATCH 201/716] [DemosBerlinBridge] add bridge (#3800) --- bridges/DemosBerlinBridge.php | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 bridges/DemosBerlinBridge.php diff --git a/bridges/DemosBerlinBridge.php b/bridges/DemosBerlinBridge.php new file mode 100644 index 00000000..05fd2335 --- /dev/null +++ b/bridges/DemosBerlinBridge.php @@ -0,0 +1,62 @@ + [ + 'name' => 'Tage', + 'type' => 'number', + 'title' => 'Einträge für die nächsten Tage zurückgeben', + 'required' => true, + 'defaultValue' => 7, + ] + ]]; + + public function getIcon() + { + return 'https://www.berlin.de/i9f/r1/images/favicon/favicon.ico'; + } + + public function collectData() + { + $json = getContents('https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/index/all.json'); + $jsonFile = json_decode($json, true); + + $daysInterval = DateInterval::createFromDateString($this->getInput('days') . ' day'); + $maxTargetDate = date_add(new DateTime('now'), $daysInterval); + + foreach ($jsonFile['index'] as $entry) { + $entryDay = implode('-', array_reverse(explode('.', $entry['datum']))); // dd.mm.yyyy to yyyy-mm-dd + $ts = (new DateTime())->setTimestamp(strtotime($entryDay)); + if ($ts <= $maxTargetDate) { + $item = []; + $item['uri'] = 'https://www.berlin.de/polizei/service/versammlungsbehoerde/versammlungen-aufzuege/index.php/detail/' . $entry['id']; + $item['timestamp'] = $entryDay . ' ' . $entry['von']; + $item['title'] = $entry['thema']; + $location = $entry['strasse_nr'] . ' ' . $entry['plz']; + $locationQuery = http_build_query(['query' => $location]); + $item['content'] = <<{$entry['thema']} +

📅

+ + 📍 {$location} + +

{$entry['aufzugsstrecke']}

+ HTML; + $item['uid'] = $this->getSanitizedHash($entry['datum'] . '-' . $entry['von'] . '-' . $entry['bis'] . '-' . $entry['thema']); + + $this->items[] = $item; + } + } + } + + private function getSanitizedHash($string) + { + return hash('sha1', preg_replace('/[^a-zA-Z0-9]/', '', strtolower($string))); + } +} From ef711cb30b026b581573c5dc5f4de88b47f105b9 Mon Sep 17 00:00:00 2001 From: knrdl <35548889+knrdl@users.noreply.github.com> Date: Mon, 13 Nov 2023 00:12:39 +0100 Subject: [PATCH 202/716] [KleinanzeigenBridge] add new bridge (#3798) * [KleinanzeigenBridge] add new bridge * [KleinanzeigenBridge] fix missing timestamp * [KleinanzeigenBridge] linting * [KleinanzeigenBridge] fix end of list detection --- bridges/KleinanzeigenBridge.php | 150 ++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 bridges/KleinanzeigenBridge.php diff --git a/bridges/KleinanzeigenBridge.php b/bridges/KleinanzeigenBridge.php new file mode 100644 index 00000000..e0535b59 --- /dev/null +++ b/bridges/KleinanzeigenBridge.php @@ -0,0 +1,150 @@ + [ + 'query' => [ + 'name' => 'query', + 'required' => false, + 'title' => 'query term', + ], + 'location' => [ + 'name' => 'location', + 'required' => false, + 'title' => 'e.g. Berlin', + ], + 'radius' => [ + 'name' => 'radius', + 'required' => false, + 'type' => 'number', + 'title' => 'search radius in kilometers', + 'defaultValue' => 10, + ], + 'pages' => [ + 'name' => 'pages', + 'required' => true, + 'type' => 'number', + 'title' => 'how many pages to fetch', + 'defaultValue' => 2, + ] + ], + 'By profile' => [ + 'userid' => [ + 'name' => 'user id', + 'required' => true, + 'type' => 'number', + 'exampleValue' => 12345678 + ], + 'pages' => [ + 'name' => 'pages', + 'required' => true, + 'type' => 'number', + 'title' => 'how many pages to fetch', + 'defaultValue' => 2, + ] + ], + ]; + + public function getIcon() + { + return 'https://www.kleinanzeigen.de/favicon.ico'; + } + + public function getName() + { + switch ($this->queriedContext) { + case 'By profile': + return 'Kleinanzeigen Profil'; + case 'By search': + return 'Kleinanzeigen ' . $this->getInput('query') . ' / ' . $this->getInput('location'); + default: + return parent::getName(); + } + } + + public function collectData() + { + if ($this->queriedContext === 'By profile') { + for ($i = 1; $i <= $this->getInput('pages'); $i++) { + $html = getSimpleHTMLDOM(self::URI . '/s-bestandsliste.html?userId=' . $this->getInput('userid') . '&pageNum=' . $i . '&sortingField=SORTING_DATE'); + + $foundItem = false; + foreach ($html->find('article.aditem') as $element) { + $this->addItem($element); + $foundItem = true; + } + if (!$foundItem) { + break; + } + } + } + + if ($this->queriedContext === 'By search') { + $locationID = ''; + if ($this->getInput('location')) { + $json = getContents(self::URI . '/s-ort-empfehlungen.json?' . http_build_query(['query' => $this->getInput('location')])); + $jsonFile = json_decode($json, true); + $locationID = str_replace('_', '', array_key_first($jsonFile)); + } + for ($i = 1; $i <= $this->getInput('pages'); $i++) { + $searchUrl = self::URI . '/s-walled-garden/'; + if ($i != 1) { + $searchUrl .= 'seite:' . $i . '/'; + } + if ($this->getInput('query')) { + $searchUrl .= urlencode($this->getInput('query')) . '/k0'; + } + if ($locationID) { + $searchUrl .= 'l' . $locationID; + } + if ($this->getInput('radius')) { + $searchUrl .= 'r' . $this->getInput('radius'); + } + + $html = getSimpleHTMLDOM($searchUrl); + + // end of list if returned page is not the expected one + if ($html->find('.pagination-current', 0)->plaintext != $i) { + break; + } + + foreach ($html->find('ul#srchrslt-adtable article.aditem') as $element) { + $this->addItem($element); + } + } + } + } + + private function addItem($element) + { + $item = []; + + $item['uid'] = $element->getAttribute('data-adid'); + $item['uri'] = self::URI . $element->getAttribute('data-href'); + + $item['title'] = $element->find('h2', 0)->plaintext; + $item['timestamp'] = $element->find('div.aditem-main--top--right', 0)->plaintext; + $imgUrl = str_replace( + 'rule=$_2.JPG', + 'rule=$_57.JPG', + str_replace( + 'rule=$_35.JPG', + 'rule=$_57.JPG', + $element->find('img', 0) ? $element->find('img', 0)->getAttribute('src') : '' + ) + ); //enhance img quality + $textContainer = $element->find('div.aditem-main', 0); + $textContainer->find('a', 0)->href = self::URI . $textContainer->find('a', 0)->href; // add domain to url + $item['content'] = '' . + $textContainer->outertext; + + $this->items[] = $item; + } +} From 2b741b1c1bf3ef352cea0903bd1561bc00ff45b0 Mon Sep 17 00:00:00 2001 From: joaomqc Date: Wed, 15 Nov 2023 15:26:25 +0000 Subject: [PATCH 203/716] [SongkickBridge] add new bridge (#3803) * [SongkickBridge] add new bridge * [SongkickBridge] fix var reference and outdoor category * [SongkickBridge] remove unnecessary string concat * [SongkickBridge] fix if clause formatting * [SongkickBridge] fix formatting and event title --- bridges/SongkickBridge.php | 92 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 bridges/SongkickBridge.php diff --git a/bridges/SongkickBridge.php b/bridges/SongkickBridge.php new file mode 100644 index 00000000..bfe29865 --- /dev/null +++ b/bridges/SongkickBridge.php @@ -0,0 +1,92 @@ + [ + 'name' => 'Artist ID', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '2506696-imagine-dragons', + ] + ] ]; + + const ARTIST_URI = 'https://www.songkick.com/artists/%s/'; + const CALENDAR_URI = self::ARTIST_URI . 'calendar'; + + private $name = ''; + + public function getURI() + { + return sprintf(self::ARTIST_URI, $this->getInput('artistid')); + } + + public function getName() + { + if (!empty($this->name)) { + return $this->name . ' - ' . parent::getName(); + } + return parent::getName(); + } + + public function getIcon() + { + return 'https://assets.sk-static.com/images/nw/furniture/songkick-logo.svg'; + } + + public function collectData() + { + $url = sprintf(self::CALENDAR_URI, $this->getInput('artistid')); + + $dom = getSimpleHTMLDOM($url); + + $jsonscript = $dom->find('div.microformat > script', 0); + + if (empty($this->name) && $jsonscript) { + $this->name = json_decode($jsonscript->innertext)[0]->name; + } + + $dom = $dom->find('div.container > div.row > div.primary', 0); + + if (!$dom) { + throw new Exception(sprintf('Unable to find css selector on `%s`', $url)); + } + $dom = defaultLinkTo($dom, $this->getURI()); + + foreach ($dom->find('div[@id="calendar-summary"] > ol > li') as $article) { + $detailsobj = json_decode($article->find('div.microformat > script', 0)->innertext)[0]; + + $a = $article->find('a', 0); + + $details = $a->find('div.event-details', 0); + $title = $details->find('.secondary-detail', 0)->plaintext; + $city = $details->find('.primary-detail', 0)->plaintext; + $event = $detailsobj->location->name; + + $content = 'City: ' . $city . '
Event: ' . $event . '
Date: ' . $article->title; + + $categories = []; + if ($details->hasClass('concert')) { + $categories[] = 'concert'; + } + if ($details->hasClass('festival')) { + $categories[] = 'festival'; + } + if (!is_null($details->find('.outdoor', 0))) { + $categories[] = 'outdoor'; + } + + $this->items[] = [ + 'title' => $title, + 'uri' => $a->href, + 'content' => $content, + 'categories' => $categories, + ]; + } + } +} From b037d1b4d1f0b0f422e21125ddef00a58e185ed1 Mon Sep 17 00:00:00 2001 From: Matt DeMoss Date: Tue, 21 Nov 2023 11:00:02 -0500 Subject: [PATCH 204/716] [Threads] add bridge (#3805) * initial working Threads bridge * properly specify a default limit * phpcs formatted --- bridges/ThreadsBridge.php | 120 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 bridges/ThreadsBridge.php diff --git a/bridges/ThreadsBridge.php b/bridges/ThreadsBridge.php new file mode 100644 index 00000000..b7e5cd1a --- /dev/null +++ b/bridges/ThreadsBridge.php @@ -0,0 +1,120 @@ + [ + 'u' => [ + 'name' => 'username', + 'required' => true, + 'exampleValue' => 'zuck', + 'title' => 'Insert a user name' + ], + 'limit' => [ + 'name' => 'Limit', + 'type' => 'number', + 'required' => false, + 'title' => 'Specify number of posts to fetch', + 'defaultValue' => 5 + ] + ] + ]; + + protected $feedName = self::NAME; + public function getName() + { + return $this->feedName; + } + + public function detectParameters($url) + { + // By username + $regex = '/^(https?:\/\/)?(www\.)?threads\.net\/(@)?([^\/?\n]+)/'; + if (preg_match($regex, $url, $matches) > 0) { + $params['context'] = 'By username'; + $params['u'] = urldecode($matches[3]); + return $params; + } + return null; + } + + public function getURI() + { + return self::URI . '@' . $this->getInput('u'); + } + + // https://stackoverflow.com/a/3975706/421140 + // Found this in FlaschenpostBridge, modified to return an array and take an object. + private function recursiveFind($haystack, $needle) + { + $found = []; + $iterator = new \RecursiveArrayIterator($haystack); + $recursive = new \RecursiveIteratorIterator( + $iterator, + \RecursiveIteratorIterator::SELF_FIRST + ); + foreach ($recursive as $key => $value) { + if ($key === $needle) { + $found[] = $value; + } + } + return $found; + } + + public function collectData() + { + $html = getSimpleHTMLDOMCached($this->getURI(), static::CACHE_TIMEOUT); + Debug::log(sprintf('Fetched: %s', $this->getURI())); + $jsonBlobs = $html->find('script[type="application/json"]'); + Debug::log(sprintf('%d JSON blobs found.', count($jsonBlobs))); + $gatheredCodes = []; + $limit = $this->getInput('limit'); + foreach ($jsonBlobs as $jsonBlob) { + // The structure of the JSON document is likely to change, but we're looking for a "code" inside a "post" + foreach ($this->recursiveFind($this->recursiveFind(json_decode($jsonBlob->innertext), 'post'), 'code') as $candidateCode) { + // code should be like CzZk4-USq1O or Cy3m1VnRiwP or Cywjyrdv9T6 or CzZk4-USq1O + if (grapheme_strlen($candidateCode) == 11 and !in_array($candidateCode, $gatheredCodes)) { + $gatheredCodes[] = $candidateCode; + if (count($gatheredCodes) >= $limit) { + break 2; + } + } + } + } + Debug::log(sprintf('Candidate codes found in JSON in script tags: %s', print_r($gatheredCodes, true))); + + $this->feedName = html_entity_decode($html->find('meta[property=og:title]', 0)->content); + // todo: meta[property=og:description] could populate the feed description + + foreach ($gatheredCodes as $postCode) { + $item = []; + // post URL is like: https://www.threads.net/@zuck/post/Czrr520PZfh + $item['uri'] = $this->getURI() . '/post/' . $postCode; + $articleHtml = getSimpleHTMLDOMCached($item['uri'], 15778800); // cache time: six months + + // Relying on meta tags ought to be more reliable. + if ($articleHtml->find('meta[property=og:type]', 0)->content != 'article') { + continue; + } + $item['title'] = $articleHtml->find('meta[property=og:description]', 0)->content; + $item['content'] = $articleHtml->find('meta[property=og:description]', 0)->content; + $item['author'] = html_entity_decode($articleHtml->find('meta[property=og:title]', 0)->content); + + $imageUrl = $articleHtml->find('meta[property=og:image]', 0); + if ($imageUrl) { + $item['enclosures'][] = html_entity_decode($imageUrl->content); + } + + // todo: parse hashtags out of content for $item['categories'] + // todo: try to scrape out a timestamp for $item['timestamp'], it's not in the meta tags + + $this->items[] = $item; + } + } +} From 609eed1791598a798e7f94ac694d014605ec11ee Mon Sep 17 00:00:00 2001 From: George Sokianos Date: Tue, 28 Nov 2023 21:54:39 +0000 Subject: [PATCH 205/716] KoFiBridge fix the "Call to a member function find() on null" line 39 (#3807) --- bridges/KoFiBridge.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bridges/KoFiBridge.php b/bridges/KoFiBridge.php index c1600590..da8f1e7d 100644 --- a/bridges/KoFiBridge.php +++ b/bridges/KoFiBridge.php @@ -27,12 +27,15 @@ class KoFiBridge extends BridgeAbstract if (isset($titleWrapper[0])) { $item = []; $item['title'] = $element->find('div.content-link-text div')[0]->plaintext; - // $item['timestamp'] = strtotime($element->find('div.feeditem-time', 0)->plaintext); - $item['uri'] = self::URI . $element->find('div.fi-post-item-large a')[0]->href; + $uri = $element->find('div.content-link-text div')[2]->find('a')[0]->onclick; + $uri = trim(str_replace('window.location =', '', $uri)); + $uri = trim(str_replace(''', '', $uri)); + $uri = trim(str_replace(';', '', $uri)); + $item['uri'] = self::URI . $uri; + if (isset($element->find('div.fi-post-item-large div.content-link-post img')[0])) { $item['enclosures'][] = $element->find('div.fi-post-item-large div.content-link-post img')[0]->src; } - // $item['content'] = $element->find('div.content-link-text div#content-link', 0)->plaintext; $html = getSimpleHTMLDOM($item['uri']); $feedItemTime = $html->find('div.feeditem-time', 0); From ccc20849ffe297e09e046a086734c2c90e94420a Mon Sep 17 00:00:00 2001 From: Michael Bemmerl Date: Thu, 30 Nov 2023 16:52:51 +0000 Subject: [PATCH 206/716] [SchweinfurtBuergerinformationenBridge] Don't include images with data URIs as enclosures. (#3811) See also setEnclosures() in FeedItem.php: URIs with a path are required. --- bridges/SchweinfurtBuergerinformationenBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/SchweinfurtBuergerinformationenBridge.php b/bridges/SchweinfurtBuergerinformationenBridge.php index 349a9d8a..d1f5db15 100644 --- a/bridges/SchweinfurtBuergerinformationenBridge.php +++ b/bridges/SchweinfurtBuergerinformationenBridge.php @@ -107,9 +107,9 @@ class SchweinfurtBuergerinformationenBridge extends BridgeAbstract ]; // Let's see if there are images in the content, and if yes, attach - // them as enclosures, but not images which are used for linking to an external site. + // them as enclosures, but not images which are used for linking to an external site and data URIs. foreach ($images as $image) { - if ($image->class != 'imgextlink') { + if ($image->class != 'imgextlink' && parse_url($image->src, PHP_URL_SCHEME) != 'data') { $item['enclosures'][] = $image->src; } } From 44ff2f2cf8cc4403c7fd2af182607c59c74e14ba Mon Sep 17 00:00:00 2001 From: Niehztog Date: Thu, 30 Nov 2023 17:53:47 +0100 Subject: [PATCH 207/716] adds Super Mario Bros. Wonder to NintendoBridge (#3810) --- bridges/NintendoBridge.php | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/bridges/NintendoBridge.php b/bridges/NintendoBridge.php index 34550737..1f463e91 100644 --- a/bridges/NintendoBridge.php +++ b/bridges/NintendoBridge.php @@ -18,6 +18,7 @@ class NintendoBridge extends XPathAbstract 'Splatoon 2' => 's2', 'Super Mario 3D All-Stars' => 'sm3as', 'Super Mario 3D World + Bowser’s Fury' => 'sm3wbf', + 'Super Mario Bros. Wonder' => 'smbw', 'Super Mario Maker 2' => 'smm2', 'Super Mario Odyssey' => 'smo', 'Super Smash Bros. Ultimate' => 'ssbu', @@ -62,6 +63,7 @@ class NintendoBridge extends XPathAbstract 's2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Splatoon-2-1482897.html', 'sm3as' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-All-Stars-1844226.html', 'sm3wbf' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-3D-World-Bowser-s-Fury-1920668.html', + 'smbw' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Bros-Wonder-2485410.html', 'smm2' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Maker-2-1586745.html', 'smo' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Mario-Odyssey-1482901.html', 'ssbu' => 'https://www.nintendo.co.uk/Support/Nintendo-Switch/Game-Updates/How-to-Update-Super-Smash-Bros-Ultimate-1484130.html', @@ -73,7 +75,7 @@ class NintendoBridge extends XPathAbstract ]; const XPATH_EXPRESSION_ITEM = '//div[@class="col-xs-12 content"]/div[starts-with(@id,"v") and @class="collapse"]'; const XPATH_EXPRESSION_ITEM_FIRMWARE = '//div[@id="latest" and @class="collapse" and @rel="1"]'; - const XPATH_EXPRESSION_ITEM_TITLE = './/h2[1]/node()'; + const XPATH_EXPRESSION_ITEM_TITLE = '(.//h2[1] | .//strong[1])[1]/node()'; const XPATH_EXPRESSION_ITEM_CONTENT = '.'; const XPATH_EXPRESSION_ITEM_URI = '//link[@rel="canonical"]/@href'; @@ -124,6 +126,15 @@ class NintendoBridge extends XPathAbstract 'pt' => 'ançada no dia ', 'en' => 'eleased ', ], + 'smbw' => [ + 'de' => 'eröffentlicht am ', + 'es' => 'isponible desde el ', + 'fr' => 'atée du ', + 'it' => 'istribuita il ', + 'nl' => 'itgebracht op ', + 'pt' => 'ançada a ', + 'en' => 'eleased ', + ], 'smm2' => [ 'de' => 'eröffentlicht am ', 'es' => 'isponible desde el ', @@ -235,6 +246,15 @@ class NintendoBridge extends XPathAbstract 'pt' => 'd/m/y', 'en' => 'F j, Y', ], + 'smbw' => [ + 'de' => 'd. m Y', + 'es' => 'j \d\e m \d\e Y', + 'fr' => 'd/m/Y', + 'it' => 'j m Y', + 'nl' => 'd m Y', + 'pt' => 'j \d\e m \d\e Y', + 'en' => 'j F Y', + ], 'smm2' => [ 'de' => 'd.m.Y', 'es' => 'd-m-Y', From 206edaedf5397aee35848002b3274417007a86c1 Mon Sep 17 00:00:00 2001 From: Nick McCarthy Date: Fri, 1 Dec 2023 21:36:26 +0000 Subject: [PATCH 208/716] [GoogleScholarBridge] Minor patch (#3814) * Do not add RSS entry if Check for updates is found in the article title - avoids repeat entries --- bridges/GoogleScholarBridge.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bridges/GoogleScholarBridge.php b/bridges/GoogleScholarBridge.php index 981355dd..11dc123b 100644 --- a/bridges/GoogleScholarBridge.php +++ b/bridges/GoogleScholarBridge.php @@ -2,7 +2,7 @@ class GoogleScholarBridge extends BridgeAbstract { - const NAME = 'Google Scholar v2'; + const NAME = 'Google Scholar'; const URI = 'https://scholar.google.com/'; const DESCRIPTION = 'Search for publications or follow authors on Google Scholar.'; const MAINTAINER = 'nicholasmccarthy'; @@ -193,6 +193,11 @@ class GoogleScholarBridge extends BridgeAbstract $articleUrl = $articleTitleElement->find('a', 0)->href; $articleTitle = $articleTitleElement->plaintext; + // Break the loop if 'Check for Updates' is found in the article title + if (strpos($articleTitle, 'Check for updates') !== false) { + break; + } + $articleDateElement = $publication->find('div[class="gs_a"]', 0); $articleDate = $articleDateElement ? $articleDateElement->plaintext : ''; From f3df283c4d93a90f81bda7c466e8b9937f178acf Mon Sep 17 00:00:00 2001 From: Eugene Molotov Date: Sun, 3 Dec 2023 22:54:23 +0500 Subject: [PATCH 209/716] [VkBridge] Fix single photo duplication (#3816) --- bridges/VkBridge.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 60e4315b..503bc4d0 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -315,6 +315,13 @@ class VkBridge extends BridgeAbstract $copy_quote->outertext = "
Reposted ($copy_quote_author):
$copy_quote_content"; } + foreach ($post->find('.PrimaryAttachment .PhotoPrimaryAttachment') as $pa) { + $img = $pa->find('.PhotoPrimaryAttachment__imageElement', 0); + if (is_object($img)) { + $pa->outertext = $img->outertext; + } + } + foreach ($post->find('.SecondaryAttachment') as $sa) { $sa_href = $sa->getAttribute('href'); if (!$sa_href) { From deb9a7269e166c74dfd9140da3da74923bb67608 Mon Sep 17 00:00:00 2001 From: knrdl <35548889+knrdl@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:07:22 +0100 Subject: [PATCH 210/716] [MotatosBridge] add bridge (#3799) * [MotatosBridge] add bridge * [MotatosBridge] fix uid as string * [MotatosBridge] add support for all regions * [MotatosBridge] fix: region: "required" attribute not supported for list --- bridges/MotatosBridge.php | 102 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 bridges/MotatosBridge.php diff --git a/bridges/MotatosBridge.php b/bridges/MotatosBridge.php new file mode 100644 index 00000000..6833521a --- /dev/null +++ b/bridges/MotatosBridge.php @@ -0,0 +1,102 @@ + [ + 'name' => 'Region', + 'type' => 'list', + 'title' => 'Choose country', + 'values' => [ + 'Austria' => 'at', + 'Denmark' => 'dk', + 'Finland' => 'fi', + 'Germany' => 'de', + 'Sweden' => 'se', + ], + ], + ]]; + + public function getName() + { + switch ($this->getInput('region')) { + case 'at': + return 'Motatos'; + case 'dk': + return 'Motatos'; + case 'de': + return 'Motatos'; + case 'fi': + return 'Matsmart'; + case 'se': + return 'Matsmart'; + default: + return self::NAME; + } + } + + public function getURI() + { + switch ($this->getInput('region')) { + case 'at': + return 'https://www.motatos.at/neu-im-shop'; + case 'dk': + return 'https://www.motatos.dk/nye-varer'; + case 'de': + return 'https://www.motatos.de/neu-im-shop'; + case 'fi': + return 'https://www.matsmart.fi/uusimmat'; + case 'se': + return 'https://www.matsmart.se/nyinkommet'; + default: + return self::URI; + } + } + + public function getIcon() + { + return 'https://www.motatos.de/favicon.ico'; + } + + private function getApiUrl() + { + switch ($this->getInput('region')) { + case 'at': + return 'https://api.findify.io/v4/4359f7b3-17e0-4f74-9fdb-e6606dfed25c/smart-collection/new-arrivals'; + case 'dk': + return 'https://api.findify.io/v4/3709426e-621a-49df-bd61-ac8543452022/smart-collection/new-arrivals'; + case 'de': + return 'https://api.findify.io/v4/2a044754-6cda-4541-b159-39133b75386c/smart-collection/new-arrivals'; + case 'fi': + return 'https://api.findify.io/v4/63946f89-2a82-4839-a412-883b79144f7b/smart-collection/new-arrivals'; + case 'se': + return 'https://api.findify.io/v4/3ae86b36-a1bd-4442-a3d9-2af6845908e6/smart-collection/new-arrivals'; + } + } + + public function collectData() + { + // motatos uses this api to dynamically load more items on page scroll + $json = getContents($this->getApiUrl() . '?t_client=0&user={%22uid%22:%220%22,%22sid%22:%220%22}'); + $jsonFile = json_decode($json, true); + + foreach ($jsonFile['items'] as $entry) { + $item = []; + $item['uid'] = $entry['custom_fields']['uuid'][0]; + $item['uri'] = $entry['product_url']; + $item['timestamp'] = $entry['created_at'] / 1000; + $item['title'] = $entry['title']; + $item['content'] = <<{$entry['title']} + +

{$entry['price'][0]}€

+ HTML; + $this->items[] = $item; + } + } +} From c3d93835239fe6b5d70033340d450cd9d88c477f Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 8 Dec 2023 06:24:43 +0100 Subject: [PATCH 211/716] [FindfeedAction.php] Use relative URL in Feed Link (#3820) The FindFeed action used absolute URL. This breaks the usage of RSS Bridge behind a reverse proxy in a container. Fixes #3801 for the Find Feed action --- actions/FindfeedAction.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/FindfeedAction.php b/actions/FindfeedAction.php index 25fe4714..fe5ceef9 100644 --- a/actions/FindfeedAction.php +++ b/actions/FindfeedAction.php @@ -56,7 +56,7 @@ class FindfeedAction implements ActionInterface $bridgeParams['bridge'] = $bridgeClassName; $bridgeParams['format'] = $format; $content = [ - 'url' => get_home_page_url() . '?action=display&' . http_build_query($bridgeParams), + 'url' => './?action=display&' . http_build_query($bridgeParams), 'bridgeParams' => $bridgeParams, 'bridgeData' => $bridgeData, 'bridgeMeta' => [ From 3ef0226a087fadfa565b70b0868fc988f927cfac Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 8 Dec 2023 06:25:39 +0100 Subject: [PATCH 212/716] [PepperBridgeAbstract] Fix Detection of "no deals found" and more (#3821) - CSS styles showing there were no deals found has changed : CSS class was updated - Relative Date handling : the minimum granularity of a relative date is the minute on the site. Seconds are therefore meaningless, and are now deleted. MydealsBridge was missing one relateve date prefix : now every date is parsed (I hope so !) --- bridges/MydealsBridge.php | 4 +++- bridges/PepperBridgeAbstract.php | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index de702583..22b46413 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2068,7 +2068,9 @@ class MydealsBridge extends PepperBridgeAbstract 'relative-date-alt-prefixes' => [ 'aktualisiert vor ', 'kommentiert vor ', - 'heiß seit ' + 'eingestellt vor ', + 'heiß seit ', + 'vor ' ], 'relative-date-ignore-suffix' => [ '/von.*$/' diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 875ed8f6..8e8a2e8d 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -94,7 +94,7 @@ class PepperBridgeAbstract extends BridgeAbstract ); // If there is no results, we don't parse the content because it display some random deals - $noresult = $html->find('h3[class=size--all-l]', 0); + $noresult = $html->find('h3[class*=text--b]', 0); if ($noresult != null && strpos($noresult->plaintext, $this->i8n('no-results')) !== false) { $this->items = []; } else { @@ -542,6 +542,10 @@ HEREDOC; { $date = new DateTime(); + // The minimal amount of time substracted is a minute : the seconds in the resulting date would be related to the execution time of the script. + // This make no sense, so we set the seconds manually to "00". + $date->setTime($date->format('H'), $date->format('i'), 0); + // In case of update date, replace it by the regular relative date first word $str = str_replace($this->i8n('relative-date-alt-prefixes'), $this->i8n('local-time-relative')[0], $str); @@ -559,6 +563,8 @@ HEREDOC; '' ]; $date->modify(str_replace($search, $replace, $str)); + + return $date->getTimestamp(); } From 4a398a5b14d6f5c2bb12bf0156348798f4e4fa37 Mon Sep 17 00:00:00 2001 From: Raymond Berger Date: Sat, 9 Dec 2023 11:52:57 +0100 Subject: [PATCH 213/716] Update FacebookBridge.md - not working (#3823) --- docs/10_Bridge_Specific/FacebookBridge.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/10_Bridge_Specific/FacebookBridge.md b/docs/10_Bridge_Specific/FacebookBridge.md index c2a1fd0e..f24f8aa8 100644 --- a/docs/10_Bridge_Specific/FacebookBridge.md +++ b/docs/10_Bridge_Specific/FacebookBridge.md @@ -1,18 +1,18 @@ FacebookBridge =============== -Resume of the actual state of this bridge: +State of this bridge: +- Facebook Groups (and probably other sections too) do not work at all - No maintainer -- Need Cookies consent -- New design architecture deployed +- Needs cookie consent support for public pages +- Needs login support (see [this example]([url](https://github.com/RSS-Bridge/rss-bridge/issues/1891)) for Instagram) for private groups -Due [facebook-redesing](https://engineering.fb.com/2020/05/08/web/facebook-redesign/) +Due to the 2020 [Facebook redesign](https://engineering.fb.com/2020/05/08/web/facebook-redesign/) and the requirement to [accept cookies](https://www.facebook.com/business/help/348535683460989) -users start getting [Problems with Facebook on public RSS-Bridge instances](https://github.com/RSS-Bridge/rss-bridge/issues/2047 ) +users are getting [problems with Facebook on public RSS-Bridge instances](https://github.com/RSS-Bridge/rss-bridge/issues/2047). +Relevant Info +-------------- -[Facebook Cookies](https://www.facebook.com/policy/cookies/) - -"Datr" is a unique identifier for your browser and it has a lifespan of two years. - -"c_user" and "xs" cookies to verify the account and have a lifespan of 365 days - +- [Facebook Cookies](https://www.facebook.com/policy/cookies/) +- "Datr" is a unique identifier for your browser and it has a lifespan of two years. +- "c_user" and "xs" cookies to verify the account and have a lifespan of 365 days From a3b064f4eef00ca5559cd23f2c7a47ded8083f21 Mon Sep 17 00:00:00 2001 From: Guillaume Lacasa Date: Mon, 11 Dec 2023 17:38:39 +0100 Subject: [PATCH 214/716] Find PanneauPocket city id from page URL (#3825) Co-authored-by: Guillaume Lacasa --- bridges/PanneauPocketBridge.php | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/bridges/PanneauPocketBridge.php b/bridges/PanneauPocketBridge.php index 8547a500..464d56c5 100644 --- a/bridges/PanneauPocketBridge.php +++ b/bridges/PanneauPocketBridge.php @@ -12,6 +12,12 @@ class PanneauPocketBridge extends BridgeAbstract 'name' => 'Choisir une ville', 'type' => 'list', 'values' => self::CITIES, + ], + 'cityName' => [ + 'name' => 'Ville', + ], + 'cityId' => [ + 'name' => 'Identifiant', ] ] ]; @@ -113,8 +119,14 @@ class PanneauPocketBridge extends BridgeAbstract public function collectData() { - $matchedCity = array_search($this->getInput('cities'), self::CITIES); - $city = strtolower($this->getInput('cities') . '-' . $matchedCity); + $cityId = $this->getInput('cityId'); + if ($cityId != null) { + $cityName = $this->getInput('cityName'); + $city = strtolower($cityId . '-' . $cityName); + } else { + $matchedCity = array_search($this->getInput('cities'), self::CITIES); + $city = strtolower($this->getInput('cities') . '-' . $matchedCity); + } $url = sprintf('https://app.panneaupocket.com/ville/%s', urlencode($city)); $html = getSimpleHTMLDOM($url); @@ -136,6 +148,18 @@ class PanneauPocketBridge extends BridgeAbstract } } + public function detectParameters($url) + { + $params = []; + $regex = '/\/ville\/(\d+)-([a-z0-9-]+)/'; + if (preg_match($regex, $url, $matches)) { + $params['cityId'] = $matches[1]; + $params['cityName'] = $matches[2]; + return $params; + } + return null; + } + /** * Produce self::CITIES array */ From 0b67544f86e887d0bf4544b452fef816ec262928 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Wed, 13 Dec 2023 21:09:48 +0100 Subject: [PATCH 215/716] [PepperBridgeAbstract] Fix temperature handling (#3828) Website has changed how the temperature is renderd : the bridge does follow the new website structure --- bridges/PepperBridgeAbstract.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 8e8a2e8d..6cb0f302 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -117,8 +117,7 @@ class PepperBridgeAbstract extends BridgeAbstract . $this->getSource($deal) . $deal->find('div[class*=' . $selectorDescription . ']', 0)->innertext . '' - . $deal->find('div[class*=' . $selectorHot . ']', 0) - ->find('span', 0)->outertext + . $this->getTemperature($deal) . ''; // Check if a clock icon is displayed on the deal @@ -368,6 +367,16 @@ HEREDOC; } } + /** + * Get the temperature from a Deal if it exists + * @return string String of the deal temperature + */ + private function getTemperature($deal) + { + $data = Json::decode($deal->find('div[class=js-vue2]', 0)->getAttribute('data-vue2')); + return $data['props']['thread']['temperature'] . '°'; + } + /** * Get the source of a Deal if it exists * @return string String of the deal source From f01729c86f29b61d4a50ea8f76c639cd1fc19f5a Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 21:40:13 +0100 Subject: [PATCH 216/716] fix(arstechnica): plus a few unrelated tweaks (#3829) --- actions/FrontpageAction.php | 1 + bridges/ArsTechnicaBridge.php | 9 ++++++++- bridges/VkBridge.php | 2 +- caches/FileCache.php | 2 +- index.php | 6 +++++- lib/Configuration.php | 15 ++------------- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/actions/FrontpageAction.php b/actions/FrontpageAction.php index 64281b1e..ad48927d 100644 --- a/actions/FrontpageAction.php +++ b/actions/FrontpageAction.php @@ -31,6 +31,7 @@ final class FrontpageAction implements ActionInterface } } + // todo: cache this renderered template return render(__DIR__ . '/../templates/frontpage.html.php', [ 'messages' => $messages, 'admin_email' => Configuration::getConfig('admin', 'email'), diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index 5b3283b5..613c1c58 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -37,7 +37,14 @@ class ArsTechnicaBridge extends FeedExpander { $item_html = getSimpleHTMLDOMCached($item['uri'] . '&'); $item_html = defaultLinkTo($item_html, self::URI); - $item['content'] = $item_html->find('.amp-wp-article-content', 0); + + $item_content = $item_html->find('.article-content.post-page', 0); + if (!$item_content) { + // The dom selector probably broke. Let's just return the item as-is + return $item; + } + + $item['content'] = $item_content; // remove various ars advertising $item['content']->find('#social-left', 0)->remove(); diff --git a/bridges/VkBridge.php b/bridges/VkBridge.php index 503bc4d0..980b4154 100644 --- a/bridges/VkBridge.php +++ b/bridges/VkBridge.php @@ -523,7 +523,7 @@ class VkBridge extends BridgeAbstract } if (!preg_match('#^https?://vk.com/#', $uri)) { - returnServerError('Unexpected redirect location'); + returnServerError('Unexpected redirect location: ' . $uri); } $redirects++; diff --git a/caches/FileCache.php b/caches/FileCache.php index 2f4b3ad5..09d12791 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -49,8 +49,8 @@ class FileCache implements CacheInterface { $item = [ 'key' => $key, - 'value' => $value, 'expiration' => $ttl === null ? 0 : time() + $ttl, + 'value' => $value, ]; $cacheFile = $this->createCacheFile($key); $bytes = file_put_contents($cacheFile, serialize($item), LOCK_EX); diff --git a/index.php b/index.php index 123f6ecd..14713e06 100644 --- a/index.php +++ b/index.php @@ -6,7 +6,11 @@ if (version_compare(\PHP_VERSION, '7.4.0') === -1) { require_once __DIR__ . '/lib/bootstrap.php'; -Configuration::verifyInstallation(); +$errors = Configuration::checkInstallation(); +if ($errors) { + die('
' . implode("\n", $errors) . '
'); +} + $customConfig = []; if (file_exists(__DIR__ . '/config.ini.php')) { $customConfig = parse_ini_file(__DIR__ . '/config.ini.php', true, INI_SCANNER_TYPED); diff --git a/lib/Configuration.php b/lib/Configuration.php index d699178f..ac7d29bf 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -15,15 +15,7 @@ final class Configuration { } - /** - * Verifies the current installation of RSS-Bridge and PHP. - * - * Returns an error message and aborts execution if the installation does - * not satisfy the requirements of RSS-Bridge. - * - * @return void - */ - public static function verifyInstallation() + public static function checkInstallation(): array { $errors = []; @@ -57,10 +49,7 @@ final class Configuration if (!extension_loaded('json')) { $errors[] = 'json extension not loaded'; } - - if ($errors) { - throw new \Exception(sprintf('Configuration error: %s', implode(', ', $errors))); - } + return $errors; } public static function loadConfiguration(array $customConfig = [], array $env = []) From d157816e07e47dfdc8583d5fcee1925031aa6496 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 21:56:14 +0100 Subject: [PATCH 217/716] fix(reddit): cache tweak for 403 forbidden (#3830) --- actions/DisplayAction.php | 9 +++++++-- bridges/RedditBridge.php | 5 +++++ lib/contents.php | 1 + lib/http.php | 1 + 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index e3b25fef..43563996 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -19,6 +19,7 @@ class DisplayAction implements ActionInterface 'message' => 'RSS-Bridge is down for maintenance.', ]), 503); } + $cacheKey = 'http_' . json_encode($request); /** @var Response $cachedResponse */ $cachedResponse = $this->cache->get($cacheKey); @@ -80,16 +81,19 @@ class DisplayAction implements ActionInterface $this->cache->set($cacheKey, $response, $ttl); } - if (in_array($response->getCode(), [429, 503])) { - $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); // average 20m + if (in_array($response->getCode(), [403, 429, 503])) { + // Cache these responses for about ~20 mins on average + $this->cache->set($cacheKey, $response, 60 * 15 + rand(1, 60 * 10)); } if ($response->getCode() === 500) { $this->cache->set($cacheKey, $response, 60 * 15); } + if (rand(1, 100) === 2) { $this->cache->prune(); } + return $response; } @@ -187,6 +191,7 @@ class DisplayAction implements ActionInterface private function logBridgeError($bridgeName, $code) { + // todo: it's not really necessary to json encode $report $cacheKey = 'error_reporting_' . $bridgeName . '_' . $code; $report = $this->cache->get($cacheKey); if ($report) { diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index f761afaa..c393c146 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -85,6 +85,11 @@ class RedditBridge extends BridgeAbstract if ($e->getCode() === 429) { $this->cache->set($cacheKey, true, 60 * 16); } + if ($e->getCode() === 403) { + // 403 Forbidden + // This can possibly mean that reddit has permanently blocked this server's ip address + $this->cache->set($cacheKey, true, 60 * 61); + } throw $e; } } diff --git a/lib/contents.php b/lib/contents.php index 055d6bf3..a4def21a 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -71,6 +71,7 @@ function getContents( // Ignore invalid 'Last-Modified' HTTP header value } } + // todo: to be nice nice citizen we should also check for Etag } $response = $httpClient->request($url, $config); diff --git a/lib/http.php b/lib/http.php index c5c57d05..eb70705f 100644 --- a/lib/http.php +++ b/lib/http.php @@ -2,6 +2,7 @@ class HttpException extends \Exception { + // todo: should include the failing http response (if present) } final class CloudFlareException extends HttpException From 0c4b498d4f41d8402bbc33cbf2864e13c0d76ba2 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 22:06:47 +0100 Subject: [PATCH 218/716] fix(reddit): tweak internal cache logic (#3831) --- bridges/RedditBridge.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index c393c146..2b7fe84f 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -75,20 +75,26 @@ class RedditBridge extends BridgeAbstract public function collectData() { - $cacheKey = 'reddit_rate_limit'; - if ($this->cache->get($cacheKey)) { + $forbiddenKey = 'reddit_forbidden'; + if ($this->cache->get($forbiddenKey)) { + throw new HttpException('403 Forbidden', 403); + } + + $rateLimitKey = 'reddit_rate_limit'; + if ($this->cache->get($rateLimitKey)) { throw new HttpException('429 Too Many Requests', 429); } + try { $this->collectDataInternal(); } catch (HttpException $e) { - if ($e->getCode() === 429) { - $this->cache->set($cacheKey, true, 60 * 16); - } if ($e->getCode() === 403) { // 403 Forbidden // This can possibly mean that reddit has permanently blocked this server's ip address - $this->cache->set($cacheKey, true, 60 * 61); + $this->cache->set($forbiddenKey, true, 60 * 61); + } + if ($e->getCode() === 429) { + $this->cache->set($rateLimitKey, true, 60 * 16); } throw $e; } From 38e9c396cfe6b933f1752942385dcf5ee05730d6 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 13 Dec 2023 22:20:21 +0100 Subject: [PATCH 219/716] fix(codeberg): css selector tweak (#3832) * fix(codeberg): css selector tweak * yup --- bridges/CodebergBridge.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bridges/CodebergBridge.php b/bridges/CodebergBridge.php index 2a450477..79dd706c 100644 --- a/bridges/CodebergBridge.php +++ b/bridges/CodebergBridge.php @@ -79,9 +79,9 @@ class CodebergBridge extends BridgeAbstract public function collectData() { - $html = getSimpleHTMLDOM($this->getURI()); - - $html = defaultLinkTo($html, $this->getURI()); + $url = $this->getURI(); + $html = getSimpleHTMLDOM($url); + $html = defaultLinkTo($html, $url); switch ($this->queriedContext) { case 'Commits': @@ -205,22 +205,22 @@ class CodebergBridge extends BridgeAbstract */ private function extractIssues($html) { - $div = $html->find('div.issue.list', 0); + $issueList = $html->find('div#issue-list', 0); - foreach ($div->find('li.item') as $li) { + foreach ($issueList->find('div.flex-item') as $div) { $item = []; - $number = trim($li->find('a.index,ml-0.mr-2', 0)->plaintext); + $number = trim($div->find('a.index,ml-0.mr-2', 0)->plaintext); - $item['title'] = $li->find('a.title', 0)->plaintext . ' (' . $number . ')'; - $item['uri'] = $li->find('a.title', 0)->href; + $item['title'] = $div->find('a.issue-title', 0)->plaintext . ' (' . $number . ')'; + $item['uri'] = $div->find('a.issue-title', 0)->href; - $time = $li->find('relative-time.time-since', 0); + $time = $div->find('relative-time.time-since', 0); if ($time) { $item['timestamp'] = $time->datetime; } - $item['author'] = $li->find('div.desc', 0)->find('a', 1)->plaintext; + //$item['author'] = $li->find('div.desc', 0)->find('a', 1)->plaintext; // Fetch issue page $issuePage = getSimpleHTMLDOMCached($item['uri'], 3600); @@ -228,7 +228,7 @@ class CodebergBridge extends BridgeAbstract $item['content'] = $issuePage->find('div.timeline-item.comment.first', 0)->find('div.render-content.markup', 0); - foreach ($li->find('a.ui.label') as $label) { + foreach ($div->find('a.ui.label') as $label) { $item['categories'][] = $label->plaintext; } From d127bf6e009318914f9b35222fcd97ff137dad57 Mon Sep 17 00:00:00 2001 From: Arnav Jain Date: Fri, 15 Dec 2023 23:36:50 +0100 Subject: [PATCH 220/716] [DagensNyheterDirektBridge] New bridge (#3834) * [DagensNyheterDirektBridge] New bridge * [DagensNyheterDirektBridge] Lint: Replace all tabs with space * [DagensNyheterDirektBridge] Lint: Lines Add empty lines and move start brace to new line * [DagensNyheterDirektBridge] Lint: short- array syntax * [DagensNyheterDirektBridge] Lint: short array syntax Fix incorrect line ending * [DagensNyheterDirektBridge] Lint: further lint fixes * [DagensNyheterDirektBridge] Lint: final fixes --- bridges/DagensNyheterDirektBridge.php | 62 +++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 bridges/DagensNyheterDirektBridge.php diff --git a/bridges/DagensNyheterDirektBridge.php b/bridges/DagensNyheterDirektBridge.php new file mode 100644 index 00000000..4d1629fb --- /dev/null +++ b/bridges/DagensNyheterDirektBridge.php @@ -0,0 +1,62 @@ +find('article') as $element) { + $link = $element->find('button', 0)->getAttribute('data-link'); + $datetime = $element->getAttribute('data-publication-time'); + $url = self::BASEURL . $link; + $title = $element->find('h2', 0)->plaintext; + $author = $element->find('div.ds-byline__titles', 0)->plaintext; + // Debug::log($link); + // Debug::log($datetime); + // Debug::log($title); + // Debug::log($url); + // Debug::log($author); + + $article_content = $element->find('div.direkt-post__content', 0); + $article_html = ''; + + $figure = $element->find('figure', 0); + + if ($figure) { + $article_html = $figure->find('img', 0) . '

' . $figure->find('figcaption', 0) . '

'; + } + + foreach ($article_content->find('p') as $p) { + $article_html = $article_html . $p; + } + + $this->items[] = [ + 'uri' => $url, + 'title' => $title, + 'author' => trim($author), + 'timestamp' => $datetime, + 'content' => trim($article_html), + ]; + + if (count($this->items) > self::LIMIT) { + break; + } + } + } +} From 4e1fa946b4915164c0ec2f09090999597b9d69ad Mon Sep 17 00:00:00 2001 From: ash <153942603+xz47sv@users.noreply.github.com> Date: Fri, 15 Dec 2023 23:39:04 +0100 Subject: [PATCH 221/716] add rb.ash.fail to list of public hosts (#3835) --- docs/01_General/06_Public_Hosts.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index de538cf1..c9572824 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -20,7 +20,8 @@ | ![](https://iplookup.flagfox.net/images/h16/NL.png) | https://feed.eugenemolotov.ru | ![](https://img.shields.io/website/https/feed.eugenemolotov.ru.svg) | [@em92](https://github.com/em92) | Hosted in Amsterdam, Netherlands | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rss-bridge.mediani.de | ![](https://img.shields.io/website/https/rss-bridge.mediani.de.svg) | [@sokai](https://github.com/sokai) | Hosted with Netcup, Germany | | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | -| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud +| ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud | +| ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rb.ash.fail | ![](https://img.shields.io/website/https/rb.ash.fail.svg) | [@ash](https://ash.fail/contact.html) | Hosted with Hostaris, Germany ## Inactive instances From d4ae55733b6f7684de0b878d91e53c8a5f917d41 Mon Sep 17 00:00:00 2001 From: Tone <66808319+Tone866@users.noreply.github.com> Date: Fri, 15 Dec 2023 23:39:27 +0100 Subject: [PATCH 222/716] Update GolemBridge.php (#3836) deleted the code which adds the author to the feed, because the author is already in the original feed, so it is not needed. --- bridges/GolemBridge.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index 6699e433..debf5b29 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -82,11 +82,6 @@ class GolemBridge extends FeedExpander // URI without RSS feed reference $item['uri'] = $articlePage->find('head meta[name="twitter:url"]', 0)->content; - $author = $articlePage->find('article header .authors .authors__name', 0); - if ($author) { - $item['author'] = $author->plaintext; - } - $categories = $articlePage->find('ul.tags__list li'); foreach ($categories as $category) { $trimmedcategories[] = trim(html_entity_decode($category->plaintext)); From 0116dde27549fcdaf5600bc255173047e1fce4f9 Mon Sep 17 00:00:00 2001 From: Mynacol Date: Sat, 16 Dec 2023 10:43:27 +0100 Subject: [PATCH 223/716] [GolemBridge] Add h2 elements from article content Else some headers are just missing. Example article with previously missing movie names: https://www.golem.de/news/science-fiction-die-zehn-besten-filme-aus-den-spannenden-70ern-2312-179557.html --- bridges/GolemBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index debf5b29..c1b03433 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -132,7 +132,7 @@ class GolemBridge extends FeedExpander $img->src = $img->getAttribute('data-src-full'); } - foreach ($content->find('p, h1, h3, img[src*="."]') as $element) { + foreach ($content->find('p, h1, h2, h3, img[src*="."]') as $element) { $item .= $element; } From c5f586497f3d23be61a6e8a5fe0f948f98a5b2f6 Mon Sep 17 00:00:00 2001 From: Mynacol Date: Sat, 16 Dec 2023 11:21:19 +0100 Subject: [PATCH 224/716] [GolemBridge] Remove multi-page page headers On multi-page articles like [1], all the pages after the first one have a page header that we add in the article content. When we tack the pages together again, we don't need those extra page headers. [1] https://www.golem.de/news/science-fiction-die-zehn-besten-filme-aus-den-spannenden-70ern-2312-179557.html --- bridges/GolemBridge.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/bridges/GolemBridge.php b/bridges/GolemBridge.php index c1b03433..599d713a 100644 --- a/bridges/GolemBridge.php +++ b/bridges/GolemBridge.php @@ -116,9 +116,6 @@ class GolemBridge extends FeedExpander // reload html, as remove() is buggy $article = str_get_html($article->outertext); - if ($pageHeader = $article->find('header.paged-cluster-header h1', 0)) { - $item .= $pageHeader; - } $header = $article->find('header', 0); foreach ($header->find('p, figure') as $element) { From b34fa2d278eb14172cbeac4ddf34dca90716b5cb Mon Sep 17 00:00:00 2001 From: Brendan Kidwell Date: Sun, 17 Dec 2023 11:08:40 -0500 Subject: [PATCH 225/716] RumbleBridge - new selector needed on user/channel page (#3843) --- bridges/RumbleBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/RumbleBridge.php b/bridges/RumbleBridge.php index 08b416bf..d5b82136 100644 --- a/bridges/RumbleBridge.php +++ b/bridges/RumbleBridge.php @@ -39,7 +39,7 @@ class RumbleBridge extends BridgeAbstract } $dom = getSimpleHTMLDOM($url); - foreach ($dom->find('li.video-listing-entry') as $video) { + foreach ($dom->find('ol.thumbnail__grid div.thumbnail__grid--item') as $video) { $datetime = $video->find('time', 0)->getAttribute('datetime'); $this->items[] = [ From 3944ae68cbe8b8dd4fd653a288cffdb42cd3802e Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 19 Dec 2023 07:53:25 +0100 Subject: [PATCH 226/716] fix(reddit): use old.reddit.com instead of www.reddit.com (#3848) --- bridges/RedditBridge.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index 2b7fe84f..bb3e7afc 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -1,10 +1,15 @@ Date: Tue, 19 Dec 2023 08:46:37 +0100 Subject: [PATCH 227/716] fix(gatesnotes): the unfucked their json (#3849) --- bridges/GatesNotesBridge.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bridges/GatesNotesBridge.php b/bridges/GatesNotesBridge.php index 24ba9b2e..0d919968 100644 --- a/bridges/GatesNotesBridge.php +++ b/bridges/GatesNotesBridge.php @@ -23,12 +23,14 @@ class GatesNotesBridge extends BridgeAbstract $cleanedContent = str_replace([ '', '', - '\r\n', ], '', $rawContent); - $cleanedContent = str_replace('\"', '"', $cleanedContent); - $cleanedContent = trim($cleanedContent, '"'); + // $cleanedContent = str_replace('\"', '"', $cleanedContent); + // $cleanedContent = trim($cleanedContent, '"'); $json = Json::decode($cleanedContent, false); + if (is_string($json)) { + throw new \Exception('wtf? ' . $json); + } foreach ($json as $article) { $item = []; From 98a94855dc6b909b75629c6630c3795c68e7d560 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 20 Dec 2023 03:16:25 +0100 Subject: [PATCH 228/716] feat: embed response in http exception (#3847) --- bridges/GettrBridge.php | 10 +++++++++- config.default.ini.php | 3 ++- lib/contents.php | 15 ++------------- lib/http.php | 24 +++++++++++++++++++++++- templates/exception.html.php | 23 +++++++++++++++++++++++ 5 files changed, 59 insertions(+), 16 deletions(-) diff --git a/bridges/GettrBridge.php b/bridges/GettrBridge.php index 74804043..d3b9b899 100644 --- a/bridges/GettrBridge.php +++ b/bridges/GettrBridge.php @@ -33,7 +33,15 @@ class GettrBridge extends BridgeAbstract $user, min($this->getInput('limit'), 20) ); - $data = json_decode(getContents($api), false); + try { + $json = getContents($api); + } catch (HttpException $e) { + if ($e->getCode() === 400 && str_contains($e->response->getBody(), 'E_USER_NOTFOUND')) { + throw new \Exception('User not found: ' . $user); + } + throw $e; + } + $data = json_decode($json, false); foreach ($data->result->aux->post as $post) { $this->items[] = [ diff --git a/config.default.ini.php b/config.default.ini.php index 52786aef..201b1414 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -47,7 +47,8 @@ enable_debug_mode = false enable_maintenance_mode = false [http] -timeout = 60 +; Operation timeout in seconds +timeout = 30 useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0" ; Max http response size in MB diff --git a/lib/contents.php b/lib/contents.php index a4def21a..8676a2a8 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -101,19 +101,8 @@ function getContents( $response = $response->withBody($cachedResponse->getBody()); break; default: - $exceptionMessage = sprintf( - '%s resulted in %s %s %s', - $url, - $response->getCode(), - $response->getStatusLine(), - // If debug, include a part of the response body in the exception message - Debug::isEnabled() ? mb_substr($response->getBody(), 0, 500) : '', - ); - - if (CloudFlareException::isCloudFlareResponse($response)) { - throw new CloudFlareException($exceptionMessage, $response->getCode()); - } - throw new HttpException(trim($exceptionMessage), $response->getCode()); + $e = HttpException::fromResponse($response, $url); + throw $e; } if ($returnFull === true) { // todo: return the actual response object diff --git a/lib/http.php b/lib/http.php index eb70705f..bfa6b6bf 100644 --- a/lib/http.php +++ b/lib/http.php @@ -2,7 +2,29 @@ class HttpException extends \Exception { - // todo: should include the failing http response (if present) + public ?Response $response; + + public function __construct(string $message = '', int $statusCode = 0, ?Response $response = null) + { + parent::__construct($message, $statusCode); + $this->response = $response ?? new Response('', 0); + } + + public static function fromResponse(Response $response, string $url): HttpException + { + $message = sprintf( + '%s resulted in %s %s %s', + $url, + $response->getCode(), + $response->getStatusLine(), + // If debug, include a part of the response body in the exception message + Debug::isEnabled() ? mb_substr($response->getBody(), 0, 500) : '', + ); + if (CloudFlareException::isCloudFlareResponse($response)) { + return new CloudFlareException($message, $response->getCode(), $response); + } + return new HttpException(trim($message), $response->getCode(), $response); + } } final class CloudFlareException extends HttpException diff --git a/templates/exception.html.php b/templates/exception.html.php index dac0ad26..e1dd97c1 100644 --- a/templates/exception.html.php +++ b/templates/exception.html.php @@ -16,6 +16,13 @@

+ getCode() === 400): ?> +

400 Bad Request

+

+ This is usually caused by an incorrectly constructed http request. +

+ + getCode() === 404): ?>

404 Page Not Found

@@ -40,6 +47,22 @@

+ getCode() === 0): ?> +

+ See + + https://curl.haxx.se/libcurl/c/libcurl-errors.html + + for description of the curl error code. +

+ +

+ + https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/getCode()) ?> + +

+ + getCode() === 10): ?>

The rss feed is completely empty

From 4e40e032b0fcac52bc74ba5994cefe1d00debf45 Mon Sep 17 00:00:00 2001 From: Mynacol Date: Wed, 20 Dec 2023 22:18:10 +0100 Subject: [PATCH 229/716] Remove matrix reference The main communications platform is still Libera.chat, matrix was only provided by the hosted IRC-Matrix bridge. The bridge was turned off already and won't come back. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 570fb87d..2a762d45 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ Officially hosted instance: https://rss-bridge.org/bridge01/ [![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg?logo=github)](https://github.com/rss-bridge/rss-bridge/releases/latest) [![irc.libera.chat](https://img.shields.io/badge/irc.libera.chat-%23rssbridge-blue.svg)](https://web.libera.chat/#rssbridge) -[![Chat on Matrix](https://matrix.to/img/matrix-badge.svg)](https://matrix.to/#/#rssbridge:libera.chat) [![Actions Status](https://img.shields.io/github/actions/workflow/status/RSS-Bridge/rss-bridge/tests.yml?branch=master&label=GitHub%20Actions&logo=github)](https://github.com/RSS-Bridge/rss-bridge/actions) ||| From 4c5cf89725e7ebd975eb6ec5136b5e3927df07fe Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 21 Dec 2023 09:18:21 +0100 Subject: [PATCH 230/716] fix(rumble): not all videos have a datetime (#3852) --- bridges/RumbleBridge.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bridges/RumbleBridge.php b/bridges/RumbleBridge.php index d5b82136..f6bfca7d 100644 --- a/bridges/RumbleBridge.php +++ b/bridges/RumbleBridge.php @@ -40,15 +40,18 @@ class RumbleBridge extends BridgeAbstract $dom = getSimpleHTMLDOM($url); foreach ($dom->find('ol.thumbnail__grid div.thumbnail__grid--item') as $video) { - $datetime = $video->find('time', 0)->getAttribute('datetime'); - - $this->items[] = [ + $item = [ 'title' => $video->find('h3', 0)->plaintext, 'uri' => self::URI . $video->find('a', 0)->href, - 'timestamp' => (new \DateTimeImmutable($datetime))->getTimestamp(), 'author' => $account . '@rumble.com', 'content' => defaultLinkTo($video, self::URI)->innertext, ]; + $time = $video->find('time', 0); + if ($time) { + $publishedAt = new \DateTimeImmutable($time->getAttribute('datetime')); + $item['timestamp'] = $publishedAt->getTimestamp(); + } + $this->items[] = $item; } } From f40f99740588b09033917fd38132a99875495540 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 21 Dec 2023 09:24:22 +0100 Subject: [PATCH 231/716] fix: various small fixes (#3853) --- bridges/ARDAudiothekBridge.php | 20 +++++++++++++------- bridges/CarThrottleBridge.php | 6 ++---- bridges/EZTVBridge.php | 2 +- bridges/TrelloBridge.php | 2 +- bridges/YoutubeBridge.php | 6 +++++- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/bridges/ARDAudiothekBridge.php b/bridges/ARDAudiothekBridge.php index 2c1958f3..619c0911 100644 --- a/bridges/ARDAudiothekBridge.php +++ b/bridges/ARDAudiothekBridge.php @@ -63,11 +63,13 @@ class ARDAudiothekBridge extends BridgeAbstract public function collectData() { - $oldTz = date_default_timezone_get(); + $path = $this->getInput('path'); + $limit = $this->getInput('limit'); + $oldTz = date_default_timezone_get(); date_default_timezone_set('Europe/Berlin'); - $pathComponents = explode('/', $this->getInput('path')); + $pathComponents = explode('/', $path); if (empty($pathComponents)) { returnClientError('Path may not be empty'); } @@ -82,17 +84,21 @@ class ARDAudiothekBridge extends BridgeAbstract } $url = self::APIENDPOINT . 'programsets/' . $showID . '/'; - $rawJSON = getContents($url); - $processedJSON = json_decode($rawJSON)->data->programSet; + $json1 = getContents($url); + $data1 = Json::decode($json1, false); + $processedJSON = $data1->data->programSet; + if (!$processedJSON) { + throw new \Exception('Unable to find show id: ' . $showID); + } - $limit = $this->getInput('limit'); $answerLength = 1; $offset = 0; $numberOfElements = 1; while ($answerLength != 0 && $offset < $numberOfElements && (is_null($limit) || $offset < $limit)) { - $rawJSON = getContents($url . '?offset=' . $offset); - $processedJSON = json_decode($rawJSON)->data->programSet; + $json2 = getContents($url . '?offset=' . $offset); + $data2 = Json::decode($json2, false); + $processedJSON = $data2->data->programSet; $answerLength = count($processedJSON->items->nodes); $offset = $offset + $answerLength; diff --git a/bridges/CarThrottleBridge.php b/bridges/CarThrottleBridge.php index 913b686c..70d7b54e 100644 --- a/bridges/CarThrottleBridge.php +++ b/bridges/CarThrottleBridge.php @@ -9,8 +9,7 @@ class CarThrottleBridge extends BridgeAbstract public function collectData() { - $news = getSimpleHTMLDOMCached(self::URI . 'news') - or returnServerError('could not retrieve page'); + $news = getSimpleHTMLDOMCached(self::URI . 'news'); $this->items[] = []; @@ -22,8 +21,7 @@ class CarThrottleBridge extends BridgeAbstract $item['uri'] = self::URI . $titleElement->getAttribute('href'); $item['title'] = $titleElement->innertext; - $articlePage = getSimpleHTMLDOMCached($item['uri']) - or returnServerError('could not retrieve page'); + $articlePage = getSimpleHTMLDOMCached($item['uri']); $authorDiv = $articlePage->find('div.author div'); if ($authorDiv) { diff --git a/bridges/EZTVBridge.php b/bridges/EZTVBridge.php index 73318f0c..25a88124 100644 --- a/bridges/EZTVBridge.php +++ b/bridges/EZTVBridge.php @@ -96,7 +96,7 @@ class EZTVBridge extends BridgeAbstract protected function getItemFromTorrent($torrent) { $item = []; - $item['uri'] = $torrent->episode_url; + $item['uri'] = $torrent->episode_url ?? $torrent->torrent_url; $item['author'] = $torrent->imdb_id; $item['timestamp'] = $torrent->date_released_unix; $item['title'] = $torrent->title; diff --git a/bridges/TrelloBridge.php b/bridges/TrelloBridge.php index a1b5cfb8..cab2bde2 100644 --- a/bridges/TrelloBridge.php +++ b/bridges/TrelloBridge.php @@ -648,7 +648,7 @@ class TrelloBridge extends BridgeAbstract $action->type ]; if (isset($action->data->card)) { - $item['categories'][] = $action->data->card->name; + $item['categories'][] = $action->data->card->name ?? $action->data->card->id; $item['uri'] = 'https://trello.com/c/' . $action->data->card->shortLink . '#action-' diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 993f8c90..6a29e387 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -164,7 +164,11 @@ class YoutubeBridge extends BridgeAbstract $jsonData = $this->extractJsonFromHtml($html); // TODO: this method returns only first 100 video items // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0]; + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0] ?? null; + if (!$jsonData) { + // playlist probably doesnt exists + throw new \Exception('Unable to find playlist: ' . $url_listing); + } $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; $item_count = count($jsonData); From ea2b4d7506f0feded2899cb0aab351fa7dca3194 Mon Sep 17 00:00:00 2001 From: July Date: Sat, 23 Dec 2023 03:42:37 -0500 Subject: [PATCH 232/716] [ArsTechnicaBridge] Properly handle paged content (#3855) * [ArsTechnicaBridge] Properly handle paged content * [ArsTechnicaBridge] Remove normal site ad wrapper --- bridges/ArsTechnicaBridge.php | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/bridges/ArsTechnicaBridge.php b/bridges/ArsTechnicaBridge.php index 613c1c58..2c631871 100644 --- a/bridges/ArsTechnicaBridge.php +++ b/bridges/ArsTechnicaBridge.php @@ -35,39 +35,34 @@ class ArsTechnicaBridge extends FeedExpander protected function parseItem(array $item) { - $item_html = getSimpleHTMLDOMCached($item['uri'] . '&'); + $item_html = getSimpleHTMLDOMCached($item['uri']); $item_html = defaultLinkTo($item_html, self::URI); + $item['content'] = $item_html->find('.article-content', 0); - $item_content = $item_html->find('.article-content.post-page', 0); - if (!$item_content) { - // The dom selector probably broke. Let's just return the item as-is - return $item; + $pages = $item_html->find('nav.page-numbers > .numbers > a', -2); + if (null !== $pages) { + for ($i = 2; $i <= $pages->innertext; $i++) { + $page_url = $item['uri'] . '&page=' . $i; + $page_html = getSimpleHTMLDOMCached($page_url); + $page_html = defaultLinkTo($page_html, self::URI); + $item['content'] .= $page_html->find('.article-content', 0); + } + $item['content'] = str_get_html($item['content']); } - $item['content'] = $item_content; - // remove various ars advertising $item['content']->find('#social-left', 0)->remove(); foreach ($item['content']->find('.ars-component-buy-box') as $ad) { $ad->remove(); } - foreach ($item['content']->find('i-amphtml-sizer') as $ad) { + foreach ($item['content']->find('.ad_wrapper') as $ad) { $ad->remove(); } foreach ($item['content']->find('.sidebar') as $ad) { $ad->remove(); } - foreach ($item['content']->find('a') as $link) { //remove amp redirect links - $url = $link->getAttribute('href'); - if (str_contains($url, 'go.redirectingat.com')) { - $url = extractFromDelimiters($url, 'url=', '&'); - $url = urldecode($url); - $link->setAttribute('href', $url); - } - } - - $item['content'] = backgroundToImg(str_replace('data-amp-original-style="background-image', 'style="background-image', $item['content'])); + $item['content'] = backgroundToImg($item['content']); $item['uid'] = explode('=', $item['uri'])[1]; From 98dafb61ae5519b7c6c4be2d7dd4d66b6bd6a4eb Mon Sep 17 00:00:00 2001 From: xduugu Date: Sat, 23 Dec 2023 08:43:01 +0000 Subject: [PATCH 233/716] [ARDAudiothekBridge] add duration to feed items (#3854) --- bridges/ARDAudiothekBridge.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bridges/ARDAudiothekBridge.php b/bridges/ARDAudiothekBridge.php index 619c0911..02b6b007 100644 --- a/bridges/ARDAudiothekBridge.php +++ b/bridges/ARDAudiothekBridge.php @@ -125,6 +125,10 @@ class ARDAudiothekBridge extends BridgeAbstract $item['categories'] = [$category]; } + $item['itunes'] = [ + 'duration' => $audio->duration, + ]; + $this->items[] = $item; } } From 9f163ab7c651f44c1d6266ca817aca2c0f208f51 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Mon, 25 Dec 2023 14:51:51 +0100 Subject: [PATCH 234/716] [FreeTelechargerBridge] Update to the new URL (#3856) * [FreeTelechargerBridge] Update to the new URL Website has changed URL and some design : this bridge is now adapted to thoses changes * [FreeTelechargerBridge] Fix example value Example valuse seems to use an "old" template, switch to a newer example that use the new template * [FreeTelechargerBridge] Fix notice Fix notice --- bridges/FreeTelechargerBridge.php | 61 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/bridges/FreeTelechargerBridge.php b/bridges/FreeTelechargerBridge.php index 8362b4ff..f0e5d35a 100644 --- a/bridges/FreeTelechargerBridge.php +++ b/bridges/FreeTelechargerBridge.php @@ -3,7 +3,7 @@ class FreeTelechargerBridge extends BridgeAbstract { const NAME = 'Free-Telecharger'; - const URI = 'https://www.free-telecharger.live/'; + const URI = 'https://www.free-telecharger.art/'; const DESCRIPTION = 'Suivi de série sur Free-Telecharger'; const MAINTAINER = 'sysadminstory'; const PARAMETERS = [ @@ -12,43 +12,46 @@ class FreeTelechargerBridge extends BridgeAbstract 'name' => 'URL de la série', 'type' => 'text', 'required' => true, - 'title' => 'URL d\'une série sans le https://www.free-telecharger.live/', + 'title' => 'URL d\'une série sans le https://www.free-telecharger.art/', 'pattern' => 'series.*\.html', - 'exampleValue' => 'series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html' + 'exampleValue' => 'series-vf-hd/151432-wolf-saison-1-complete-web-dl-720p.html' ], ] ]; const CACHE_TIMEOUT = 3600; + private string $showTitle; + private string $showTechDetails; + public function collectData() { - $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')); + $html = getSimpleHTMLDOM(self::URI . $this->getInput('url')); - // Find all block content of the page - $blocks = $html->find('div[class=block1]'); + // Find all block content of the page + $blocks = $html->find('div[class=block1]'); - // Global Infos block - $infosBlock = $blocks[0]; - // Links block - $linksBlock = $blocks[2]; + // Global Infos block + $infosBlock = $blocks[0]; + // Links block + $linksBlock = $blocks[2]; - // Extract Global Show infos - $this->showTitle = trim($infosBlock->find('div[class=titre1]', 0)->find('font', 0)->plaintext); - $this->showTechDetails = trim($infosBlock->find('div[align=center]', 0)->find('b', 0)->plaintext); + // Extract Global Show infos + $this->showTitle = trim($infosBlock->find('div[class=titre1]', 0)->find('font', 0)->plaintext); + $this->showTechDetails = trim($infosBlock->find('div[align=center]', 0)->find('b', 0)->plaintext); - // Get Episodes names and links - $episodes = $linksBlock->find('div[id=link]', 0)->find('font[color=#ff6600]'); - $links = $linksBlock->find('div[id=link]', 0)->find('a'); + // Get Episodes names and links + $episodes = $linksBlock->find('div[id=link]', 0)->find('font[color=#e93100]'); + $links = $linksBlock->find('div[id=link]', 0)->find('a'); foreach ($episodes as $index => $episode) { - $item = []; // Create an empty item - $item['title'] = $this->showTitle . ' ' . $this->showTechDetails . ' - ' . ltrim(trim($episode->plaintext), '-'); - $item['uri'] = $links[$index]->href; - $item['content'] = '' . $item['title'] . ''; - $item['uid'] = hash('md5', $item['uri']); + $item = []; // Create an empty item + $item['title'] = $this->showTitle . ' ' . $this->showTechDetails . ' - ' . ltrim(trim($episode->plaintext), '-'); + $item['uri'] = $links[$index]->href; + $item['content'] = '' . $item['title'] . ''; + $item['uid'] = hash('md5', $item['uri']); - $this->items[] = $item; // Add this item to the list + $this->items[] = $item; // Add this item to the list } } @@ -57,7 +60,7 @@ class FreeTelechargerBridge extends BridgeAbstract switch ($this->queriedContext) { case 'Suivi de publication de série': return $this->showTitle . ' ' . $this->showTechDetails . ' - ' . self::NAME; - break; + break; default: return self::NAME; } @@ -68,7 +71,7 @@ class FreeTelechargerBridge extends BridgeAbstract switch ($this->queriedContext) { case 'Suivi de publication de série': return self::URI . $this->getInput('url'); - break; + break; default: return self::URI; } @@ -76,14 +79,14 @@ class FreeTelechargerBridge extends BridgeAbstract public function detectParameters($url) { - // Example: https://www.free-telecharger.live/series-vf-hd/145458-the-last-of-us-saison-1-web-dl-720p.html + // Example: https://www.free-telecharger.art/series-vf-hd/151432-wolf-saison-1-complete-web-dl-720p.html $params = []; - $regex = '/^https:\/\/www.*\.free-telecharger\.live\/(series.*\.html)/'; + $regex = '/^https:\/\/www.*\.free-telecharger\.art\/(series.*\.html)/'; if (preg_match($regex, $url, $matches) > 0) { - $params['context'] = 'Suivi de publication de série'; - $params['url'] = urldecode($matches[1]); - return $params; + $params['context'] = 'Suivi de publication de série'; + $params['url'] = urldecode($matches[1]); + return $params; } return null; From c9074facfed51371a59dd189648c5a80751feb4e Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Tue, 26 Dec 2023 12:18:42 +0100 Subject: [PATCH 235/716] [GreatFonBridge] Remove bridge (#3857) Website is unreliable, it's not useful to keep this bridge. --- bridges/GreatFonBridge.php | 140 ------------------------------------- 1 file changed, 140 deletions(-) delete mode 100644 bridges/GreatFonBridge.php diff --git a/bridges/GreatFonBridge.php b/bridges/GreatFonBridge.php deleted file mode 100644 index 2951634c..00000000 --- a/bridges/GreatFonBridge.php +++ /dev/null @@ -1,140 +0,0 @@ - [ - 'u' => [ - 'name' => 'username', - 'type' => 'text', - 'title' => 'Instagram username you want to follow', - 'exampleValue' => 'aesoprockwins', - 'required' => true, - ], - ] - ]; - const TEST_DETECT_PARAMETERS = [ - 'https://www.instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], - 'https://instagram.com/instagram/' => ['context' => 'Username', 'u' => 'instagram'], - 'https://greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], - 'https://www.greatfon.com/v/instagram' => ['context' => 'Username', 'u' => 'instagram'], - ]; - - public function collectData() - { - $username = $this->getInput('u'); - $html = getSimpleHTMLDOMCached(self::URI . '/v/' . $username); - $html = defaultLinkTo($html, self::URI); - - foreach ($html->find('div[class*=content__item]') as $post) { - // Skip the ads - if (!str_contains($post->class, 'ads')) { - $url = $post->find('a[href^=https://greatfon.com/c/]', 0)->href; - $date = $this->parseDate($post->find('div[class=content__time-text]', 0)->plaintext); - $description = $post->find('img', 0)->alt; - $imageUrl = $post->find('img', 0)->src; - $author = $username; - $uid = $url; - $title = 'Post - ' . $username . ' - ' . $this->descriptionToTitle($description); - - // Checking post type - $isVideo = (bool) $post->find('div[class=content__camera]', 0); - $videoNote = $isVideo ? '

(video)

' : ''; - - $this->items[] = [ - 'uri' => $url, - 'author' => $author, - 'timestamp' => $date, - 'title' => $title, - 'thumbnail' => $imageUrl, - 'enclosures' => [$imageUrl], - 'content' => << - {$description} - -{$videoNote} -

{$description}

-HTML, - 'uid' => $uid - ]; - } - } - } - - private function parseDate($content) - { - // Parse date, and transform the date into a timetamp, even in a case of a relative date - $date = date_create(); - - // Content trimmed to be sure that the "article" is at the beginning of the string and remove "ago" to make it a valid PHP date interval - $dateString = trim(str_replace(' ago', '', $content)); - - // Replace the article "an" or "a" by the number "1" to be a valid PHP date interval - $dateString = preg_replace('/^((an|a) )/m', '1 ', $dateString); - - $relativeDate = date_interval_create_from_date_string($dateString); - if ($relativeDate) { - date_sub($date, $relativeDate); - // As the relative interval has the precision of a day for date older than 24 hours, we can remove the hour of the date, as it is not relevant - date_time_set($date, 0, 0, 0, 0); - } else { - $this->logger->info(sprintf('Unable to parse date string: %s', $dateString)); - } - return date_format($date, 'r'); - } - - public function getURI() - { - if (!is_null($this->getInput('u'))) { - return urljoin(self::URI, '/v/' . $this->getInput('u')); - } - - return parent::getURI(); - } - - public function getIcon() - { - return static::URI . '/images/favicon-hub-3ede543aa6d1225e8dc016ccff6879c8.ico?vsn=d'; - } - - private function descriptionToTitle($description) - { - return strlen($description) > 60 ? mb_substr($description, 0, 57) . '...' : $description; - } - - public function getName() - { - if (!is_null($this->getInput('u'))) { - return 'Username ' . $this->getInput('u') . ' - GreatFon Bridge'; - } - return parent::getName(); - } - - public function detectParameters($url) - { - $regex = '/^http(s|):\/\/((www\.|)(instagram.com)\/([a-zA-Z0-9_\.]{1,30})(\/reels\/|\/tagged\/|\/|)|(www\.|)(greatfon.com)\/v\/([a-zA-Z0-9_\.]{1,30}))/'; - if (preg_match($regex, $url, $matches) > 0) { - $params['context'] = 'Username'; - // Extract detected domain using the regex - $domain = $matches[8] ?? $matches[4]; - if ($domain == 'greatfon.com') { - $params['u'] = $matches[9]; - return $params; - } elseif ($domain == 'instagram.com') { - $params['u'] = $matches[5]; - return $params; - } else { - return null; - } - } else { - return null; - } - } -} From 19384463857c35b1d3ef0a7dbbbcc40d2f0cba0c Mon Sep 17 00:00:00 2001 From: Florent V Date: Tue, 26 Dec 2023 12:19:08 +0100 Subject: [PATCH 236/716] [EdfPricesBridge] add new bridge (#3846) * [EdfPricesBridge] add new brige * [EdfPricesBridge] bad refactor * [EdfPricesBridge] support php 7.4 --------- Co-authored-by: Florent VIOLLEAU --- bridges/EdfPricesBridge.php | 106 ++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 bridges/EdfPricesBridge.php diff --git a/bridges/EdfPricesBridge.php b/bridges/EdfPricesBridge.php new file mode 100644 index 00000000..f67ed30b --- /dev/null +++ b/bridges/EdfPricesBridge.php @@ -0,0 +1,106 @@ + [ + 'name' => 'Choisir un contrat', + 'type' => 'list', + // we can add later HCHP, EJP, base + 'values' => ['Tempo' => '/energie/edf/tarifs/tempo'], + ] + ] + ]; + const CACHE_TIMEOUT = 7200; // 2h + + /** + * @param simple_html_dom $html + * @param string $contractUri + * @return void + */ + private function tempo(simple_html_dom $html, string $contractUri): void + { + // current color and next + $daysDom = $html->find('#calendrier', 0)->nextSibling()->find('.card--ejp'); + if ($daysDom && count($daysDom) === 2) { + foreach ($daysDom as $dayDom) { + $day = trim($dayDom->find('.card__title', 0)->innertext) . '/' . (new \DateTime('now'))->format(('Y')); + $dayColor = $dayDom->find('.card-ejp__icon span', 0)->innertext; + + $text = $day . ' - ' . $dayColor; + $item['uri'] = self::URI . $contractUri; + $item['title'] = $text; + $item['author'] = self::MAINTAINER; + $item['content'] = $text; + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + } + } + + // colors + $ulDom = $html->find('#tarif-de-l-offre-edf-tempo-current-date-html-year', 0)->nextSibling()->nextSibling()->nextSibling(); + $elementsDom = $ulDom->find('li'); + if ($elementsDom && count($elementsDom) === 3) { + foreach ($elementsDom as $elementDom) { + $item = []; + + $matches = []; + preg_match_all('/Jour (.*) : Heures (.*) : (.*) € \/ Heures (.*) : (.*) €/um', $elementDom->innertext, $matches, PREG_SET_ORDER, 0); + + if ($matches && count($matches[0]) === 6) { + for ($i = 0; $i < 2; $i++) { + $text = 'Jour ' . $matches[0][1] . ' - Heures ' . $matches[0][2 + 2 * $i] . ' : ' . $matches[0][3 + 2 * $i] . '€'; + $item['uri'] = self::URI . $contractUri; + $item['title'] = $text; + $item['author'] = self::MAINTAINER; + $item['content'] = $text; + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + } + } + } + } + + // powers + $ulPowerContract = $ulDom->nextSibling()->nextSibling(); + $elementsPowerContractDom = $ulPowerContract->find('li'); + if ($elementsPowerContractDom && count($elementsPowerContractDom) === 4) { + foreach ($elementsPowerContractDom as $elementPowerContractDom) { + $item = []; + + $matches = []; + preg_match_all('/(.*) kVA : (.*) €/um', $elementPowerContractDom->innertext, $matches, PREG_SET_ORDER, 0); + + if ($matches && count($matches[0]) === 3) { + $text = $matches[0][1] . ' kVA : ' . $matches[0][2] . '€'; + $item['uri'] = self::URI . $contractUri; + $item['title'] = $text; + $item['author'] = self::MAINTAINER; + $item['content'] = $text; + $item['uid'] = hash('sha256', $item['title']); + + $this->items[] = $item; + } + } + } + } + + public function collectData() + { + $contract = $this->getKey('contract'); + $contractUri = $this->getInput('contract'); + $html = getSimpleHTMLDOM(self::URI . $contractUri); + + if ($contract === 'Tempo') { + $this->tempo($html, $contractUri); + } + } +} From ad2d4c7b1b538868070e0264f3692542883cac50 Mon Sep 17 00:00:00 2001 From: Florent V Date: Tue, 26 Dec 2023 12:20:49 +0100 Subject: [PATCH 237/716] [BridgeAbstract] use getParameters instead of static to allow overriding it from bridges (#3858) --- lib/BridgeAbstract.php | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index a7b811a8..0f86f454 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -154,8 +154,8 @@ abstract class BridgeAbstract { // Import and assign all inputs to their context foreach ($input as $name => $value) { - foreach (static::PARAMETERS as $context => $set) { - if (array_key_exists($name, static::PARAMETERS[$context])) { + foreach ($this->getParameters() as $context => $set) { + if (array_key_exists($name, $this->getParameters()[$context])) { $this->inputs[$context][$name]['value'] = $value; } } @@ -163,16 +163,16 @@ abstract class BridgeAbstract // Apply default values to missing data $contexts = [$queriedContext]; - if (array_key_exists('global', static::PARAMETERS)) { + if (array_key_exists('global', $this->getParameters())) { $contexts[] = 'global'; } foreach ($contexts as $context) { - if (!isset(static::PARAMETERS[$context])) { + if (!isset($this->getParameters()[$context])) { // unknown context provided by client, throw exception here? or continue? } - foreach (static::PARAMETERS[$context] as $name => $properties) { + foreach ($this->getParameters()[$context] as $name => $properties) { if (isset($this->inputs[$context][$name]['value'])) { continue; } @@ -204,8 +204,8 @@ abstract class BridgeAbstract } // Copy global parameter values to the guessed context - if (array_key_exists('global', static::PARAMETERS)) { - foreach (static::PARAMETERS['global'] as $name => $properties) { + if (array_key_exists('global', $this->getParameters())) { + foreach ($this->getParameters()['global'] as $name => $properties) { if (isset($input[$name])) { $value = $input[$name]; } else { @@ -246,8 +246,8 @@ abstract class BridgeAbstract if (!isset($this->inputs[$this->queriedContext][$input]['value'])) { return null; } - if (array_key_exists('global', static::PARAMETERS)) { - if (array_key_exists($input, static::PARAMETERS['global'])) { + if (array_key_exists('global', $this->getParameters())) { + if (array_key_exists($input, $this->getParameters()['global'])) { $context = 'global'; } } @@ -256,7 +256,7 @@ abstract class BridgeAbstract } $needle = $this->inputs[$this->queriedContext][$input]['value']; - foreach (static::PARAMETERS[$context][$input]['values'] as $first_level_key => $first_level_value) { + foreach ($this->getParameters()[$context][$input]['values'] as $first_level_key => $first_level_value) { if (!is_array($first_level_value) && $needle === (string)$first_level_value) { return $first_level_key; } elseif (is_array($first_level_value)) { @@ -273,7 +273,7 @@ abstract class BridgeAbstract { $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/'; if ( - empty(static::PARAMETERS) + empty($this->getParameters()) && preg_match($regex, $url, $urlMatches) > 0 && preg_match($regex, static::URI, $bridgeUriMatches) > 0 && $urlMatches[3] === $bridgeUriMatches[3] From c8178e1fc409635af1a40167c4f511feb8d3df7f Mon Sep 17 00:00:00 2001 From: Damien Calesse <2787828+kranack@users.noreply.github.com> Date: Wed, 27 Dec 2023 13:17:49 +0100 Subject: [PATCH 238/716] [SensCritique] Fix bridge (#3860) --- bridges/SensCritiqueBridge.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bridges/SensCritiqueBridge.php b/bridges/SensCritiqueBridge.php index b823b55c..005704e1 100644 --- a/bridges/SensCritiqueBridge.php +++ b/bridges/SensCritiqueBridge.php @@ -57,7 +57,7 @@ class SensCritiqueBridge extends BridgeAbstract } $html = getSimpleHTMLDOM($uri); // This selector name looks like it's automatically generated - $list = $html->find('div.Universes__WrapperProducts-sc-1qa2w66-0.eVdcAv', 0); + $list = $html->find('div[data-testid="row"]', 0); $this->extractDataFromList($list); } @@ -69,6 +69,7 @@ class SensCritiqueBridge extends BridgeAbstract if ($list === null) { returnClientError('Cannot extract data from list'); } + foreach ($list->find('div[data-testid="product-list-item"]') as $movie) { $item = []; $item['title'] = $movie->find('h2 a', 0)->plaintext; From 5ab1924c4f96937885e12bcbd16b7bfb83a3c15b Mon Sep 17 00:00:00 2001 From: tillcash Date: Thu, 28 Dec 2023 18:20:34 +0530 Subject: [PATCH 239/716] Add WorldbankBridge and OglafBridge (#3862) * Add WorldbankBridge and OglafBridge * Update OglafBridge.php Remove redundant parent call to parseItem and rename formal argument to improve code clarity. * Update WorldbankBridge.php fix lint --- bridges/OglafBridge.php | 35 +++++++++++++++++++++++++ bridges/WorldbankBridge.php | 52 +++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 bridges/OglafBridge.php create mode 100644 bridges/WorldbankBridge.php diff --git a/bridges/OglafBridge.php b/bridges/OglafBridge.php new file mode 100644 index 00000000..1f4bc1af --- /dev/null +++ b/bridges/OglafBridge.php @@ -0,0 +1,35 @@ + [ + 'name' => 'limit (max 20)', + 'type' => 'number', + 'defaultValue' => 10, + 'required' => true, + ] + ] + ]; + + public function collectData() + { + $url = self::URI . 'feeds/rss/'; + $limit = min(20, $this->getInput('limit')); + $this->collectExpandableDatas($url, $limit); + } + + protected function parseItem($item) + { + $html = getSimpleHTMLDOMCached($item['uri']); + $comicImage = $html->find('img[id="strip"]', 0); + $item['content'] = $comicImage; + + return $item; + } +} diff --git a/bridges/WorldbankBridge.php b/bridges/WorldbankBridge.php new file mode 100644 index 00000000..9b40e86e --- /dev/null +++ b/bridges/WorldbankBridge.php @@ -0,0 +1,52 @@ + [ + 'name' => 'Language', + 'type' => 'list', + 'defaultValue' => 'English', + 'values' => [ + 'English' => 'English', + 'French' => 'French', + ] + ], + 'limit' => [ + 'name' => 'limit (max 100)', + 'type' => 'number', + 'defaultValue' => 5, + 'required' => true, + ] + ] + ]; + + public function collectData() + { + $apiUrl = 'https://search.worldbank.org/api/v2/news?format=json&rows=' + . min(100, $this->getInput('limit')) + . '&lang_exact=' . $this->getInput('lang'); + + $jsonData = json_decode(getContents($apiUrl)); + + // Remove unnecessary data from the original object + if (isset($jsonData->documents->facets)) { + unset($jsonData->documents->facets); + } + + foreach ($jsonData->documents as $element) { + $this->items[] = [ + 'uid' => $element->id, + 'timestamp' => $element->lnchdt, + 'title' => $element->title->{'cdata!'}, + 'uri' => $element->url, + 'content' => $element->descr->{'cdata!'}, + ]; + } + } +} From f67d2eb88adc597cc57fbfc402c28725b671e5a3 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Thu, 28 Dec 2023 13:53:06 +0100 Subject: [PATCH 240/716] [TikTokBridge] Use embed iframe to bypass scraping protection (#3864) The Tiktok Website was totally changed using some "scraping" protection (passing as parameter value generated somewhere in the bunch of javascript to the "API URL" that was before). The iframe embed does not have such protection. It has less information (no date, ...) but it's better than nothing ! --- bridges/TikTokBridge.php | 66 ++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 43 deletions(-) diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php index 73a18b04..6590df66 100644 --- a/bridges/TikTokBridge.php +++ b/bridges/TikTokBridge.php @@ -8,12 +8,12 @@ class TikTokBridge extends BridgeAbstract const MAINTAINER = 'VerifiedJoseph'; const PARAMETERS = [ 'By user' => [ - 'username' => [ - 'name' => 'Username', - 'type' => 'text', - 'required' => true, - 'exampleValue' => '@tiktok', - ] + 'username' => [ + 'name' => 'Username', + 'type' => 'text', + 'required' => true, + 'exampleValue' => '@tiktok', + ] ]]; const TEST_DETECT_PARAMETERS = [ @@ -24,53 +24,33 @@ class TikTokBridge extends BridgeAbstract const CACHE_TIMEOUT = 900; // 15 minutes - private $feedName = ''; - public function collectData() { - $html = getSimpleHTMLDOM($this->getURI()); + $html = getSimpleHTMLDOMCached('https://www.tiktok.com/embed/' . $this->processUsername()); - $title = $html->find('h1', 0)->plaintext ?? self::NAME; - $this->feedName = htmlspecialchars_decode($title); + $author = $html->find('span[data-e2e=creator-profile-userInfo-TUXText]', 0)->plaintext ?? self::NAME; - $var = $html->find('script[id=SIGI_STATE]', 0); - if (!$var) { - throw new \Exception('Unable to find tiktok user data for ' . $this->processUsername()); - } - $SIGI_STATE_RAW = $var->innertext; - $SIGI_STATE = Json::decode($SIGI_STATE_RAW, false); + $videos = $html->find('div[data-e2e=common-videoList-VideoContainer]'); - if (!isset($SIGI_STATE->ItemModule)) { - return; - } - - foreach ($SIGI_STATE->ItemModule as $key => $value) { + foreach ($videos as $video) { $item = []; - $link = 'https://www.tiktok.com/@' . $value->author . '/video/' . $value->id; - $image = $value->video->dynamicCover; - if (empty($image)) { - $image = $value->video->cover; - } - $views = $value->stats->playCount; - $hastags = []; - foreach ($value->textExtra as $tag) { - $hastags[] = $tag->hashtagName; - } - $hastags_str = ''; - foreach ($hastags as $tag) { - $hastags_str .= '#' . $tag . ' '; - } + // Handle link "untracking" + $linkParts = parse_url($video->find('a', 0)->href); + $link = $linkParts['scheme'] . '://' . $linkParts['host'] . '/' . $linkParts['path']; + + $image = $video->find('video', 0)->poster; + $views = $video->find('div[data-e2e=common-Video-Count]', 0)->plaintext; + + $enclosures = [$image]; $item['uri'] = $link; - $item['title'] = $value->desc; - $item['timestamp'] = $value->createTime; - $item['author'] = '@' . $value->author; - $item['enclosures'][] = $image; - $item['categories'] = $hastags; + $item['title'] = 'Video'; + $item['author'] = '@' . $author; + $item['enclosures'] = $enclosures; $item['content'] = << -

{$views} views


Hashtags: {$hastags_str} +

{$views} views


EOD; $this->items[] = $item; @@ -91,7 +71,7 @@ EOD; { switch ($this->queriedContext) { case 'By user': - return $this->feedName . ' (' . $this->processUsername() . ') - TikTok'; + return $this->processUsername() . ' - TikTok'; default: return parent::getName(); } From 2032ed18c49a82fc2e634dfa6f2b91e652228876 Mon Sep 17 00:00:00 2001 From: Damien Calesse <2787828+kranack@users.noreply.github.com> Date: Thu, 28 Dec 2023 19:51:15 +0100 Subject: [PATCH 241/716] [SensCritique] Update the content to add the image (#3865) --- bridges/SensCritiqueBridge.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bridges/SensCritiqueBridge.php b/bridges/SensCritiqueBridge.php index 005704e1..f6a2ea16 100644 --- a/bridges/SensCritiqueBridge.php +++ b/bridges/SensCritiqueBridge.php @@ -71,10 +71,17 @@ class SensCritiqueBridge extends BridgeAbstract } foreach ($list->find('div[data-testid="product-list-item"]') as $movie) { + $synopsis = $movie->find('p[data-testid="synopsis"]', 0); + $item = []; $item['title'] = $movie->find('h2 a', 0)->plaintext; - // todo: fix image - $item['content'] = $movie->innertext; + $item['content'] = sprintf( + '

%s

%s

%s', + $movie->find('span[data-testid="poster-img-wrapper"]', 0)->{'data-srcname'}, + $movie->find('p[data-testid="other-infos"]', 0)->innertext, + $movie->find('p[data-testid="creators"]', 0)->innertext, + $synopsis ? sprintf('

%s

', $synopsis->innertext) : '' + ); $item['id'] = $this->getURI() . ltrim($movie->find('a', 0)->href, '/'); $item['uri'] = $this->getURI() . ltrim($movie->find('a', 0)->href, '/'); $this->items[] = $item; From 7dbe10658213e165c07faac01a8c79771b4917c8 Mon Sep 17 00:00:00 2001 From: Dag Date: Thu, 28 Dec 2023 23:26:14 +0100 Subject: [PATCH 242/716] docs(nginx, phpfpm): improve install and config instructions (#3866) --- README.md | 186 ++++++++++++++++++++++++++++++++++--------- caches/FileCache.php | 1 + index.php | 3 +- 3 files changed, 152 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 2a762d45..34efc8de 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,15 @@ ![RSS-Bridge](static/logo_600px.png) -RSS-Bridge is a web application. +RSS-Bridge is a PHP web application. It generates web feeds for websites that don't have one. Officially hosted instance: https://rss-bridge.org/bridge01/ +IRC channel #rssbridge at https://libera.chat/ + + [![LICENSE](https://img.shields.io/badge/license-UNLICENSE-blue.svg)](UNLICENSE) [![GitHub release](https://img.shields.io/github/release/rss-bridge/rss-bridge.svg?logo=github)](https://github.com/rss-bridge/rss-bridge/releases/latest) [![irc.libera.chat](https://img.shields.io/badge/irc.libera.chat-%23rssbridge-blue.svg)](https://web.libera.chat/#rssbridge) @@ -48,53 +51,146 @@ Check out RSS-Bridge right now on https://rss-bridge.org/bridge01/ Alternatively find another [public instance](https://rss-bridge.github.io/rss-bridge/General/Public_Hosts.html). -## Tutorial - -### Install with composer or git - Requires minimum PHP 7.4. +## Tutorial + +### How to install on traditional shared web hosting + +RSS-Bridge can basically be unzipped in a web folder. Should be working instantly. + +Latest zip as of Sep 2023: https://github.com/RSS-Bridge/rss-bridge/archive/refs/tags/2023-09-24.zip + +### How to install on Debian 12 (nginx + php-fpm) + +These instructions have been tested on a fresh Debian 12 VM from Digital Ocean (1vcpu-512mb-10gb, 5 USD/month). + ```shell -apt install nginx php-fpm php-mbstring php-simplexml php-curl +timedatectl set-timezone Europe/Oslo + +apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl + +# Create a new user account +useradd --shell /bin/bash --create-home rss-bridge + +cd /var/www + +# Create folder and change ownership +mkdir rss-bridge && chown rss-bridge:rss-bridge rss-bridge/ + +# Become user +su rss-bridge + +# Fetch latest master +git clone https://github.com/RSS-Bridge/rss-bridge.git rss-bridge/ +cd rss-bridge + +# Copy over the default config +cp -v config.default.ini.php config.ini.php + +# Give full permissions only to owner (rss-bridge) +chmod 700 -R ./ + +# Give read and execute to others (nginx and php-fpm) +chmod o+rx ./ ./static + +# Give read to others (nginx) +chmod o+r -R ./static ``` +Nginx config: + +```nginx +# /etc/nginx/sites-enabled/rss-bridge.conf + +server { + listen 80; + server_name example.com; + access_log /var/log/nginx/rss-bridge.access.log; + error_log /var/log/nginx/rss-bridge.error.log; + + # Intentionally not setting a root folder here + + # autoindex is off by default but feels good to explicitly turn off + autoindex off; + + # Static content only served here + location /static/ { + alias /var/www/rss-bridge/static/; + } + + # Pass off to php-fpm only when location is exactly / + location = / { + root /var/www/rss-bridge/; + include snippets/fastcgi-php.conf; + fastcgi_pass unix:/run/php/rss-bridge.sock; + } + + # Reduce spam + location = /favicon.ico { + access_log off; + log_not_found off; + } + + # Reduce spam + location = /robots.txt { + access_log off; + log_not_found off; + } +} +``` + +PHP FPM pool config: +```ini +; /etc/php/8.2/fpm/pool.d/rss-bridge.conf + +[rss-bridge] + +user = rss-bridge +group = rss-bridge + +listen = /run/php/rss-bridge.sock + +listen.owner = www-data +listen.group = www-data + +pm = static +pm.max_children = 10 +pm.max_requests = 500 +``` + +PHP ini config: +```ini +; /etc/php/8.2/fpm/conf.d/30-rss-bridge.ini + +max_execution_time = 20 +memory_limit = 64M +``` + +Restart fpm and nginx: + +```shell +# Lint and restart php-fpm +php-fpm8.2 -t +systemctl restart php8.2-fpm + +# Lint and restart nginx +nginx -t +systemctl restart nginx +``` + +### How to install from Composer + +Install the latest release. + ```shell cd /var/www composer create-project -v --no-dev rss-bridge/rss-bridge ``` -```shell -cd /var/www -git clone https://github.com/RSS-Bridge/rss-bridge.git -``` +### How to install with Caddy -Config: - -```shell -# Give the http user write permission to the cache folder -chown www-data:www-data /var/www/rss-bridge/cache - -# Optionally copy over the default config file -cp config.default.ini.php config.ini.php -``` - -Example config for nginx: - -```nginx -# /etc/nginx/sites-enabled/rssbridge -server { - listen 80; - server_name example.com; - root /var/www/rss-bridge; - index index.php; - - location ~ \.php$ { - include snippets/fastcgi-php.conf; - fastcgi_read_timeout 60s; - fastcgi_pass unix:/run/php/php-fpm.sock; - } -} -``` +TODO. See https://github.com/RSS-Bridge/rss-bridge/issues/3785 ### Install from Docker Hub: @@ -163,6 +259,22 @@ Learn more in ## How-to +### How to fix "PHP Fatal error: Uncaught Exception: The FileCache path is not writable" + +```shell +# Give rssbridge ownership +chown rssbridge:rssbridge -R /var/www/rss-bridge/cache + +# Or, give www-data ownership +chown www-data:www-data -R /var/www/rss-bridge/cache + +# Or, give everyone write permission +chmod 777 -R /var/www/rss-bridge/cache + +# Or last ditch effort (CAREFUL) +rm -rf /var/www/rss-bridge/cache/ && mkdir /var/www/rss-bridge/cache/ +``` + ### How to create a new bridge from scratch Create the new bridge in e.g. `bridges/BearBlogBridge.php`: diff --git a/caches/FileCache.php b/caches/FileCache.php index 09d12791..7a0eb81d 100644 --- a/caches/FileCache.php +++ b/caches/FileCache.php @@ -54,6 +54,7 @@ class FileCache implements CacheInterface ]; $cacheFile = $this->createCacheFile($key); $bytes = file_put_contents($cacheFile, serialize($item), LOCK_EX); + // todo: Consider tightening the permissions of the created file. It usually allow others to read, depending on umask if ($bytes === false) { // Consider just logging the error here throw new \Exception(sprintf('Failed to write to: %s', $cacheFile)); diff --git a/index.php b/index.php index 14713e06..c2c546a1 100644 --- a/index.php +++ b/index.php @@ -8,7 +8,8 @@ require_once __DIR__ . '/lib/bootstrap.php'; $errors = Configuration::checkInstallation(); if ($errors) { - die('
' . implode("\n", $errors) . '
'); + print '
' . implode("\n", $errors) . '
'; + exit(1); } $customConfig = []; From fac1f5cd88f04855a891aeb7341f783e57ce5b3c Mon Sep 17 00:00:00 2001 From: Dag Date: Sat, 30 Dec 2023 01:33:31 +0100 Subject: [PATCH 243/716] refactor(reddit) (#3869) * refactor * yup * fix also reporterre --- bridges/RedditBridge.php | 66 +++++++++++------------------------- bridges/ReporterreBridge.php | 44 +++++++++++++----------- 2 files changed, 44 insertions(+), 66 deletions(-) diff --git a/bridges/RedditBridge.php b/bridges/RedditBridge.php index bb3e7afc..618463a6 100644 --- a/bridges/RedditBridge.php +++ b/bridges/RedditBridge.php @@ -173,7 +173,7 @@ class RedditBridge extends BridgeAbstract $item['author'] = $data->author; $item['uid'] = $data->id; $item['timestamp'] = $data->created_utc; - $item['uri'] = $this->encodePermalink($data->permalink); + $item['uri'] = $this->urlEncodePathParts($data->permalink); $item['categories'] = []; @@ -193,13 +193,11 @@ class RedditBridge extends BridgeAbstract if ($post->kind == 't1') { // Comment - $item['content'] - = htmlspecialchars_decode($data->body_html); + $item['content'] = htmlspecialchars_decode($data->body_html); } elseif ($data->is_self) { // Text post - $item['content'] - = htmlspecialchars_decode($data->selftext_html); + $item['content'] = htmlspecialchars_decode($data->selftext_html); } elseif (isset($data->post_hint) && $data->post_hint == 'link') { // Link with preview @@ -215,18 +213,11 @@ class RedditBridge extends BridgeAbstract $embed = ''; } - $item['content'] = $this->template( - $data->url, - $data->thumbnail, - $data->domain - ) . $embed; - } elseif (isset($data->post_hint) ? $data->post_hint == 'image' : false) { + $item['content'] = $this->createFigureLink($data->url, $data->thumbnail, $data->domain) . $embed; + } elseif (isset($data->post_hint) && $data->post_hint == 'image') { // Single image - $item['content'] = $this->link( - $this->encodePermalink($data->permalink), - '' - ); + $item['content'] = $this->createLink($this->urlEncodePathParts($data->permalink), ''); } elseif ($data->is_gallery ?? false) { // Multiple images @@ -246,32 +237,18 @@ class RedditBridge extends BridgeAbstract end($data->preview->images[0]->resolutions); $index = key($data->preview->images[0]->resolutions); - $item['content'] = $this->template( - $data->url, - $data->preview->images[0]->resolutions[$index]->url, - 'Video' - ); - } elseif (isset($data->media) ? $data->media->type == 'youtube.com' : false) { + $item['content'] = $this->createFigureLink($data->url, $data->preview->images[0]->resolutions[$index]->url, 'Video'); + } elseif (isset($data->media) && $data->media->type == 'youtube.com') { // Youtube link - - $item['content'] = $this->template( - $data->url, - $data->media->oembed->thumbnail_url, - 'YouTube' - ); + $item['content'] = $this->createFigureLink($data->url, $data->media->oembed->thumbnail_url, 'YouTube'); + //$item['content'] = htmlspecialchars_decode($data->media->oembed->html); } elseif (explode('.', $data->domain)[0] == 'self') { // Crossposted text post // TODO (optionally?) Fetch content of the original post. - - $item['content'] = $this->link( - $this->encodePermalink($data->permalink), - 'Crossposted from r/' - . explode('.', $data->domain)[1] - ); + $item['content'] = $this->createLink($this->urlEncodePathParts($data->permalink), 'Crossposted from r/' . explode('.', $data->domain)[1]); } else { // Link WITHOUT preview - - $item['content'] = $this->link($data->url, $data->domain); + $item['content'] = $this->createLink($data->url, $data->domain); } $this->items[] = $item; @@ -279,7 +256,7 @@ class RedditBridge extends BridgeAbstract } // Sort the order to put the latest posts first, even for mixed subreddits usort($this->items, function ($a, $b) { - return $a['timestamp'] < $b['timestamp']; + return $b['timestamp'] <=> $a['timestamp']; }); } @@ -299,24 +276,19 @@ class RedditBridge extends BridgeAbstract } } - private function encodePermalink($link) + private function urlEncodePathParts($link) { - return self::URI . implode( - '/', - array_map('urlencode', explode('/', $link)) - ); + return self::URI . implode('/', array_map('urlencode', explode('/', $link))); } - private function template($href, $src, $caption) + private function createFigureLink($href, $src, $caption) { - return '
' - . $caption . '
'; + return sprintf('
%s
', $href, $caption, $src); } - private function link($href, $text) + private function createLink($href, $text) { - return '' . $text . ''; + return sprintf('%s', $href, $text); } public function detectParameters($url) diff --git a/bridges/ReporterreBridge.php b/bridges/ReporterreBridge.php index 18378d24..78c60d5f 100644 --- a/bridges/ReporterreBridge.php +++ b/bridges/ReporterreBridge.php @@ -1,11 +1,35 @@ find('item') as $element) { + if ($limit < 5) { + $item = []; + $item['title'] = html_entity_decode($element->find('title', 0)->plaintext); + $item['timestamp'] = strtotime($element->find('dc:date', 0)->plaintext); + $item['uri'] = $element->find('guid', 0)->innertext; + //$item['content'] = html_entity_decode($this->extractContent($item['uri'])); + $item['content'] = htmlspecialchars_decode($element->find('description', 0)->plaintext); + $this->items[] = $item; + $limit++; + } + } + } private function extractContent($url) { @@ -22,22 +46,4 @@ class ReporterreBridge extends BridgeAbstract $text = strip_tags($text, '


'); return $text; } - - public function collectData() - { - $html = getSimpleHTMLDOM(self::URI . 'spip.php?page=backend'); - $limit = 0; - - foreach ($html->find('item') as $element) { - if ($limit < 5) { - $item = []; - $item['title'] = html_entity_decode($element->find('title', 0)->plaintext); - $item['timestamp'] = strtotime($element->find('dc:date', 0)->plaintext); - $item['uri'] = $element->find('guid', 0)->innertext; - $item['content'] = html_entity_decode($this->extractContent($item['uri'])); - $this->items[] = $item; - $limit++; - } - } - } } From ef378663aaa98ef54c7145781e8ab1e35fe50e7d Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 2 Jan 2024 16:21:52 +0100 Subject: [PATCH 244/716] test: happy new year (#3873) * test: happy new year * yup --- tests/FeedItemTest.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/FeedItemTest.php b/tests/FeedItemTest.php index 0e7af222..3390e7b3 100644 --- a/tests/FeedItemTest.php +++ b/tests/FeedItemTest.php @@ -41,7 +41,8 @@ class FeedItemTest extends TestCase $this->assertSame(64800, $item->getTimestamp()); $item->setTimestamp('1st jan last year'); - // This will fail at 2024-01-01 hehe - $this->assertSame(1640995200, $item->getTimestamp()); + + // This will fail at 2025-01-01 hehe + $this->assertSame(1672531200, $item->getTimestamp()); } } From e904de2dc987d6578f9fd5f527aa736801c2185c Mon Sep 17 00:00:00 2001 From: Damien Calesse <2787828+kranack@users.noreply.github.com> Date: Tue, 2 Jan 2024 16:22:39 +0100 Subject: [PATCH 245/716] [YGGTorrent] Update URI (#3871) --- bridges/YGGTorrentBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridges/YGGTorrentBridge.php b/bridges/YGGTorrentBridge.php index f0c31f11..018bcfc4 100644 --- a/bridges/YGGTorrentBridge.php +++ b/bridges/YGGTorrentBridge.php @@ -7,7 +7,7 @@ class YGGTorrentBridge extends BridgeAbstract { const MAINTAINER = 'teromene'; const NAME = 'Yggtorrent Bridge'; - const URI = 'https://www5.yggtorrent.fi'; + const URI = 'https://www3.yggtorrent.qa'; const DESCRIPTION = 'Returns torrent search from Yggtorrent'; const PARAMETERS = [ From 0f6fa8034b04e1e007158ef0c5cc784bf8d7ef45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Kol=C3=A1=C5=99?= Date: Tue, 2 Jan 2024 16:23:13 +0100 Subject: [PATCH 246/716] Fixed selector in CeskaTelevizeBridge (#3872) * Fixed selector in CeskaTelevizeBridge * Fixed also description selector --- bridges/CeskaTelevizeBridge.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridges/CeskaTelevizeBridge.php b/bridges/CeskaTelevizeBridge.php index 003cd4c7..be00d664 100644 --- a/bridges/CeskaTelevizeBridge.php +++ b/bridges/CeskaTelevizeBridge.php @@ -57,9 +57,9 @@ class CeskaTelevizeBridge extends BridgeAbstract $this->feedName .= " ({$category})"; } - foreach ($html->find('#episodeListSection a[data-testid=next-link]') as $element) { + foreach ($html->find('#episodeListSection a[data-testid=card]') as $element) { $itemTitle = $element->find('h3', 0); - $itemContent = $element->find('div[class^=content-]', 0); + $itemContent = $element->find('p[class^=content-]', 0); $itemDate = $element->find('div[class^=playTime-] span', 0); $itemThumbnail = $element->find('img', 0); $itemUri = self::URI . $element->getAttribute('href'); From 12395fcf2d87939a8a95d8bbc95e188e171bfbca Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Fri, 5 Jan 2024 07:22:16 +0100 Subject: [PATCH 247/716] Docker fix default fastcgi.logging (#3875) Mistake from https://github.com/RSS-Bridge/rss-bridge/pull/3500 Wrong file extension: should have been `.ini` and not `.conf` otherwise it has no effect. See https://github.com/docker-library/php/pull/1360 and https://github.com/docker-library/php/issues/878#issuecomment-938595965 --- Dockerfile | 2 +- config/php.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f504b51f..2f1f4f3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ ENV CURL_IMPERSONATE ff91esr COPY ./config/nginx.conf /etc/nginx/sites-available/default COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf -COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.conf +COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.ini COPY --chown=www-data:www-data ./ /app/ diff --git a/config/php.ini b/config/php.ini index 115f1c89..383afffb 100644 --- a/config/php.ini +++ b/config/php.ini @@ -1,4 +1,4 @@ ; Inspired by https://github.com/docker-library/php/blob/master/8.2/bookworm/fpm/Dockerfile -; https://github.com/docker-library/php/issues/878#issuecomment-938595965' +; https://github.com/docker-library/php/issues/878#issuecomment-938595965 fastcgi.logging = Off From 55ffac5bae8d84ff1b42339d1114117cf32a6854 Mon Sep 17 00:00:00 2001 From: sysadminstory Date: Fri, 5 Jan 2024 07:23:40 +0100 Subject: [PATCH 248/716] [PepperBridgeAbstract, DealabsBridge, HotUKDealsBridge, MydealsBridge] (#3876) Fix the Deal source link The HTML does not contain the link to the "Deal source anymore", now only an attribute does contain the information about the Deal Source. The JSON data is now extraced for each Deal, and used to get the Temperature and Deal Source. --- bridges/DealabsBridge.php | 1 + bridges/HotUKDealsBridge.php | 1 + bridges/MydealsBridge.php | 1 + bridges/PepperBridgeAbstract.php | 29 +++++++++++++++++++++-------- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/bridges/DealabsBridge.php b/bridges/DealabsBridge.php index a904c3ff..4d39502c 100644 --- a/bridges/DealabsBridge.php +++ b/bridges/DealabsBridge.php @@ -1910,6 +1910,7 @@ class DealabsBridge extends PepperBridgeAbstract 'context-talk' => 'Surveillance Discussion', 'uri-group' => 'groupe/', 'uri-deal' => 'bons-plans/', + 'uri-merchant' => 'search/bons-plans?merchant-id=', 'request-error' => 'Impossible de joindre Dealabs', 'thread-error' => 'Impossible de déterminer l\'ID de la discussion. Vérifiez l\'URL que vous avez entré', 'no-results' => 'Il n'y a rien à afficher pour le moment :(', diff --git a/bridges/HotUKDealsBridge.php b/bridges/HotUKDealsBridge.php index 69301c42..a7e62250 100644 --- a/bridges/HotUKDealsBridge.php +++ b/bridges/HotUKDealsBridge.php @@ -3274,6 +3274,7 @@ class HotUKDealsBridge extends PepperBridgeAbstract 'context-talk' => 'Discussion Monitoring', 'uri-group' => 'tag/', 'uri-deal' => 'deals/', + 'uri-merchant' => 'search/deals?merchant-id=', 'request-error' => 'Could not request HotUKDeals', 'thread-error' => 'Unable to determine the thread ID. Check the URL you entered', 'no-results' => 'Ooops, looks like we could', diff --git a/bridges/MydealsBridge.php b/bridges/MydealsBridge.php index 22b46413..d7e074a9 100644 --- a/bridges/MydealsBridge.php +++ b/bridges/MydealsBridge.php @@ -2021,6 +2021,7 @@ class MydealsBridge extends PepperBridgeAbstract 'context-talk' => 'Überwachung Diskussion', 'uri-group' => 'gruppe/', 'uri-deal' => 'deals/', + 'uri-merchant' => 'search/gutscheine?merchant-id=', 'request-error' => 'Could not request mydeals', 'thread-error' => 'Die ID der Diskussion kann nicht ermittelt werden. Überprüfen Sie die eingegebene URL', 'no-results' => 'Ups, wir konnten nichts', diff --git a/bridges/PepperBridgeAbstract.php b/bridges/PepperBridgeAbstract.php index 6cb0f302..73bd194d 100644 --- a/bridges/PepperBridgeAbstract.php +++ b/bridges/PepperBridgeAbstract.php @@ -104,6 +104,9 @@ class PepperBridgeAbstract extends BridgeAbstract $item['title'] = $this->getTitle($deal); $item['author'] = $deal->find('span.thread-username', 0)->plaintext; + // Get the JSON Data stored as vue + $jsonDealData = $this->getDealJsonData($deal); + $item['content'] = '
find('div[class=js-vue2]', 0)->getAttribute('data-vue2')); - return $data['props']['thread']['temperature'] . '°'; + return $data; } /** * Get the source of a Deal if it exists * @return string String of the deal source */ - private function getSource($deal) + private function getSource($jsonData) { - if (($origin = $deal->find('button[class*=text--color-greyShade]', 0)) != null) { - $path = str_replace(' ', '/', trim(Json::decode($origin->{'data-cloak-link'})['path'])); - $text = $origin->find('span[class*=link]', 0); + if ($jsonData['props']['thread']['merchant'] != null) { + $path = $this->i8n('uri-merchant') . $jsonData['props']['thread']['merchant']['merchantId']; + $text = $jsonData['props']['thread']['merchant']['merchantName']; return ''; } else { return ''; From ea58c8d2bcd17b09e7d9dea64297ea44885a3933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B5=D0=B7=D0=B4=D0=B0=D0=BB=D0=B8=D1=81=D1=8C?= =?UTF-8?q?=D0=BA=D0=BE?= <105280814+uandreew@users.noreply.github.com> Date: Sat, 6 Jan 2024 19:13:50 +0200 Subject: [PATCH 249/716] Update 06_Public_Hosts.md (#3877) --- docs/01_General/06_Public_Hosts.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md index c9572824..4aa905da 100644 --- a/docs/01_General/06_Public_Hosts.md +++ b/docs/01_General/06_Public_Hosts.md @@ -22,6 +22,7 @@ | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.foxhaven.cyou| ![](https://img.shields.io/badge/website-up-brightgreen) | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland) | | ![](https://iplookup.flagfox.net/images/h16/PL.png) | https://rss.m3wz.su| ![](https://img.shields.io/badge/website-up-brightgreen) | [@m3oweezed](https://m3wz.su/en/about) | Poland, Hosted with Timeweb Cloud | | ![](https://iplookup.flagfox.net/images/h16/DE.png) | https://rb.ash.fail | ![](https://img.shields.io/website/https/rb.ash.fail.svg) | [@ash](https://ash.fail/contact.html) | Hosted with Hostaris, Germany +| ![](https://iplookup.flagfox.net/images/h16/UA.png) | https://rss.noleron.com | ![](https://img.shields.io/website/https/rss.noleron.com) | [@ihor](https://noleron.com/about) | Hosted with Hosting Ukraine, Ukraine ## Inactive instances From 3ce94409ab650e042993480d638482a89901776d Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 9 Jan 2024 20:18:33 +0100 Subject: [PATCH 250/716] feat: support itunes namespace in top channel feed (#3776) Also preserves other properties. --- actions/DisplayAction.php | 11 +- bridges/ItakuBridge.php | 6 +- formats/AtomFormat.php | 79 ++++++------ formats/HtmlFormat.php | 12 +- formats/JsonFormat.php | 18 +-- formats/MrssFormat.php | 118 ++++++++++-------- formats/PlaintextFormat.php | 6 +- lib/BridgeAbstract.php | 55 +++++--- lib/FormatAbstract.php | 75 +++++------ lib/bootstrap.php | 3 - tests/FormatTest.php | 72 +++++++++++ tests/Formats/BaseFormatTest.php | 2 +- .../expectedAtomFormat/feed.common.xml | 6 +- .../samples/expectedAtomFormat/feed.empty.xml | 6 +- .../expectedAtomFormat/feed.emptyItems.xml | 6 +- .../expectedAtomFormat/feed.microblog.xml | 6 +- .../expectedMrssFormat/feed.common.xml | 6 +- .../samples/expectedMrssFormat/feed.empty.xml | 2 +- .../expectedMrssFormat/feed.emptyItems.xml | 2 +- .../expectedMrssFormat/feed.microblog.xml | 6 +- tests/Formats/samples/feed.empty.json | 2 +- tests/Formats/samples/feed.emptyItems.json | 2 +- 22 files changed, 298 insertions(+), 203 deletions(-) create mode 100644 tests/FormatTest.php diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php index 43563996..080da52e 100644 --- a/actions/DisplayAction.php +++ b/actions/DisplayAction.php @@ -100,7 +100,7 @@ class DisplayAction implements ActionInterface private function createResponse(array $request, BridgeAbstract $bridge, FormatAbstract $format) { $items = []; - $infos = []; + $feed = []; try { $bridge->loadConfiguration(); @@ -116,12 +116,7 @@ class DisplayAction implements ActionInterface } $items = $feedItems; } - $infos = [ - 'name' => $bridge->getName(), - 'uri' => $bridge->getURI(), - 'donationUri' => $bridge->getDonationURI(), - 'icon' => $bridge->getIcon() - ]; + $feed = $bridge->getFeed(); } catch (\Exception $e) { if ($e instanceof HttpException) { // Reproduce (and log) these responses regardless of error output and report limit @@ -155,7 +150,7 @@ class DisplayAction implements ActionInterface } $format->setItems($items); - $format->setExtraInfos($infos); + $format->setFeed($feed); $now = time(); $format->setLastModified($now); $headers = [ diff --git a/bridges/ItakuBridge.php b/bridges/ItakuBridge.php index 149757f5..0577752c 100644 --- a/bridges/ItakuBridge.php +++ b/bridges/ItakuBridge.php @@ -280,7 +280,7 @@ class ItakuBridge extends BridgeAbstract $opt['range'] = ''; $user_id = $this->getInput('user_id') ?? $this->getOwnerID($this->getInput('user')); - $data = $this->getFeed( + $data = $this->getFeedData( $opt, $user_id ); @@ -289,7 +289,7 @@ class ItakuBridge extends BridgeAbstract if ($this->queriedContext === 'Home feed') { $opt['order'] = $this->getInput('order'); $opt['range'] = $this->getInput('range'); - $data = $this->getFeed($opt); + $data = $this->getFeedData($opt); } foreach ($data['results'] as $record) { @@ -409,7 +409,7 @@ class ItakuBridge extends BridgeAbstract return $this->getData($url, false, true); } - private function getFeed(array $opt, $ownerID = null) + private function getFeedData(array $opt, $ownerID = null) { $url = self::URI . "/api/feed/?date_range={$opt['range']}&ordering={$opt['order']}&page=1&page_size=30&format=json"; diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index 07ca7272..1fabef2e 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -17,44 +17,61 @@ class AtomFormat extends FormatAbstract public function stringify() { $document = new \DomDocument('1.0', $this->getCharset()); + $document->formatOutput = true; $feedUrl = get_current_url(); - $extraInfos = $this->getExtraInfos(); - if (empty($extraInfos['uri'])) { - $uri = REPOSITORY; - } else { - $uri = $extraInfos['uri']; - } - - $document->formatOutput = true; $feed = $document->createElementNS(self::ATOM_NS, 'feed'); $document->appendChild($feed); $feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:media', self::MRSS_NS); - $title = $document->createElement('title'); - $feed->appendChild($title); - $title->setAttribute('type', 'text'); - $title->appendChild($document->createTextNode($extraInfos['name'])); + $feedArray = $this->getFeed(); + foreach ($feedArray as $feedKey => $feedValue) { + if (in_array($feedKey, ['donationUri'])) { + continue; + } + if ($feedKey === 'name') { + $title = $document->createElement('title'); + $feed->appendChild($title); + $title->setAttribute('type', 'text'); + $title->appendChild($document->createTextNode($feedValue)); + } elseif ($feedKey === 'icon') { + if ($feedValue) { + $icon = $document->createElement('icon'); + $feed->appendChild($icon); + $icon->appendChild($document->createTextNode($feedValue)); + + $logo = $document->createElement('logo'); + $feed->appendChild($logo); + $logo->appendChild($document->createTextNode($feedValue)); + } + } elseif ($feedKey === 'uri') { + if ($feedValue) { + $linkAlternate = $document->createElement('link'); + $feed->appendChild($linkAlternate); + $linkAlternate->setAttribute('rel', 'alternate'); + $linkAlternate->setAttribute('type', 'text/html'); + $linkAlternate->setAttribute('href', $feedValue); + + $linkSelf = $document->createElement('link'); + $feed->appendChild($linkSelf); + $linkSelf->setAttribute('rel', 'self'); + $linkSelf->setAttribute('type', 'application/atom+xml'); + $linkSelf->setAttribute('href', $feedUrl); + } + } elseif ($feedKey === 'itunes') { + // todo: skip? + } else { + $element = $document->createElement($feedKey); + $feed->appendChild($element); + $element->appendChild($document->createTextNode($feedValue)); + } + } $id = $document->createElement('id'); $feed->appendChild($id); $id->appendChild($document->createTextNode($feedUrl)); - $uriparts = parse_url($uri); - if (empty($extraInfos['icon'])) { - $iconUrl = $uriparts['scheme'] . '://' . $uriparts['host'] . '/favicon.ico'; - } else { - $iconUrl = $extraInfos['icon']; - } - $icon = $document->createElement('icon'); - $feed->appendChild($icon); - $icon->appendChild($document->createTextNode($iconUrl)); - - $logo = $document->createElement('logo'); - $feed->appendChild($logo); - $logo->appendChild($document->createTextNode($iconUrl)); - $feedTimestamp = gmdate(DATE_ATOM, $this->lastModified); $updated = $document->createElement('updated'); $feed->appendChild($updated); @@ -69,17 +86,7 @@ class AtomFormat extends FormatAbstract $author->appendChild($authorName); $authorName->appendChild($document->createTextNode($feedAuthor)); - $linkAlternate = $document->createElement('link'); - $feed->appendChild($linkAlternate); - $linkAlternate->setAttribute('rel', 'alternate'); - $linkAlternate->setAttribute('type', 'text/html'); - $linkAlternate->setAttribute('href', $uri); - $linkSelf = $document->createElement('link'); - $feed->appendChild($linkSelf); - $linkSelf->setAttribute('rel', 'self'); - $linkSelf->setAttribute('type', 'application/atom+xml'); - $linkSelf->setAttribute('href', $feedUrl); foreach ($this->getItems() as $item) { $itemArray = $item->toArray(); diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php index 4933af8d..ef66f493 100644 --- a/formats/HtmlFormat.php +++ b/formats/HtmlFormat.php @@ -8,7 +8,7 @@ class HtmlFormat extends FormatAbstract { $queryString = $_SERVER['QUERY_STRING']; - $extraInfos = $this->getExtraInfos(); + $feedArray = $this->getFeed(); $formatFactory = new FormatFactory(); $buttons = []; $linkTags = []; @@ -29,9 +29,9 @@ class HtmlFormat extends FormatAbstract ]; } - if (Configuration::getConfig('admin', 'donations') && $extraInfos['donationUri'] !== '') { + if (Configuration::getConfig('admin', 'donations') && $feedArray['donationUri']) { $buttons[] = [ - 'href' => e($extraInfos['donationUri']), + 'href' => e($feedArray['donationUri']), 'value' => 'Donate to maintainer', ]; } @@ -39,7 +39,7 @@ class HtmlFormat extends FormatAbstract $items = []; foreach ($this->getItems() as $item) { $items[] = [ - 'url' => $item->getURI() ?: $extraInfos['uri'], + 'url' => $item->getURI() ?: $feedArray['uri'], 'title' => $item->getTitle() ?? '(no title)', 'timestamp' => $item->getTimestamp(), 'author' => $item->getAuthor(), @@ -51,9 +51,9 @@ class HtmlFormat extends FormatAbstract $html = render_template(__DIR__ . '/../templates/html-format.html.php', [ 'charset' => $this->getCharset(), - 'title' => $extraInfos['name'], + 'title' => $feedArray['name'], 'linkTags' => $linkTags, - 'uri' => $extraInfos['uri'], + 'uri' => $feedArray['uri'], 'buttons' => $buttons, 'items' => $items, ]); diff --git a/formats/JsonFormat.php b/formats/JsonFormat.php index dd61da41..016e75e1 100644 --- a/formats/JsonFormat.php +++ b/formats/JsonFormat.php @@ -25,18 +25,18 @@ class JsonFormat extends FormatAbstract public function stringify() { - $host = $_SERVER['HTTP_HOST'] ?? ''; - $extraInfos = $this->getExtraInfos(); + $feedArray = $this->getFeed(); + $data = [ - 'version' => 'https://jsonfeed.org/version/1', - 'title' => empty($extraInfos['name']) ? $host : $extraInfos['name'], - 'home_page_url' => empty($extraInfos['uri']) ? REPOSITORY : $extraInfos['uri'], - 'feed_url' => get_current_url(), + 'version' => 'https://jsonfeed.org/version/1', + 'title' => $feedArray['name'], + 'home_page_url' => $feedArray['uri'], + 'feed_url' => get_current_url(), ]; - if (!empty($extraInfos['icon'])) { - $data['icon'] = $extraInfos['icon']; - $data['favicon'] = $extraInfos['icon']; + if ($feedArray['icon']) { + $data['icon'] = $feedArray['icon']; + $data['favicon'] = $feedArray['icon']; } $items = []; diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index 5b96a6a7..e93a8289 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -35,16 +35,8 @@ class MrssFormat extends FormatAbstract public function stringify() { $document = new \DomDocument('1.0', $this->getCharset()); - - $feedUrl = get_current_url(); - $extraInfos = $this->getExtraInfos(); - if (empty($extraInfos['uri'])) { - $uri = REPOSITORY; - } else { - $uri = $extraInfos['uri']; - } - $document->formatOutput = true; + $feed = $document->createElement('rss'); $document->appendChild($feed); $feed->setAttribute('version', '2.0'); @@ -54,51 +46,74 @@ class MrssFormat extends FormatAbstract $channel = $document->createElement('channel'); $feed->appendChild($channel); - $title = $extraInfos['name']; - $channelTitle = $document->createElement('title'); - $channel->appendChild($channelTitle); - $channelTitle->appendChild($document->createTextNode($title)); + $feedArray = $this->getFeed(); + $uri = $feedArray['uri']; + $title = $feedArray['name']; - $link = $document->createElement('link'); - $channel->appendChild($link); - $link->appendChild($document->createTextNode($uri)); + foreach ($feedArray as $feedKey => $feedValue) { + if (in_array($feedKey, ['atom', 'donationUri'])) { + continue; + } + if ($feedKey === 'name') { + $channelTitle = $document->createElement('title'); + $channel->appendChild($channelTitle); + $channelTitle->appendChild($document->createTextNode($title)); - $description = $document->createElement('description'); - $channel->appendChild($description); - $description->appendChild($document->createTextNode($extraInfos['name'])); + $description = $document->createElement('description'); + $channel->appendChild($description); + $description->appendChild($document->createTextNode($title)); + } elseif ($feedKey === 'uri') { + $link = $document->createElement('link'); + $channel->appendChild($link); + $link->appendChild($document->createTextNode($uri)); - $allowedIconExtensions = [ - '.gif', - '.jpg', - '.png', - ]; - $icon = $extraInfos['icon']; - if (!empty($icon) && in_array(substr($icon, -4), $allowedIconExtensions)) { - $feedImage = $document->createElement('image'); - $channel->appendChild($feedImage); - $iconUrl = $document->createElement('url'); - $iconUrl->appendChild($document->createTextNode($icon)); - $feedImage->appendChild($iconUrl); - $iconTitle = $document->createElement('title'); - $iconTitle->appendChild($document->createTextNode($title)); - $feedImage->appendChild($iconTitle); - $iconLink = $document->createElement('link'); - $iconLink->appendChild($document->createTextNode($uri)); - $feedImage->appendChild($iconLink); + $linkAlternate = $document->createElementNS(self::ATOM_NS, 'link'); + $channel->appendChild($linkAlternate); + $linkAlternate->setAttribute('rel', 'alternate'); + $linkAlternate->setAttribute('type', 'text/html'); + $linkAlternate->setAttribute('href', $uri); + + $linkSelf = $document->createElementNS(self::ATOM_NS, 'link'); + $channel->appendChild($linkSelf); + $linkSelf->setAttribute('rel', 'self'); + $linkSelf->setAttribute('type', 'application/atom+xml'); + $feedUrl = get_current_url(); + $linkSelf->setAttribute('href', $feedUrl); + } elseif ($feedKey === 'icon') { + $allowedIconExtensions = [ + '.gif', + '.jpg', + '.png', + '.ico', + ]; + $icon = $feedValue; + if ($icon && in_array(substr($icon, -4), $allowedIconExtensions)) { + $feedImage = $document->createElement('image'); + $channel->appendChild($feedImage); + $iconUrl = $document->createElement('url'); + $iconUrl->appendChild($document->createTextNode($icon)); + $feedImage->appendChild($iconUrl); + $iconTitle = $document->createElement('title'); + $iconTitle->appendChild($document->createTextNode($title)); + $feedImage->appendChild($iconTitle); + $iconLink = $document->createElement('link'); + $iconLink->appendChild($document->createTextNode($uri)); + $feedImage->appendChild($iconLink); + } + } elseif ($feedKey === 'itunes') { + $feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:itunes', self::ITUNES_NS); + foreach ($feedValue as $itunesKey => $itunesValue) { + $itunesProperty = $document->createElementNS(self::ITUNES_NS, $itunesKey); + $channel->appendChild($itunesProperty); + $itunesProperty->appendChild($document->createTextNode($itunesValue)); + } + } else { + $element = $document->createElement($feedKey); + $channel->appendChild($element); + $element->appendChild($document->createTextNode($feedValue)); + } } - $linkAlternate = $document->createElementNS(self::ATOM_NS, 'link'); - $channel->appendChild($linkAlternate); - $linkAlternate->setAttribute('rel', 'alternate'); - $linkAlternate->setAttribute('type', 'text/html'); - $linkAlternate->setAttribute('href', $uri); - - $linkSelf = $document->createElementNS(self::ATOM_NS, 'link'); - $channel->appendChild($linkSelf); - $linkSelf->setAttribute('rel', 'self'); - $linkSelf->setAttribute('type', 'application/atom+xml'); - $linkSelf->setAttribute('href', $feedUrl); - foreach ($this->getItems() as $item) { $itemArray = $item->toArray(); $itemTimestamp = $item->getTimestamp(); @@ -135,6 +150,7 @@ class MrssFormat extends FormatAbstract $entry->appendChild($itunesProperty); $itunesProperty->appendChild($document->createTextNode($itunesValue)); } + if (isset($itemArray['enclosure'])) { $itunesEnclosure = $document->createElement('enclosure'); $entry->appendChild($itunesEnclosure); @@ -142,7 +158,9 @@ class MrssFormat extends FormatAbstract $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']); $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']); } - } if (!empty($itemUri)) { + } + + if (!empty($itemUri)) { $entryLink = $document->createElement('link'); $entry->appendChild($entryLink); $entryLink->appendChild($document->createTextNode($itemUri)); diff --git a/formats/PlaintextFormat.php b/formats/PlaintextFormat.php index 0a9237d0..4e18caa6 100644 --- a/formats/PlaintextFormat.php +++ b/formats/PlaintextFormat.php @@ -6,11 +6,11 @@ class PlaintextFormat extends FormatAbstract public function stringify() { - $data = []; + $feed = $this->getFeed(); foreach ($this->getItems() as $item) { - $data[] = $item->toArray(); + $feed['items'][] = $item->toArray(); } - $text = print_r($data, true); + $text = print_r($feed, true); // Remove invalid non-UTF8 characters ini_set('mbstring.substitute_character', 'none'); $text = mb_convert_encoding($text, $this->getCharset(), 'UTF-8'); diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php index 0f86f454..8001ba4f 100644 --- a/lib/BridgeAbstract.php +++ b/lib/BridgeAbstract.php @@ -40,9 +40,38 @@ abstract class BridgeAbstract abstract public function collectData(); - public function getItems() + public function getFeed(): array { - return $this->items; + return [ + 'name' => $this->getName(), + 'uri' => $this->getURI(), + 'donationUri' => $this->getDonationURI(), + 'icon' => $this->getIcon(), + ]; + } + + public function getName() + { + return static::NAME; + } + + public function getURI() + { + return static::URI ?? 'https://github.com/RSS-Bridge/rss-bridge/'; + } + + public function getDonationURI(): string + { + return static::DONATION_URI; + } + + public function getIcon() + { + if (static::URI) { + // This favicon may or may not exist + return rtrim(static::URI, '/') . '/favicon.ico'; + } + return ''; } public function getOption(string $name) @@ -50,6 +79,9 @@ abstract class BridgeAbstract return $this->configuration[$name] ?? null; } + /** + * The description is currently not used in feed production + */ public function getDescription() { return static::DESCRIPTION; @@ -60,29 +92,14 @@ abstract class BridgeAbstract return static::MAINTAINER; } - public function getName() - { - return static::NAME; - } - - public function getIcon() - { - return static::URI . '/favicon.ico'; - } - public function getParameters(): array { return static::PARAMETERS; } - public function getURI() + public function getItems() { - return static::URI; - } - - public function getDonationURI(): string - { - return static::DONATION_URI; + return $this->items; } public function getCacheTimeout() diff --git a/lib/FormatAbstract.php b/lib/FormatAbstract.php index c76d1e42..28eb4bbf 100644 --- a/lib/FormatAbstract.php +++ b/lib/FormatAbstract.php @@ -9,10 +9,43 @@ abstract class FormatAbstract protected string $charset = 'UTF-8'; protected array $items = []; protected int $lastModified; - protected array $extraInfos = []; + + protected array $feed = []; abstract public function stringify(); + public function setFeed(array $feed) + { + $default = [ + 'name' => '', + 'uri' => '', + 'icon' => '', + 'donationUri' => '', + ]; + $this->feed = array_merge($default, $feed); + } + + public function getFeed(): array + { + return $this->feed; + } + + /** + * @param FeedItem[] $items + */ + public function setItems(array $items): void + { + $this->items = $items; + } + + /** + * @return FeedItem[] The items + */ + public function getItems(): array + { + return $this->items; + } + public function getMimeType(): string { return static::MIME_TYPE; @@ -32,44 +65,4 @@ abstract class FormatAbstract { $this->lastModified = $lastModified; } - - /** - * @param FeedItem[] $items - */ - public function setItems(array $items): void - { - $this->items = $items; - } - - /** - * @return FeedItem[] The items - */ - public function getItems(): array - { - return $this->items; - } - - public function setExtraInfos(array $infos = []) - { - $extras = [ - 'name', - 'uri', - 'icon', - 'donationUri', - ]; - foreach ($extras as $extra) { - if (!isset($infos[$extra])) { - $infos[$extra] = ''; - } - } - $this->extraInfos = $infos; - } - - public function getExtraInfos(): array - { - if (!$this->extraInfos) { - $this->setExtraInfos(); - } - return $this->extraInfos; - } } diff --git a/lib/bootstrap.php b/lib/bootstrap.php index a95de9dd..85d823e9 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -9,9 +9,6 @@ const PATH_LIB_CACHES = __DIR__ . '/../caches/'; /** Path to the cache folder */ const PATH_CACHE = __DIR__ . '/../cache/'; -/** URL to the RSS-Bridge repository */ -const REPOSITORY = 'https://github.com/RSS-Bridge/rss-bridge/'; - // Allow larger files for simple_html_dom // todo: extract to config (if possible) const MAX_FILE_SIZE = 10000000; diff --git a/tests/FormatTest.php b/tests/FormatTest.php new file mode 100644 index 00000000..b5df395c --- /dev/null +++ b/tests/FormatTest.php @@ -0,0 +1,72 @@ + '', + 'uri' => '', + 'icon' => '', + 'donationUri' => '', + ]; + $this->assertEquals([], $sut->getFeed()); + + $sut->setFeed([ + 'name' => '0', + 'uri' => '1', + 'icon' => '2', + 'donationUri' => '3', + ]); + $expected = [ + 'name' => '0', + 'uri' => '1', + 'icon' => '2', + 'donationUri' => '3', + ]; + $this->assertEquals($expected, $sut->getFeed()); + + $sut->setFeed([]); + $expected = [ + 'name' => '', + 'uri' => '', + 'icon' => '', + 'donationUri' => '', + ]; + $this->assertEquals($expected, $sut->getFeed()); + + $sut->setFeed(['foo' => 'bar', 'foo2' => 'bar2']); + $expected = [ + 'name' => '', + 'uri' => '', + 'icon' => '', + 'donationUri' => '', + 'foo' => 'bar', + 'foo2' => 'bar2', + ]; + $this->assertEquals($expected, $sut->getFeed()); + } +} + +class TestFormat extends \FormatAbstract +{ + public function stringify() + { + } +} + +class TestBridge extends \BridgeAbstract +{ + public function collectData() + { + $this->items[] = ['title' => 'kek']; + } +} diff --git a/tests/Formats/BaseFormatTest.php b/tests/Formats/BaseFormatTest.php index 71e196f0..8999e772 100644 --- a/tests/Formats/BaseFormatTest.php +++ b/tests/Formats/BaseFormatTest.php @@ -61,7 +61,7 @@ abstract class BaseFormatTest extends TestCase $formatFactory = new FormatFactory(); $format = $formatFactory->create($formatName); $format->setItems($sample->items); - $format->setExtraInfos($sample->meta); + $format->setFeed($sample->meta); $format->setLastModified(strtotime('2000-01-01 12:00:00 UTC')); return $format->stringify(); diff --git a/tests/Formats/samples/expectedAtomFormat/feed.common.xml b/tests/Formats/samples/expectedAtomFormat/feed.common.xml index aa6d0687..455e5440 100644 --- a/tests/Formats/samples/expectedAtomFormat/feed.common.xml +++ b/tests/Formats/samples/expectedAtomFormat/feed.common.xml @@ -2,15 +2,15 @@ Sample feed with common data - https://example.com/feed?type=common&items=4 + + https://example.com/logo.png https://example.com/logo.png + https://example.com/feed?type=common&items=4 2000-01-01T12:00:00+00:00 RSS-Bridge - - Test Entry diff --git a/tests/Formats/samples/expectedAtomFormat/feed.empty.xml b/tests/Formats/samples/expectedAtomFormat/feed.empty.xml index fc04304d..083f230f 100644 --- a/tests/Formats/samples/expectedAtomFormat/feed.empty.xml +++ b/tests/Formats/samples/expectedAtomFormat/feed.empty.xml @@ -2,14 +2,12 @@ Sample feed with minimum data + + https://example.com/feed - https://github.com/favicon.ico - https://github.com/favicon.ico 2000-01-01T12:00:00+00:00 RSS-Bridge - - diff --git a/tests/Formats/samples/expectedAtomFormat/feed.emptyItems.xml b/tests/Formats/samples/expectedAtomFormat/feed.emptyItems.xml index 18572fac..d7cb461a 100644 --- a/tests/Formats/samples/expectedAtomFormat/feed.emptyItems.xml +++ b/tests/Formats/samples/expectedAtomFormat/feed.emptyItems.xml @@ -2,15 +2,13 @@ Sample feed with minimum data + + https://example.com/feed - https://github.com/favicon.ico - https://github.com/favicon.ico 2000-01-01T12:00:00+00:00 RSS-Bridge - - Sample Item #1 diff --git a/tests/Formats/samples/expectedAtomFormat/feed.microblog.xml b/tests/Formats/samples/expectedAtomFormat/feed.microblog.xml index 32bc0273..8eb0133c 100644 --- a/tests/Formats/samples/expectedAtomFormat/feed.microblog.xml +++ b/tests/Formats/samples/expectedAtomFormat/feed.microblog.xml @@ -2,15 +2,15 @@ Sample microblog feed - https://example.com/feed + + https://example.com/logo.png https://example.com/logo.png + https://example.com/feed 2000-01-01T12:00:00+00:00 RSS-Bridge - - Oh 😲 I found three monkeys 🙈🙉🙊 diff --git a/tests/Formats/samples/expectedMrssFormat/feed.common.xml b/tests/Formats/samples/expectedMrssFormat/feed.common.xml index 38a16f88..92838ae8 100644 --- a/tests/Formats/samples/expectedMrssFormat/feed.common.xml +++ b/tests/Formats/samples/expectedMrssFormat/feed.common.xml @@ -2,15 +2,15 @@ Sample feed with common data - https://example.com/blog/ Sample feed with common data + https://example.com/blog/ + + https://example.com/logo.png Sample feed with common data https://example.com/blog/ - - Test Entry diff --git a/tests/Formats/samples/expectedMrssFormat/feed.empty.xml b/tests/Formats/samples/expectedMrssFormat/feed.empty.xml index 888c42b6..40eecfc6 100644 --- a/tests/Formats/samples/expectedMrssFormat/feed.empty.xml +++ b/tests/Formats/samples/expectedMrssFormat/feed.empty.xml @@ -2,8 +2,8 @@ Sample feed with minimum data - https://github.com/RSS-Bridge/rss-bridge/ Sample feed with minimum data + https://github.com/RSS-Bridge/rss-bridge/ diff --git a/tests/Formats/samples/expectedMrssFormat/feed.emptyItems.xml b/tests/Formats/samples/expectedMrssFormat/feed.emptyItems.xml index 9e712ddd..8839f5a5 100644 --- a/tests/Formats/samples/expectedMrssFormat/feed.emptyItems.xml +++ b/tests/Formats/samples/expectedMrssFormat/feed.emptyItems.xml @@ -2,8 +2,8 @@ Sample feed with minimum data - https://github.com/RSS-Bridge/rss-bridge/ Sample feed with minimum data + https://github.com/RSS-Bridge/rss-bridge/ diff --git a/tests/Formats/samples/expectedMrssFormat/feed.microblog.xml b/tests/Formats/samples/expectedMrssFormat/feed.microblog.xml index 81dac87a..63c04c0f 100644 --- a/tests/Formats/samples/expectedMrssFormat/feed.microblog.xml +++ b/tests/Formats/samples/expectedMrssFormat/feed.microblog.xml @@ -2,15 +2,15 @@ Sample microblog feed - https://example.com/blog/ Sample microblog feed + https://example.com/blog/ + + https://example.com/logo.png Sample microblog feed https://example.com/blog/ - - 1918f084648b82057c1dd3faa3d091da82a6fac2 diff --git a/tests/Formats/samples/feed.empty.json b/tests/Formats/samples/feed.empty.json index aac09f64..7b1a2eae 100644 --- a/tests/Formats/samples/feed.empty.json +++ b/tests/Formats/samples/feed.empty.json @@ -6,7 +6,7 @@ }, "meta": { "name": "Sample feed with minimum data", - "uri": "", + "uri": "https://github.com/RSS-Bridge/rss-bridge/", "icon": "" }, "items": [] diff --git a/tests/Formats/samples/feed.emptyItems.json b/tests/Formats/samples/feed.emptyItems.json index 0287d428..4d077487 100644 --- a/tests/Formats/samples/feed.emptyItems.json +++ b/tests/Formats/samples/feed.emptyItems.json @@ -6,7 +6,7 @@ }, "meta": { "name": "Sample feed with minimum data", - "uri": "", + "uri": "https://github.com/RSS-Bridge/rss-bridge/", "icon": "" }, "items": [ From 0bf5dbbc0ba46cc27fe40b554b0c3c0ba705ef8b Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 9 Jan 2024 20:33:35 +0100 Subject: [PATCH 251/716] chore: add tools for manually administrating the configured cache (#3867) --- README.md | 36 +++++++++++++++++++++++--- bridges/PixivBridge.php | 29 ++++++++++----------- docs/10_Bridge_Specific/PixivBridge.md | 15 ++++++++--- index.php | 25 +++--------------- lib/CacheFactory.php | 1 + lib/Configuration.php | 2 +- lib/bootstrap.php | 15 +++++++++++ lib/logger.php | 1 + phpcs.xml | 8 +++++- templates/exception.html.php | 8 ++++++ 10 files changed, 95 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 34efc8de..46bb5a69 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ Browse http://localhost:3000/ [![Deploy to Cloudron](https://cloudron.io/img/button.svg)](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html) [![Run on PikaPods](https://www.pikapods.com/static/run-button.svg)](https://www.pikapods.com/pods?run=rssbridge) -The Heroku quick deploy currently does not work. It might possibly work if you fork this repo and +The Heroku quick deploy currently does not work. It might work if you fork this repo and modify the `repository` in `scalingo.json`. See https://github.com/RSS-Bridge/rss-bridge/issues/2688 Learn more in @@ -259,11 +259,29 @@ Learn more in ## How-to +### How to remove all cache items + +As current user: + + bin/cache-clear + +As user rss-bridge: + + sudo -u rss-bridge bin/cache-clear + +As root: + + sudo bin/cache-clear + +### How to remove all expired cache items + + bin/cache-clear + ### How to fix "PHP Fatal error: Uncaught Exception: The FileCache path is not writable" ```shell -# Give rssbridge ownership -chown rssbridge:rssbridge -R /var/www/rss-bridge/cache +# Give rss-bridge ownership +chown rss-bridge:rss-bridge -R /var/www/rss-bridge/cache # Or, give www-data ownership chown www-data:www-data -R /var/www/rss-bridge/cache @@ -275,6 +293,16 @@ chmod 777 -R /var/www/rss-bridge/cache rm -rf /var/www/rss-bridge/cache/ && mkdir /var/www/rss-bridge/cache/ ``` +### How to fix "attempt to write a readonly database" + +The sqlite files (db, wal and shm) are not writeable. + + chown -v rss-bridge:rss-bridge cache/* + +### How to fix "Unable to prepare statement: 1, no such table: storage" + + rm cache/* + ### How to create a new bridge from scratch Create the new bridge in e.g. `bridges/BearBlogBridge.php`: @@ -389,6 +417,8 @@ These commands require that you have installed the dev dependencies in `composer ./vendor/bin/phpunit ./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./ +https://github.com/squizlabs/PHP_CodeSniffer/wiki + ### How to spawn a minimal development environment php -S 127.0.0.1:9001 diff --git a/bridges/PixivBridge.php b/bridges/PixivBridge.php index c4f5277f..fc4443ed 100644 --- a/bridges/PixivBridge.php +++ b/bridges/PixivBridge.php @@ -1,9 +1,11 @@ [ 'posts' => [ @@ -251,14 +252,13 @@ class PixivBridge extends BridgeAbstract $img_url = preg_replace('/https:\/\/i\.pximg\.net/', $proxy_url, $result['url']); } } else { - //else cache and use image. - $img_url = $this->cacheImage( - $result['url'], - $result['id'], - array_key_exists('illustType', $result) - ); + $img_url = $result['url']; + // Temporarily disabling caching of the image + //$img_url = $this->cacheImage($result['url'], $result['id'], array_key_exists('illustType', $result)); } - $item['content'] = ""; + + // Currently, this might result in broken image due to their strict referrer check + $item['content'] = sprintf('', $img_url, $img_url); // Additional content items if (array_key_exists('pageCount', $result)) { @@ -318,7 +318,7 @@ class PixivBridge extends BridgeAbstract if ( !(strlen($proxy) > 0 && preg_match('/https?:\/\/.*/', $proxy)) ) { - return returnServerError('Invalid proxy_url value set. The proxy must include the HTTP/S at the beginning of the url.'); + returnServerError('Invalid proxy_url value set. The proxy must include the HTTP/S at the beginning of the url.'); } } @@ -326,8 +326,7 @@ class PixivBridge extends BridgeAbstract if ($cookie) { $isAuth = $this->loadCacheValue('is_authenticated'); if (!$isAuth) { - $res = $this->getData('https://www.pixiv.net/ajax/webpush', true, true) - or returnServerError('Invalid PHPSESSID cookie provided. Please check the 🍪 and try again.'); + $res = $this->getData('https://www.pixiv.net/ajax/webpush', true, true); if ($res['error'] === false) { $this->saveCacheValue('is_authenticated', true); } @@ -374,11 +373,11 @@ class PixivBridge extends BridgeAbstract if ($cache) { $data = $this->loadCacheValue($url); if (!$data) { - $data = getContents($url, $httpHeaders, $curlOptions, true) or returnServerError("Could not load $url"); + $data = getContents($url, $httpHeaders, $curlOptions, true); $this->saveCacheValue($url, $data); } } else { - $data = getContents($url, $httpHeaders, $curlOptions, true) or returnServerError("Could not load $url"); + $data = getContents($url, $httpHeaders, $curlOptions, true); } $this->checkCookie($data['headers']); diff --git a/docs/10_Bridge_Specific/PixivBridge.md b/docs/10_Bridge_Specific/PixivBridge.md index b782a445..ba8da2d8 100644 --- a/docs/10_Bridge_Specific/PixivBridge.md +++ b/docs/10_Bridge_Specific/PixivBridge.md @@ -2,9 +2,14 @@ PixivBridge =============== # Image proxy -As Pixiv requires images to be loaded with the `Referer "https://www.pixiv.net/"` header set, caching or image proxy is required to use this bridge. -To turn off image caching, set the `proxy_url` value in this bridge's configuration section of `config.ini.php` to the url of the proxy. The bridge will then use the proxy in this format (essentially replacing `https://i.pximg.net` with the proxy): +As Pixiv requires images to be loaded with the `Referer "https://www.pixiv.net/"` header set, +caching or image proxy is required to use this bridge. + +To turn off image caching, set the `proxy_url` value in this bridge's configuration section of `config.ini.php` +to the url of the proxy. + +The bridge will then use the proxy in this format (essentially replacing `https://i.pximg.net` with the proxy): Before: `https://i.pximg.net/img-original/img/0000/00/00/00/00/00/12345678_p0.png` @@ -15,9 +20,11 @@ proxy_url = "https://proxy.example.com" ``` # Authentication -Authentication is required to view and search R-18+ and non-public images. To enable this, set the following in this bridge's configuration in `config.ini.php`. -``` +Authentication is required to view and search R-18+ and non-public images. +To enable this, set the following in this bridge's configuration in `config.ini.php`. + +```ini ; from cookie "PHPSESSID". Recommend to get in incognito browser. cookie = "00000000_hashedsessionidhere" ``` \ No newline at end of file diff --git a/index.php b/index.php index c2c546a1..126200da 100644 --- a/index.php +++ b/index.php @@ -1,33 +1,14 @@ ' . implode("\n", $errors) . ''; - exit(1); -} - -$customConfig = []; -if (file_exists(__DIR__ . '/config.ini.php')) { - $customConfig = parse_ini_file(__DIR__ . '/config.ini.php', true, INI_SCANNER_TYPED); -} -Configuration::loadConfiguration($customConfig, getenv()); - // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED); date_default_timezone_set(Configuration::getConfig('system', 'timezone')); -$rssBridge = new RssBridge(); - set_exception_handler(function (\Throwable $e) { - http_response_code(500); - print render(__DIR__ . '/templates/exception.html.php', ['e' => $e]); RssBridge::getLogger()->error('Uncaught Exception', ['e' => $e]); - exit(1); + http_response_code(500); + exit(render(__DIR__ . '/templates/exception.html.php', ['e' => $e])); }); set_error_handler(function ($code, $message, $file, $line) { @@ -63,4 +44,6 @@ register_shutdown_function(function () { } }); +$rssBridge = new RssBridge(); + $rssBridge->main($argv ?? []); diff --git a/lib/CacheFactory.php b/lib/CacheFactory.php index df78d9cb..90aa21ba 100644 --- a/lib/CacheFactory.php +++ b/lib/CacheFactory.php @@ -37,6 +37,7 @@ class CacheFactory if ($index === false) { throw new \InvalidArgumentException(sprintf('Invalid cache name: "%s"', $name)); } + $className = $cacheNames[$index] . 'Cache'; if (!preg_match('/^[A-Z][a-zA-Z0-9-]*$/', $className)) { throw new \InvalidArgumentException(sprintf('Invalid cache classname: "%s"', $className)); diff --git a/lib/Configuration.php b/lib/Configuration.php index ac7d29bf..ab1c9cdf 100644 --- a/lib/Configuration.php +++ b/lib/Configuration.php @@ -59,7 +59,7 @@ final class Configuration } $config = parse_ini_file(__DIR__ . '/../config.default.ini.php', true, INI_SCANNER_TYPED); if (!$config) { - throw new \Exception('Error parsing config'); + throw new \Exception('Error parsing ini config'); } foreach ($config as $header => $section) { foreach ($section as $key => $value) { diff --git a/lib/bootstrap.php b/lib/bootstrap.php index 85d823e9..fe2069d3 100644 --- a/lib/bootstrap.php +++ b/lib/bootstrap.php @@ -1,5 +1,9 @@ ' . implode("\n", $errors) . ''); +} + +$customConfig = []; +if (file_exists(__DIR__ . '/../config.ini.php')) { + $customConfig = parse_ini_file(__DIR__ . '/../config.ini.php', true, INI_SCANNER_TYPED); +} +Configuration::loadConfiguration($customConfig, getenv()); diff --git a/lib/logger.php b/lib/logger.php index 7a902b5b..e579915d 100644 --- a/lib/logger.php +++ b/lib/logger.php @@ -149,6 +149,7 @@ final class StreamHandler ); error_log($text); if ($record['level'] < Logger::ERROR && Debug::isEnabled()) { + // The record level is INFO or WARNING here // Not a good idea to print here because http headers might not have been sent print sprintf("
%s
\n", e($text)); } diff --git a/phpcs.xml b/phpcs.xml index 5e50470a..21e1f50a 100644 --- a/phpcs.xml +++ b/phpcs.xml @@ -1,6 +1,11 @@ - Created with the PHP Coding Standard Generator. http://edorian.github.com/php-coding-standard-generator/ + + Originally created with the PHP Coding Standard Generator. + But later manually tweaked. + http://edorian.github.com/php-coding-standard-generator/ + + ./static ./vendor ./templates @@ -11,6 +16,7 @@ + diff --git a/templates/exception.html.php b/templates/exception.html.php index e1dd97c1..62ac90b4 100644 --- a/templates/exception.html.php +++ b/templates/exception.html.php @@ -23,6 +23,14 @@

+ getCode() === 403): ?> +

403 Forbidden

+

+ The HTTP 403 Forbidden response status code indicates that the + server understands the request but refuses to authorize it. +

+ + getCode() === 404): ?>

404 Page Not Found

From 0c08f791efbfc6dd92f89d922984a6a41583de44 Mon Sep 17 00:00:00 2001 From: ORelio Date: Tue, 9 Jan 2024 20:34:56 +0100 Subject: [PATCH 252/716] CssSelectorComplexBridge: Use cookies everywhere (#3827) (#3870) --- bridges/CssSelectorComplexBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/CssSelectorComplexBridge.php b/bridges/CssSelectorComplexBridge.php index e661fe18..67ad4c92 100644 --- a/bridges/CssSelectorComplexBridge.php +++ b/bridges/CssSelectorComplexBridge.php @@ -245,7 +245,7 @@ class CssSelectorComplexBridge extends BridgeAbstract protected function getTitle($page, $title_cleanup) { if (is_string($page)) { - $page = getSimpleHTMLDOMCached($page); + $page = getSimpleHTMLDOMCached($page, $this->getHeaders()); } $title = html_entity_decode($page->find('title', 0)->plaintext); if (!empty($title)) { @@ -302,7 +302,7 @@ class CssSelectorComplexBridge extends BridgeAbstract protected function htmlFindEntryElements($page, $entry_selector, $url_selector, $url_pattern = '', $limit = 0) { if (is_string($page)) { - $page = getSimpleHTMLDOM($page); + $page = getSimpleHTMLDOM($page, $this->getHeaders()); } $entryElements = $page->find($entry_selector); @@ -355,7 +355,7 @@ class CssSelectorComplexBridge extends BridgeAbstract */ protected function fetchArticleElementFromPage($entry_url, $content_selector) { - $entry_html = getSimpleHTMLDOMCached($entry_url); + $entry_html = getSimpleHTMLDOMCached($entry_url, $this->getHeaders()); $article_content = $entry_html->find($content_selector, 0); if (is_null($article_content)) { From 1fecc4cfc13072856d68b7a33233a4e5e54a72db Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 9 Jan 2024 21:28:43 +0100 Subject: [PATCH 253/716] Revert "CssSelectorComplexBridge: Use cookies everywhere (#3827) (#3870)" (#3881) This reverts commit 0c08f791efbfc6dd92f89d922984a6a41583de44. --- bridges/CssSelectorComplexBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/CssSelectorComplexBridge.php b/bridges/CssSelectorComplexBridge.php index 67ad4c92..e661fe18 100644 --- a/bridges/CssSelectorComplexBridge.php +++ b/bridges/CssSelectorComplexBridge.php @@ -245,7 +245,7 @@ class CssSelectorComplexBridge extends BridgeAbstract protected function getTitle($page, $title_cleanup) { if (is_string($page)) { - $page = getSimpleHTMLDOMCached($page, $this->getHeaders()); + $page = getSimpleHTMLDOMCached($page); } $title = html_entity_decode($page->find('title', 0)->plaintext); if (!empty($title)) { @@ -302,7 +302,7 @@ class CssSelectorComplexBridge extends BridgeAbstract protected function htmlFindEntryElements($page, $entry_selector, $url_selector, $url_pattern = '', $limit = 0) { if (is_string($page)) { - $page = getSimpleHTMLDOM($page, $this->getHeaders()); + $page = getSimpleHTMLDOM($page); } $entryElements = $page->find($entry_selector); @@ -355,7 +355,7 @@ class CssSelectorComplexBridge extends BridgeAbstract */ protected function fetchArticleElementFromPage($entry_url, $content_selector) { - $entry_html = getSimpleHTMLDOMCached($entry_url, $this->getHeaders()); + $entry_html = getSimpleHTMLDOMCached($entry_url); $article_content = $entry_html->find($content_selector, 0); if (is_null($article_content)) { From 2e5d2a88f39afccefab58b4fb40d22da7794a4b8 Mon Sep 17 00:00:00 2001 From: Dag Date: Tue, 9 Jan 2024 21:36:42 +0100 Subject: [PATCH 254/716] fix: only escape iframe,script and link for html output (#3882) --- formats/AtomFormat.php | 2 +- formats/JsonFormat.php | 2 +- formats/MrssFormat.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php index 1fabef2e..5c9f2b6a 100644 --- a/formats/AtomFormat.php +++ b/formats/AtomFormat.php @@ -179,7 +179,7 @@ class AtomFormat extends FormatAbstract $content = $document->createElement('content'); $content->setAttribute('type', 'html'); - $content->appendChild($document->createTextNode(break_annoying_html_tags($entryContent))); + $content->appendChild($document->createTextNode($entryContent)); $entry->appendChild($content); foreach ($item->getEnclosures() as $enclosure) { diff --git a/formats/JsonFormat.php b/formats/JsonFormat.php index 016e75e1..586aae0a 100644 --- a/formats/JsonFormat.php +++ b/formats/JsonFormat.php @@ -47,7 +47,7 @@ class JsonFormat extends FormatAbstract $entryTitle = $item->getTitle(); $entryUri = $item->getURI(); $entryTimestamp = $item->getTimestamp(); - $entryContent = $item->getContent() ? break_annoying_html_tags($item->getContent()) : ''; + $entryContent = $item->getContent() ?? ''; $entryEnclosures = $item->getEnclosures(); $entryCategories = $item->getCategories(); diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php index e93a8289..aaa1d0cd 100644 --- a/formats/MrssFormat.php +++ b/formats/MrssFormat.php @@ -119,7 +119,7 @@ class MrssFormat extends FormatAbstract $itemTimestamp = $item->getTimestamp(); $itemTitle = $item->getTitle(); $itemUri = $item->getURI(); - $itemContent = $item->getContent() ? break_annoying_html_tags($item->getContent()) : ''; + $itemContent = $item->getContent() ?? ''; $itemUid = $item->getUid(); $isPermaLink = 'false'; From 491cb50219d8f799d85bfb4e6027adf501e9afa4 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 10 Jan 2024 00:25:36 +0100 Subject: [PATCH 255/716] docs: typo (#3883) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 46bb5a69..e027d912 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ As root: ### How to remove all expired cache items - bin/cache-clear + bin/cache-prune ### How to fix "PHP Fatal error: Uncaught Exception: The FileCache path is not writable" From 0eb4f6b2678ab17255ee87bde2f919a7e6883799 Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 10 Jan 2024 20:39:15 +0100 Subject: [PATCH 256/716] fix(tiktok): remove duplicate leading slash in url path, fix #3884 (#3885) --- bridges/TikTokBridge.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bridges/TikTokBridge.php b/bridges/TikTokBridge.php index 6590df66..22fdfcef 100644 --- a/bridges/TikTokBridge.php +++ b/bridges/TikTokBridge.php @@ -35,21 +35,23 @@ class TikTokBridge extends BridgeAbstract foreach ($videos as $video) { $item = []; - // Handle link "untracking" - $linkParts = parse_url($video->find('a', 0)->href); - $link = $linkParts['scheme'] . '://' . $linkParts['host'] . '/' . $linkParts['path']; + // Omit query string (remove tracking parameters) + $a = $video->find('a', 0); + $href = $a->href; + $parsedUrl = parse_url($href); + $url = $parsedUrl['scheme'] . '://' . $parsedUrl['host'] . '/' . ltrim($parsedUrl['path'], '/'); $image = $video->find('video', 0)->poster; $views = $video->find('div[data-e2e=common-Video-Count]', 0)->plaintext; $enclosures = [$image]; - $item['uri'] = $link; + $item['uri'] = $url; $item['title'] = 'Video'; $item['author'] = '@' . $author; $item['enclosures'] = $enclosures; $item['content'] = << +

{$views} views


EOD; From c7e8ddf4865516a4bddc884cf80c058cb5aad770 Mon Sep 17 00:00:00 2001 From: ORelio Date: Wed, 10 Jan 2024 21:47:34 +0100 Subject: [PATCH 257/716] CssSelectorComplexBridge: Use cookies everywhere (RSS-Bridge#3827) (#3886) v2 after feedback from #3870 --- bridges/CssSelectorComplexBridge.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bridges/CssSelectorComplexBridge.php b/bridges/CssSelectorComplexBridge.php index e661fe18..632e6b6a 100644 --- a/bridges/CssSelectorComplexBridge.php +++ b/bridges/CssSelectorComplexBridge.php @@ -245,7 +245,7 @@ class CssSelectorComplexBridge extends BridgeAbstract protected function getTitle($page, $title_cleanup) { if (is_string($page)) { - $page = getSimpleHTMLDOMCached($page); + $page = getSimpleHTMLDOMCached($page, 86400, $this->getHeaders()); } $title = html_entity_decode($page->find('title', 0)->plaintext); if (!empty($title)) { @@ -302,7 +302,7 @@ class CssSelectorComplexBridge extends BridgeAbstract protected function htmlFindEntryElements($page, $entry_selector, $url_selector, $url_pattern = '', $limit = 0) { if (is_string($page)) { - $page = getSimpleHTMLDOM($page); + $page = getSimpleHTMLDOM($page, $this->getHeaders()); } $entryElements = $page->find($entry_selector); @@ -355,7 +355,7 @@ class CssSelectorComplexBridge extends BridgeAbstract */ protected function fetchArticleElementFromPage($entry_url, $content_selector) { - $entry_html = getSimpleHTMLDOMCached($entry_url); + $entry_html = getSimpleHTMLDOMCached($entry_url, 86400, $this->getHeaders()); $article_content = $entry_html->find($content_selector, 0); if (is_null($article_content)) { From 080e29365a24c5ad0898f2f8bf99e7068c41856b Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 10 Jan 2024 21:48:12 +0100 Subject: [PATCH 258/716] feat(http-client): add http retry count to config (#3887) --- config.default.ini.php | 5 +++++ lib/contents.php | 3 ++- lib/http.php | 30 ++++++++++++++++-------------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/config.default.ini.php b/config.default.ini.php index 201b1414..21727c5e 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -49,6 +49,11 @@ enable_maintenance_mode = false [http] ; Operation timeout in seconds timeout = 30 + +; Operation retry count in case of curl error +retries = 2 + +; User agent useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0" ; Max http response size in MB diff --git a/lib/contents.php b/lib/contents.php index 8676a2a8..9998a3f1 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -38,6 +38,7 @@ function getContents( $config = [ 'useragent' => Configuration::getConfig('http', 'useragent'), 'timeout' => Configuration::getConfig('http', 'timeout'), + 'retries' => Configuration::getConfig('http', 'retries'), 'headers' => array_merge($defaultHttpHeaders, $httpHeadersNormalized), 'curl_options' => $curlOptions, ]; @@ -71,7 +72,7 @@ function getContents( // Ignore invalid 'Last-Modified' HTTP header value } } - // todo: to be nice nice citizen we should also check for Etag + // todo: We should also check for Etag } $response = $httpClient->request($url, $config); diff --git a/lib/http.php b/lib/http.php index bfa6b6bf..405b01c6 100644 --- a/lib/http.php +++ b/lib/http.php @@ -63,7 +63,7 @@ final class CurlHttpClient implements HttpClient 'proxy' => null, 'curl_options' => [], 'if_not_modified_since' => null, - 'retries' => 3, + 'retries' => 2, 'max_filesize' => null, 'max_redirections' => 5, ]; @@ -136,26 +136,28 @@ final class CurlHttpClient implements HttpClient return $len; }); - $attempts = 0; + // This retry logic is a bit hard to understand, but it works + $tries = 0; while (true) { - $attempts++; + $tries++; $body = curl_exec($ch); if ($body !== false) { // The network call was successful, so break out of the loop break; } - if ($attempts > $config['retries']) { - // Finally give up - $curl_error = curl_error($ch); - $curl_errno = curl_errno($ch); - throw new HttpException(sprintf( - 'cURL error %s: %s (%s) for %s', - $curl_error, - $curl_errno, - 'https://curl.haxx.se/libcurl/c/libcurl-errors.html', - $url - )); + if ($tries <= $config['retries']) { + continue; } + // Max retries reached, give up + $curl_error = curl_error($ch); + $curl_errno = curl_errno($ch); + throw new HttpException(sprintf( + 'cURL error %s: %s (%s) for %s', + $curl_error, + $curl_errno, + 'https://curl.haxx.se/libcurl/c/libcurl-errors.html', + $url + )); } $statusCode = curl_getinfo($ch, CURLINFO_RESPONSE_CODE); From d9ac0195506040e68cebfc81c5753e416ab7b22f Mon Sep 17 00:00:00 2001 From: July Date: Wed, 10 Jan 2024 18:42:57 -0500 Subject: [PATCH 259/716] [AnnasArchiveBridge] Add new bridge (#3888) * [AnnasArchiveBridge] Add new bridge * [AnnasArchiveBridge] Add missing exampleValue * [AnnasArchiveBridge] Remove vestigial debug print --- bridges/AnnasArchiveBridge.php | 175 +++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 bridges/AnnasArchiveBridge.php diff --git a/bridges/AnnasArchiveBridge.php b/bridges/AnnasArchiveBridge.php new file mode 100644 index 00000000..e8a1e8c4 --- /dev/null +++ b/bridges/AnnasArchiveBridge.php @@ -0,0 +1,175 @@ + [ + 'name' => 'Query', + 'exampleValue' => 'apothecary diaries', + 'required' => true, + ], + 'ext' => [ + 'name' => 'Extension', + 'type' => 'list', + 'values' => [ + 'Any' => null, + 'azw3' => 'azw3', + 'cbr' => 'cbr', + 'cbz' => 'cbz', + 'djvu' => 'djvu', + 'epub' => 'epub', + 'fb2' => 'fb2', + 'fb2.zip' => 'fb2.zip', + 'mobi' => 'mobi', + 'pdf' => 'pdf', + ] + ], + 'lang' => [ + 'name' => 'Language', + 'type' => 'list', + 'values' => [ + 'Any' => null, + 'Afrikaans [af]' => 'af', + 'Arabic [ar]' => 'ar', + 'Bangla [bn]' => 'bn', + 'Belarusian [be]' => 'be', + 'Bulgarian [bg]' => 'bg', + 'Catalan [ca]' => 'ca', + 'Chinese [zh]' => 'zh', + 'Church Slavic [cu]' => 'cu', + 'Croatian [hr]' => 'hr', + 'Czech [cs]' => 'cs', + 'Danish [da]' => 'da', + 'Dongxiang [sce]' => 'sce', + 'Dutch [nl]' => 'nl', + 'English [en]' => 'en', + 'French [fr]' => 'fr', + 'German [de]' => 'de', + 'Greek [el]' => 'el', + 'Hebrew [he]' => 'he', + 'Hindi [hi]' => 'hi', + 'Hungarian [hu]' => 'hu', + 'Indonesian [id]' => 'id', + 'Irish [ga]' => 'ga', + 'Italian [it]' => 'it', + 'Japanese [ja]' => 'ja', + 'Kazakh [kk]' => 'kk', + 'Korean [ko]' => 'ko', + 'Latin [la]' => 'la', + 'Latvian [lv]' => 'lv', + 'Lithuanian [lt]' => 'lt', + 'Luxembourgish [lb]' => 'lb', + 'Ndolo [ndl]' => 'ndl', + 'Norwegian [no]' => 'no', + 'Persian [fa]' => 'fa', + 'Polish [pl]' => 'pl', + 'Portuguese [pt]' => 'pt', + 'Romanian [ro]' => 'ro', + 'Russian [ru]' => 'ru', + 'Serbian [sr]' => 'sr', + 'Spanish [es]' => 'es', + 'Swedish [sv]' => 'sv', + 'Tamil [ta]' => 'ta', + 'Traditional Chinese [zh‑Hant]' => 'zh‑Hant', + 'Turkish [tr]' => 'tr', + 'Ukrainian [uk]' => 'uk', + 'Unknown language' => '_empty', + 'Unknown language [und]' => 'und', + 'Unknown language [urdu]' => 'urdu', + 'Urdu [ur]' => 'ur', + 'Vietnamese [vi]' => 'vi', + 'Welsh [cy]' => 'cy', + ] + ], + 'content' => [ + 'name' => 'Type', + 'type' => 'list', + 'values' => [ + 'Any' => null, + 'Book (fiction)' => 'book_fiction', + 'Book (non‑fiction)' => 'book_nonfiction', + 'Book (unknown)' => 'book_unknown', + 'Comic book' => 'book_comic', + 'Journal article' => 'journal_article', + 'Magazine' => 'magazine', + 'Standards document' => 'standards_document', + ] + ], + 'src' => [ + 'name' => 'Source', + 'type' => 'list', + 'values' => [ + 'Any' => null, + 'Internet Archive' => 'ia', + 'Libgen.li' => 'lgli', + 'Libgen.rs' => 'lgrs', + 'Sci‑Hub' => 'scihub', + 'Z‑Library' => 'zlib', + ] + ], + ] + ]; + + public function collectData() + { + $url = $this->getURI(); + $list = getSimpleHTMLDOMCached($url); + $list = defaultLinkTo($list, self::URI); + + // Don't attempt to do anything if not found message is given + if ($list->find('.js-not-found-additional')) { + return; + } + + foreach ($list->find('.w-full > .mb-4 > div > a') as $element) { + $item = []; + $item['title'] = $element->find('h3', 0)->plaintext; + $item['author'] = $element->find('div.italic', 0)->plaintext; + $item['uri'] = $element->href; + $item['content'] = $element->plaintext; + $item['uid'] = $item['uri']; + + if ($item_html = getSimpleHTMLDOMCached($item['uri'])) { + $item_html = defaultLinkTo($item_html, self::URI); + $item['content'] .= $item_html->find('main img', 0); + $item['content'] .= $item_html->find('main .mt-4', 0); // Summary + if ($links = $item_html->find('main ul.mb-4', -1)) { + foreach ($links->find('li > a.js-download-link') as $file) { + $item['enclosures'][] = $file->href; + } + // Remove bulk torrents from enclosures list + $item['enclosures'] = array_diff($item['enclosures'], [self::URI . 'datasets']); + } + } + + $this->items[] = $item; + } + } + + public function getName() + { + $name = parent::getName(); + if ($this->getInput('q') != null) { + $name .= ' - ' . $this->getInput('q'); + } + return $name; + } + + public function getURI() + { + $params = array_filter([ // Filter to remove non-provided parameters + 'q' => $this->getInput('q'), + 'ext' => $this->getInput('ext'), + 'lang' => $this->getInput('lang'), + 'src' => $this->getInput('src'), + 'content' => $this->getInput('content'), + ]); + $url = parent::getURI() . 'search?sort=newest&' . http_build_query($params); + return $url; + } +} From d5175aebcc6f74430189caab1525e6511722a6ed Mon Sep 17 00:00:00 2001 From: July Date: Thu, 11 Jan 2024 14:09:45 -0500 Subject: [PATCH 260/716] [ScribbleHubBridge] Get author feed title regardless of CloudFlare (#3892) --- bridges/ScribbleHubBridge.php | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/bridges/ScribbleHubBridge.php b/bridges/ScribbleHubBridge.php index e7cdf337..0f7c7a6c 100644 --- a/bridges/ScribbleHubBridge.php +++ b/bridges/ScribbleHubBridge.php @@ -12,16 +12,16 @@ class ScribbleHubBridge extends FeedExpander 'uid' => [ 'name' => 'uid', 'required' => true, - // Example: Alyson Greaves's stories - 'exampleValue' => '76208', + // Example: miriamrobern's stories + 'exampleValue' => '149271', ], ], 'Series' => [ 'sid' => [ 'name' => 'sid', 'required' => true, - // Example: latest chapters from The Sisters of Dorley by Alyson Greaves - 'exampleValue' => '421879', + // Example: latest chapters from Uskweirs + 'exampleValue' => '965299', ], ] ]; @@ -52,6 +52,10 @@ class ScribbleHubBridge extends FeedExpander return []; } + if ($this->queriedContext === 'Author') { + $this->author = $item['author']; + } + $item['comments'] = $item['uri'] . '#comments'; try { @@ -90,16 +94,7 @@ class ScribbleHubBridge extends FeedExpander $name = parent::getName() . " $this->queriedContext"; switch ($this->queriedContext) { case 'Author': - try { - $page = getSimpleHTMLDOMCached(self::URI . 'profile/' . $this->getInput('uid')); - } catch (HttpException $e) { - // 403 Forbidden, This means we got anti-bot response - if ($e->getCode() === 403) { - return $name; - } - throw $e; - } - $title = html_entity_decode($page->find('.p_m_username.fp_authorname', 0)->plaintext); + $title = $this->author; break; case 'Series': try { From 191e5b0493f3fc1bf2a3fc4169333c03480be23f Mon Sep 17 00:00:00 2001 From: Dag Date: Fri, 12 Jan 2024 01:31:01 +0100 Subject: [PATCH 261/716] feat: add etag support to getContents (#3893) --- README.md | 2 +- config.default.ini.php | 2 +- lib/BridgeCard.php | 5 ++--- lib/FeedExpander.php | 2 +- lib/FeedParser.php | 4 ++-- lib/XPathAbstract.php | 5 ++++- lib/contents.php | 49 +++++++++++++++++++++++------------------- lib/http.php | 4 ++++ 8 files changed, 42 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e027d912..d6d1046c 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ PHP ini config: ```ini ; /etc/php/8.2/fpm/conf.d/30-rss-bridge.ini -max_execution_time = 20 +max_execution_time = 15 memory_limit = 64M ``` diff --git a/config.default.ini.php b/config.default.ini.php index 21727c5e..ee1e54c9 100644 --- a/config.default.ini.php +++ b/config.default.ini.php @@ -48,7 +48,7 @@ enable_maintenance_mode = false [http] ; Operation timeout in seconds -timeout = 30 +timeout = 15 ; Operation retry count in case of curl error retries = 2 diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php index 4781ebc1..a82f8e5a 100644 --- a/lib/BridgeCard.php +++ b/lib/BridgeCard.php @@ -16,7 +16,7 @@ final class BridgeCard $bridge = $bridgeFactory->create($bridgeClassName); - $isHttps = strpos($bridge->getURI(), 'https') === 0; + $isHttps = str_starts_with($bridge->getURI(), 'https'); $uri = $bridge->getURI(); $name = $bridge->getName(); @@ -113,8 +113,7 @@ EOD; } if (!$isHttps) { - $form .= '

Warning : -This bridge is not fetching its content through a secure connection
'; + $form .= '
Warning: This bridge is not fetching its content through a secure connection
'; } return $form; diff --git a/lib/FeedExpander.php b/lib/FeedExpander.php index 056578e9..c0d7e878 100644 --- a/lib/FeedExpander.php +++ b/lib/FeedExpander.php @@ -41,7 +41,7 @@ abstract class FeedExpander extends BridgeAbstract } /** - * This method is overidden by bridges + * This method is overridden by bridges * * @return array */ diff --git a/lib/FeedParser.php b/lib/FeedParser.php index 2d982de1..510bcb32 100644 --- a/lib/FeedParser.php +++ b/lib/FeedParser.php @@ -7,9 +7,9 @@ declare(strict_types=1); * * Scrapes out rss 0.91, 1.0, 2.0 and atom 1.0. * - * Produce arrays meant to be used inside rss-bridge. + * Produces array meant to be used inside rss-bridge. * - * The item structure is tweaked so that works with FeedItem + * The item structure is tweaked so that it works with FeedItem */ final class FeedParser { diff --git a/lib/XPathAbstract.php b/lib/XPathAbstract.php index e30bb5eb..2206f79a 100644 --- a/lib/XPathAbstract.php +++ b/lib/XPathAbstract.php @@ -518,7 +518,10 @@ abstract class XPathAbstract extends BridgeAbstract if (strlen($value) === 0) { return ''; } - if (strpos($value, 'http://') === 0 || strpos($value, 'https://') === 0) { + if ( + strpos($value, 'http://') === 0 + || strpos($value, 'https://') === 0 + ) { return $value; } diff --git a/lib/contents.php b/lib/contents.php index 9998a3f1..43db8c03 100644 --- a/lib/contents.php +++ b/lib/contents.php @@ -24,6 +24,32 @@ function getContents( $headerValue = trim(implode(':', array_slice($parts, 1))); $httpHeadersNormalized[$headerName] = $headerValue; } + + $requestBodyHash = null; + if (isset($curlOptions[CURLOPT_POSTFIELDS])) { + $requestBodyHash = md5(Json::encode($curlOptions[CURLOPT_POSTFIELDS], false)); + } + $cacheKey = implode('_', ['server', $url, $requestBodyHash]); + + /** @var Response $cachedResponse */ + $cachedResponse = $cache->get($cacheKey); + if ($cachedResponse) { + $lastModified = $cachedResponse->getHeader('last-modified'); + if ($lastModified) { + try { + // Some servers send Unix timestamp instead of RFC7231 date. Prepend it with @ to allow parsing as DateTime + $lastModified = new \DateTimeImmutable((is_numeric($lastModified) ? '@' : '') . $lastModified); + $config['if_not_modified_since'] = $lastModified->getTimestamp(); + } catch (Exception $e) { + // Failed to parse last-modified + } + } + $etag = $cachedResponse->getHeader('etag'); + if ($etag) { + $httpHeadersNormalized['if-none-match'] = $etag; + } + } + // Snagged from https://github.com/lwthiker/curl-impersonate/blob/main/firefox/curl_ff102 $defaultHttpHeaders = [ 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', @@ -35,6 +61,7 @@ function getContents( 'Sec-Fetch-User' => '?1', 'TE' => 'trailers', ]; + $config = [ 'useragent' => Configuration::getConfig('http', 'useragent'), 'timeout' => Configuration::getConfig('http', 'timeout'), @@ -53,28 +80,6 @@ function getContents( $config['proxy'] = Configuration::getConfig('proxy', 'url'); } - $requestBodyHash = null; - if (isset($curlOptions[CURLOPT_POSTFIELDS])) { - $requestBodyHash = md5(Json::encode($curlOptions[CURLOPT_POSTFIELDS], false)); - } - $cacheKey = implode('_', ['server', $url, $requestBodyHash]); - - /** @var Response $cachedResponse */ - $cachedResponse = $cache->get($cacheKey); - if ($cachedResponse) { - $cachedLastModified = $cachedResponse->getHeader('last-modified'); - if ($cachedLastModified) { - try { - // Some servers send Unix timestamp instead of RFC7231 date. Prepend it with @ to allow parsing as DateTime - $cachedLastModified = new \DateTimeImmutable((is_numeric($cachedLastModified) ? '@' : '') . $cachedLastModified); - $config['if_not_modified_since'] = $cachedLastModified->getTimestamp(); - } catch (Exception $dateTimeParseFailue) { - // Ignore invalid 'Last-Modified' HTTP header value - } - } - // todo: We should also check for Etag - } - $response = $httpClient->request($url, $config); switch ($response->getCode()) { diff --git a/lib/http.php b/lib/http.php index 405b01c6..90b65a6e 100644 --- a/lib/http.php +++ b/lib/http.php @@ -258,6 +258,10 @@ final class Response } /** + * HTTP response may have multiple headers with the same name. + * + * This method by default, returns only the last header. + * * @return string[]|string|null */ public function getHeader(string $name, bool $all = false) From 6eaf0eaa565361d0a18f23cdcd8df894116ad73a Mon Sep 17 00:00:00 2001 From: Dag Date: Wed, 17 Jan 2024 20:10:32 +0100 Subject: [PATCH 262/716] fix: add cache clearing tools (#3896) Forgot to add these in #3867 --- .gitignore | 1 - bin/cache-clear | 14 ++++++++++++++ bin/cache-prune | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100755 bin/cache-clear create mode 100755 bin/cache-prune diff --git a/.gitignore b/.gitignore index 9725342d..6ed95489 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ data/ *.pydevproject .project .metadata -bin/ tmp/ *.tmp *.bak diff --git a/bin/cache-clear b/bin/cache-clear new file mode 100755 index 00000000..3563abad --- /dev/null +++ b/bin/cache-clear @@ -0,0 +1,14 @@ +#!/usr/bin/env php +clear(); diff --git a/bin/cache-prune b/bin/cache-prune new file mode 100755 index 00000000..7b7a6031 --- /dev/null +++ b/bin/cache-prune @@ -0,0 +1,14 @@ +#!/usr/bin/env php +prune(); From 6408123330a28041344cccf3133981196e62a9a6 Mon Sep 17 00:00:00 2001 From: SebLaus <97241865+SebLaus@users.noreply.github.com> Date: Fri, 19 Jan 2024 03:59:47 +0100 Subject: [PATCH 263/716] [IdealoBridge] added Header with user-agent and fixed typo (#3897) * Added header with useragent * copy paste error from local test environment * Fixed missing space in New before * fixed missing space after comma in argument list --- bridges/IdealoBridge.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bridges/IdealoBridge.php b/bridges/IdealoBridge.php index 89c5f87d..cef2b812 100644 --- a/bridges/IdealoBridge.php +++ b/bridges/IdealoBridge.php @@ -42,8 +42,13 @@ class IdealoBridge extends BridgeAbstract public function collectData() { + // Needs header with user-agent to function properly. + $header = [ + 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2.1 Safari/605.1.15' + ]; + $link = $this->getInput('Link'); - $html = getSimpleHTMLDOM($link); + $html = getSimpleHTMLDOM($link, $header); // Get Productname $titleobj = $html->find('.oopStage-title', 0); @@ -80,7 +85,7 @@ class IdealoBridge extends BridgeAbstract // Generate Content if ($PriceNew > 1) { $content = "

Price New:
$PriceNew

"; - $content .= "

Price Newbefore:
$OldPriceNew

"; + $content .= "

Price New before:
$OldPriceNew

"; } if ($this->getInput('MaxPriceNew') != '') { From 12a90e20749471c1f2c794792f6b1fabcb74d13e Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 19 Jan 2024 21:30:06 +0100 Subject: [PATCH 264/716] Utils: Add Webp MIME type (#3900) --- lib/utils.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/utils.php b/lib/utils.php index e8f00f54..07806e7c 100644 --- a/lib/utils.php +++ b/lib/utils.php @@ -171,6 +171,7 @@ function parse_mime_type($url) 'jpg' => 'image/jpeg', 'gif' => 'image/gif', 'png' => 'image/png', + 'webp' => 'image/webp', 'image' => 'image/*', 'mp3' => 'audio/mpeg', ]; From bb36eb9eb831eb6bce8641323b7e5ce90798575b Mon Sep 17 00:00:00 2001 From: ORelio Date: Fri, 19 Jan 2024 21:30:53 +0100 Subject: [PATCH 265/716] [CssSelectorBridge] Time/Thumbnail improvements (#3879) (#3901) * Implement