diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index faec9f09..15297cc3 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -1,8 +1,21 @@
FROM rssbridge/rss-bridge:latest
-RUN apt-get update && \
- apt-get install --yes --no-install-recommends \
- git && \
- pecl install xdebug && \
- pear install PHP_CodeSniffer && \
- docker-php-ext-enable xdebug
\ No newline at end of file
+COPY --chmod=755 post-create-command.sh /usr/local/bin/post-create-command
+
+ADD https://raw.githubusercontent.com/docker-library/php/master/docker-php-ext-enable /usr/local/bin/docker-php-ext-enable
+RUN chmod u+x /usr/local/bin/docker-php-ext-enable
+
+ADD https://getcomposer.org/installer /usr/local/bin/composer-installer.php
+RUN chmod u+x /usr/local/bin/composer-installer.php
+RUN php /usr/local/bin/composer-installer.php --check && \
+ php /usr/local/bin/composer-installer.php --filename=composer --install-dir=/usr/local/bin
+
+RUN apt-get update && \
+ apt-get install -y \
+ git \
+ php-dev \
+ make \
+ unzip
+
+RUN pecl install xdebug && \
+ PHP_INI_DIR=/etc/php/8.2/fpm docker-php-ext-enable xdebug
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 6e625b8a..564da38a 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -6,9 +6,9 @@
"vscode": {
// Set *default* container specific settings.json values on container create.
"settings": {
- "php.validate.executablePath": "/usr/local/bin/php",
- "phpSniffer.executablesFolder": "/usr/local/bin/",
- "phpcs.executablePath": "/usr/local/bin/phpcs",
+ "php.validate.executablePath": "/usr/bin/php",
+ "phpSniffer.executablesFolder": "/root/.config/composer/vendor/bin",
+ "phpcs.executablePath": "/root/.config/composer/vendor/bin/phpcs",
"phpcs.lintOnType": false
},
@@ -22,6 +22,9 @@
]
}
},
+ "remoteEnv": {
+ "PATH": "${containerEnv:PATH}:/root/.config/composer/vendor/bin",
+ },
"forwardPorts": [3100, 9000, 9003],
- "postCreateCommand": "cp .devcontainer/nginx.conf /etc/nginx/conf.d/default.conf && cp .devcontainer/xdebug.ini /usr/local/etc/php/conf.d/xdebug.ini && mkdir .vscode && cp .devcontainer/launch.json .vscode && echo '*' > whitelist.txt && chmod a+x \"$(pwd)\" && rm -rf /var/www/html && ln -s \"$(pwd)\" /var/www/html && nginx && php-fpm -D"
+ "postCreateCommand": "/usr/local/bin/post-create-command"
}
\ No newline at end of file
diff --git a/.devcontainer/launch.json b/.devcontainer/launch.json
index e1b473b8..7fe617e9 100644
--- a/.devcontainer/launch.json
+++ b/.devcontainer/launch.json
@@ -9,7 +9,8 @@
"type": "php",
"request": "launch",
"port": 9003,
- "auto": true
+ "auto": true,
+ "log": true
},
{
"name": "Launch currently open script",
diff --git a/.devcontainer/nginx.conf b/.devcontainer/nginx.conf
index 46502cb4..0e5db6dc 100644
--- a/.devcontainer/nginx.conf
+++ b/.devcontainer/nginx.conf
@@ -12,6 +12,6 @@ server {
location ~ \.php$ {
include snippets/fastcgi-php.conf;
- fastcgi_pass 127.0.0.1:9000;
+ fastcgi_pass unix:/var/run/php/php8.2-fpm.sock;
}
-}
\ No newline at end of file
+}
diff --git a/.devcontainer/post-create-command.sh b/.devcontainer/post-create-command.sh
new file mode 100755
index 00000000..aa3819eb
--- /dev/null
+++ b/.devcontainer/post-create-command.sh
@@ -0,0 +1,27 @@
+#/bin/sh
+
+cp .devcontainer/nginx.conf /etc/nginx/conf.d/default.conf
+cp .devcontainer/xdebug.ini /etc/php/8.2/fpm/conf.d/xdebug.ini
+
+# This should download some dev-dependencies, like phpunit and the PHP code sniffers
+composer global require "phpunit/phpunit:^9"
+composer global require "squizlabs/php_codesniffer:^3.6"
+composer global require "phpcompatibility/php-compatibility:^9.3"
+
+# We need to this manually for running the PHPCompatibility ruleset
+phpcs --config-set installed_paths /root/.config/composer/vendor/phpcompatibility/php-compatibility
+
+mkdir -p .vscode
+cp .devcontainer/launch.json .vscode
+
+echo '*' > whitelist.txt
+
+chmod a+x $(pwd)
+rm -rf /var/www/html
+ln -s $(pwd) /var/www/html
+
+# Solves possible issue of cache directory not being accessible
+chown www-data:www-data -R $(pwd)/cache
+
+nginx
+php-fpm8.2 -D
\ No newline at end of file
diff --git a/.gitattributes b/.gitattributes
index 5981fb34..28053256 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -47,8 +47,6 @@ phpcs.xml export-ignore
phpcompatibility.xml export-ignore
tests/ export-ignore
cache/.gitkeep export-ignore
-bridges/DemoBridge.php export-ignore
-bridges/FeedExpanderExampleBridge.php export-ignore
## Composer
#
diff --git a/.github/.gitignore b/.github/.gitignore
new file mode 100644
index 00000000..7ebb4030
--- /dev/null
+++ b/.github/.gitignore
@@ -0,0 +1,7 @@
+# Visual Studio Code
+.vscode/*
+
+# Generated files
+comment*.md
+comment*.txt
+*.html
diff --git a/.github/ISSUE_TEMPLATE/bridge-request.md b/.github/ISSUE_TEMPLATE/bridge-request.md
index 174dc095..088cc3d6 100644
--- a/.github/ISSUE_TEMPLATE/bridge-request.md
+++ b/.github/ISSUE_TEMPLATE/bridge-request.md
@@ -49,9 +49,9 @@ Please describe what you expect from the bridge. Whenever possible provide sampl
- _Default limit_: 5
- [ ] Load full articles
- _Cache articles_ (articles are stored in a local cache on first request): yes
- - _Cache timeout_ (max = 24 hours): 24 hours
+ - _Cache timeout_ : 24 hours
- [X] Balance requests (RSS-Bridge uses cached versions to reduce bandwith usage)
- - _Timeout_ (default = 5 minutes, max = 24 hours): 5 minutes
+ - _Timeout_ (default = 5 minutes): 5 minutes
diff --git a/.github/prtester.py b/.github/prtester.py
index df6cc1ff..c5c5be22 100644
--- a/.github/prtester.py
+++ b/.github/prtester.py
@@ -1,113 +1,208 @@
+import argparse
import requests
-import itertools
+import re
from bs4 import BeautifulSoup
from datetime import datetime
-import os.path
+from typing import Iterable
+import os
+import glob
+import urllib
# This script is specifically written to be used in automation for https://github.com/RSS-Bridge/rss-bridge
#
# This will scrape the whitelisted bridges in the current state (port 3000) and the PR state (port 3001) of
# RSS-Bridge, generate a feed for each of the bridges and save the output as html files.
-# It also replaces the default static CSS link with a hardcoded link to @em92's public instance, so viewing
+# It also add a tag with the url of em's public instance, so viewing
# the HTML file locally will actually work as designed.
-def testBridges(bridges,status):
- for bridge in bridges:
- if bridge.get('data-ref'): # Some div entries are empty, this ignores those
- bridgeid = bridge.get('id')
- bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
- print(bridgeid + "\n")
- bridgestring = '/?action=display&bridge=' + bridgeid + '&format=Html'
- forms = bridge.find_all("form")
- formid = 1
- for form in forms:
- # a bridge can have multiple contexts, named 'forms' in html
- # this code will produce a fully working formstring that should create a working feed when called
- # this will create an example feed for every single context, to test them all
- formstring = ''
- errormessages = []
- parameters = form.find_all("input")
- lists = form.find_all("select")
- # this for/if mess cycles through all available input parameters, checks if it required, then pulls
- # the default or examplevalue and then combines it all together into the formstring
- # if an example or default value is missing for a required attribute, it will throw an error
- # any non-required fields are not tested!!!
- for parameter in parameters:
- if parameter.get('type') == 'hidden' and parameter.get('name') == 'context':
- cleanvalue = parameter.get('value').replace(" ","+")
- formstring = formstring + '&' + parameter.get('name') + '=' + cleanvalue
- if parameter.get('type') == 'number' or parameter.get('type') == 'text':
- if parameter.has_attr('required'):
- if parameter.get('placeholder') == '':
- if parameter.get('value') == '':
- errormessages.append(parameter.get('name'))
- else:
- formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('value')
+ARTIFACT_FILE_EXTENSION = '.html'
+
+class Instance:
+ name = ''
+ url = ''
+
+def main(instances: Iterable[Instance], with_upload: bool, with_reduced_upload: bool, title: str, output_file: str):
+ start_date = datetime.now()
+
+ prid = os.getenv('PR')
+ artifact_base_url = f'https://rss-bridge.github.io/rss-bridge-tests/prs/{prid}'
+ artifact_directory = os.getcwd()
+ for file in glob.glob(f'*{ARTIFACT_FILE_EXTENSION}', root_dir=artifact_directory):
+ os.remove(file)
+
+ table_rows = []
+ for instance in instances:
+ page = requests.get(instance.url) # Use python requests to grab the rss-bridge main page
+ soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
+ bridge_cards = soup.select('.bridge-card') # get a soup-formatted list of all bridges on the rss-bridge page
+ table_rows += testBridges(
+ instance=instance,
+ bridge_cards=bridge_cards,
+ with_upload=with_upload,
+ with_reduced_upload=with_reduced_upload,
+ artifact_directory=artifact_directory,
+ artifact_base_url=artifact_base_url) # run the main scraping code with the list of bridges
+ with open(file=output_file, mode='w+', encoding='utf-8') as file:
+ table_rows_value = '\n'.join(sorted(table_rows))
+ file.write(f'''
+## {title}
+| Bridge | Context | Status |
+| - | - | - |
+{table_rows_value}
+
+*last change: {start_date.strftime("%A %Y-%m-%d %H:%M:%S")}*
+ '''.strip())
+
+def testBridges(instance: Instance, bridge_cards: Iterable, with_upload: bool, with_reduced_upload: bool, artifact_directory: str, artifact_base_url: str) -> Iterable:
+ instance_suffix = ''
+ if instance.name:
+ instance_suffix = f' ({instance.name})'
+ table_rows = []
+ for bridge_card in bridge_cards:
+ bridgeid = bridge_card.get('id')
+ bridgeid = bridgeid.split('-')[1] # this extracts a readable bridge name from the bridge metadata
+ print(f'{bridgeid}{instance_suffix}')
+ bridge_name = bridgeid.replace('Bridge', '')
+ context_forms = bridge_card.find_all("form")
+ form_number = 1
+ for context_form in context_forms:
+ # a bridge can have multiple contexts, named 'forms' in html
+ # this code will produce a fully working url that should create a working feed when called
+ # this will create an example feed for every single context, to test them all
+ context_parameters = {}
+ error_messages = []
+ context_name = '*untitled*'
+ context_name_element = context_form.find_previous_sibling('h5')
+ if context_name_element and context_name_element.text.strip() != '':
+ context_name = context_name_element.text
+ parameters = context_form.find_all("input")
+ lists = context_form.find_all("select")
+ # this for/if mess cycles through all available input parameters, checks if it required, then pulls
+ # the default or examplevalue and then combines it all together into the url parameters
+ # if an example or default value is missing for a required attribute, it will throw an error
+ # any non-required fields are not tested!!!
+ for parameter in parameters:
+ parameter_type = parameter.get('type')
+ parameter_name = parameter.get('name')
+ if parameter_type == 'hidden':
+ context_parameters[parameter_name] = parameter.get('value')
+ if parameter_type == 'number' or parameter_type == 'text':
+ if parameter.has_attr('required'):
+ if parameter.get('placeholder') == '':
+ if parameter.get('value') == '':
+ error_messages.append(f'Missing example or default value for parameter "{parameter_name}"')
else:
- formstring = formstring + '&' + parameter.get('name') + '=' + parameter.get('placeholder')
- # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the formstring
- if parameter.get('type') == 'checkbox':
- if parameter.has_attr('checked'):
- formstring = formstring + '&' + parameter.get('name') + '=on'
- for listing in lists:
- selectionvalue = ''
- listname = listing.get('name')
- cleanlist = []
- for option in listing.contents:
- if 'optgroup' in option.name:
- cleanlist.extend(option)
+ context_parameters[parameter_name] = parameter.get('value')
else:
- cleanlist.append(option)
- firstselectionentry = 1
- for selectionentry in cleanlist:
- if firstselectionentry:
+ context_parameters[parameter_name] = parameter.get('placeholder')
+ # same thing, just for checkboxes. If a checkbox is checked per default, it gets added to the url parameters
+ if parameter_type == 'checkbox':
+ if parameter.has_attr('checked'):
+ context_parameters[parameter_name] = 'on'
+ for listing in lists:
+ selectionvalue = ''
+ listname = listing.get('name')
+ cleanlist = []
+ options = listing.find_all('option')
+ for option in options:
+ if 'optgroup' in option.name:
+ cleanlist.extend(option)
+ else:
+ cleanlist.append(option)
+ firstselectionentry = 1
+ for selectionentry in cleanlist:
+ if firstselectionentry:
+ selectionvalue = selectionentry.get('value')
+ firstselectionentry = 0
+ else:
+ if 'selected' in selectionentry.attrs:
selectionvalue = selectionentry.get('value')
- firstselectionentry = 0
- else:
- if 'selected' in selectionentry.attrs:
- selectionvalue = selectionentry.get('value')
- break
- formstring = formstring + '&' + listname + '=' + selectionvalue
- if not errormessages:
- # if all example/default values are present, form the full request string, run the request, replace the static css
- # file with the url of em's public instance and then upload it to termpad.com, a pastebin-like-site.
- r = requests.get(URL + bridgestring + formstring)
- pagetext = r.text.replace('static/style.css','https://rss-bridge.org/bridge01/static/style.css')
- pagetext = pagetext.encode("utf_8")
- termpad = requests.post(url="https://termpad.com/", data=pagetext)
- termpadurl = termpad.text
- termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
- termpadurl = termpadurl.replace('\n','')
- with open(os.getcwd() + '/comment.txt', 'a+') as file:
- file.write("\n")
- file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
+ break
+ context_parameters[listname] = selectionvalue
+ artifact_url = 'about:blank'
+ if error_messages:
+ status = ' '.join(map(lambda m: f'❌ `{m}`', error_messages))
+ else:
+ # if all example/default values are present, form the full request url, run the request, add a tag with
+ # the url of em's public instance to the response text (so that relative paths work, e.g. to the static css file) and
+ # then save it to a html file.
+ context_parameters.update({
+ 'action': 'display',
+ 'bridge': bridgeid,
+ 'format': 'Html',
+ })
+ request_url = f'{instance.url}/?{urllib.parse.urlencode(context_parameters)}'
+ response = requests.get(request_url)
+ page_text = response.text.replace('','')
+ page_text = page_text.encode("utf_8")
+ soup = BeautifulSoup(page_text, "html.parser")
+ status_messages = []
+ if response.status_code != 200:
+ status_messages += [f'❌ `HTTP status {response.status_code} {response.reason}`']
else:
- # if there are errors (which means that a required value has no example or default value), log out which error appeared
- termpad = requests.post(url="https://termpad.com/", data=str(errormessages))
- termpadurl = termpad.text
- termpadurl = termpadurl.replace('termpad.com/','termpad.com/raw/')
- termpadurl = termpadurl.replace('\n','')
- with open(os.getcwd() + '/comment.txt', 'a+') as file:
- file.write("\n")
- file.write("| [`" + bridgeid + '-' + status + '-context' + str(formid) + "`](" + termpadurl + ") | " + date_time + " |")
- formid += 1
+ feed_items = soup.select('.feeditem')
+ feed_items_length = len(feed_items)
+ if feed_items_length <= 0:
+ status_messages += [f'⚠️ `The feed has no items`']
+ elif feed_items_length == 1 and len(soup.select('.error')) > 0:
+ status_messages += [f'❌ `{getFirstLine(feed_items[0].text)}`']
+ status_messages += map(lambda e: f'❌ `{getFirstLine(e.text)}`', soup.select('.error .error-type') + soup.select('.error .error-message'))
+ for item_element in soup.select('.feeditem'): # remove all feed items to not accidentally selected
tags from item content
+ item_element.decompose()
+ status_messages += map(lambda e: f'⚠️ `{getFirstLine(e.text)}`', soup.find_all('pre'))
+ status_messages = list(dict.fromkeys(status_messages)) # remove duplicates
+ status = ' '.join(status_messages)
+ status_is_ok = status == '';
+ if status_is_ok:
+ status = '✔️'
+ if with_upload and (not with_reduced_upload or not status_is_ok):
+ filename = f'{bridge_name} {form_number}{instance_suffix}{ARTIFACT_FILE_EXTENSION}'
+ filename = re.sub(r'[^a-z0-9 \_\-\.]', '', filename, flags=re.I).replace(' ', '_')
+ with open(file=f'{artifact_directory}/{filename}', mode='wb') as file:
+ file.write(page_text)
+ artifact_url = f'{artifact_base_url}/{filename}'
+ table_rows.append(f'| {bridge_name} | [{form_number} {context_name}{instance_suffix}]({artifact_url}) | {status} |')
+ form_number += 1
+ return table_rows
-gitstatus = ["current", "pr"]
-now = datetime.now()
-date_time = now.strftime("%Y-%m-%d, %H:%M:%S")
+def getFirstLine(value: str) -> str:
+ # trim whitespace and remove text that can break the table or is simply unnecessary
+ clean_value = re.sub(r'^\[[^\]]+\]\s*rssbridge\.|[\|`]', '', value.strip())
+ first_line = next(iter(clean_value.splitlines()), '')
+ max_length = 250
+ if (len(first_line) > max_length):
+ first_line = first_line[:max_length] + '...'
+ return first_line
-with open(os.getcwd() + '/comment.txt', 'w+') as file:
- file.write(''' ## Pull request artifacts
-| file | last change |
-| ---- | ------ |''')
-
-for status in gitstatus: # run this twice, once for the current version, once for the PR version
- if status == "current":
- port = "3000" # both ports are defined in the corresponding workflow .yml file
- elif status == "pr":
- port = "3001"
- URL = "http://localhost:" + port
- page = requests.get(URL) # Use python requests to grab the rss-bridge main page
- soup = BeautifulSoup(page.content, "html.parser") # use bs4 to turn the page into soup
- bridges = soup.find_all("section") # get a soup-formatted list of all bridges on the rss-bridge page
- testBridges(bridges,status) # run the main scraping code with the list of bridges and the info if this is for the current version or the pr version
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--instances', nargs='+')
+ parser.add_argument('--no-upload', action='store_true')
+ parser.add_argument('--reduced-upload', action='store_true')
+ parser.add_argument('--title', default='Pull request artifacts')
+ parser.add_argument('--output-file', default=os.getcwd() + '/comment.txt')
+ args = parser.parse_args()
+ instances = []
+ if args.instances:
+ for instance_arg in args.instances:
+ instance_arg_parts = instance_arg.split('::')
+ instance = Instance()
+ instance.name = instance_arg_parts[1].strip() if len(instance_arg_parts) >= 2 else ''
+ instance.url = instance_arg_parts[0].strip().rstrip("/")
+ instances.append(instance)
+ else:
+ instance = Instance()
+ instance.name = 'current'
+ instance.url = 'http://localhost:3000'
+ instances.append(instance)
+ instance = Instance()
+ instance.name = 'pr'
+ instance.url = 'http://localhost:3001'
+ instances.append(instance)
+ main(
+ instances=instances,
+ with_upload=not args.no_upload,
+ with_reduced_upload=args.reduced_upload and not args.no_upload,
+ title=args.title,
+ output_file=args.output_file
+ );
diff --git a/.github/workflows/dockerbuild.yml b/.github/workflows/dockerbuild.yml
index 82d8611a..39645558 100644
--- a/.github/workflows/dockerbuild.yml
+++ b/.github/workflows/dockerbuild.yml
@@ -21,7 +21,7 @@ jobs:
-
name: Docker meta
id: docker_meta
- uses: docker/metadata-action@v4
+ uses: docker/metadata-action@v5
with:
images: |
${{ env.DOCKERHUB_SLUG }}
@@ -33,26 +33,26 @@ jobs:
type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/20') }}
-
name: Set up QEMU
- uses: docker/setup-qemu-action@v2
+ uses: docker/setup-qemu-action@v3
-
name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v2
+ uses: docker/setup-buildx-action@v3
-
name: Login to DockerHub
- uses: docker/login-action@v2
+ uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Login to GitHub Container Registry
- uses: docker/login-action@v2
+ uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
-
name: Build and push
- uses: docker/bake-action@v2
+ uses: docker/bake-action@v5
with:
files: |
./docker-bake.hcl
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
index b00c898a..e0201022 100644
--- a/.github/workflows/documentation.yml
+++ b/.github/workflows/documentation.yml
@@ -9,7 +9,7 @@ jobs:
documentation:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
with:
persist-credentials: false
- name: Setup PHP
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 83911ab6..79201edd 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -8,12 +8,12 @@ on:
jobs:
phpcs:
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
strategy:
matrix:
php-versions: ['7.4']
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
@@ -21,12 +21,12 @@ jobs:
- run: phpcs . --standard=phpcs.xml --warning-severity=0 --extensions=php -p
phpcompatibility:
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
strategy:
matrix:
php-versions: ['7.4']
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
@@ -36,9 +36,9 @@ jobs:
- run: ~/.composer/vendor/bin/phpcs . --standard=phpcompatibility.xml --warning-severity=0 --extensions=php -p
executable_php_files_check:
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- run: |
if find -name "*.php" -executable -type f -print -exec false {} +
then
diff --git a/.github/workflows/prhtmlgenerator.yml b/.github/workflows/prhtmlgenerator.yml
index cacb6642..bfc89e7a 100644
--- a/.github/workflows/prhtmlgenerator.yml
+++ b/.github/workflows/prhtmlgenerator.yml
@@ -5,24 +5,41 @@ on:
branches: [ master ]
jobs:
+ check-bridges:
+ name: Check if bridges were changed
+ runs-on: ubuntu-latest
+ outputs:
+ BRIDGES: ${{ steps.check1.outputs.BRIDGES }}
+ steps:
+ - name: Check number of bridges
+ id: check1
+ run: |
+ PR=${{github.event.number}};
+ wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
+ bridgeamount=$(cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq | wc -l);
+ echo "BRIDGES=$bridgeamount" >> "$GITHUB_OUTPUT"
test-pr:
name: Generate HTML
runs-on: ubuntu-latest
+ needs: check-bridges
+ if: needs.check-bridges.outputs.BRIDGES > 0
+ env:
+ PYTHONUNBUFFERED: 1
# Needs additional permissions https://github.com/actions/first-interaction/issues/10#issuecomment-1041402989
steps:
- name: Check out self
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
with:
ref: ${{github.event.pull_request.head.ref}}
repository: ${{github.event.pull_request.head.repo.full_name}}
- name: Check out rss-bridge
run: |
PR=${{github.event.number}};
- wget -O requirements.txt https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester-requirements.txt;
- wget https://raw.githubusercontent.com/RSS-Bridge/rss-bridge/master/.github/prtester.py;
+ wget -O requirements.txt https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester-requirements.txt;
+ wget https://raw.githubusercontent.com/$GITHUB_REPOSITORY/${{ github.event.pull_request.base.ref }}/.github/prtester.py;
wget https://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$PR.patch;
touch DEBUG;
- cat $PR.patch | grep " bridges/.*\.php" | sed "s= bridges/\(.*\)Bridge.php.*=\1=g" | sort | uniq > whitelist.txt
+ cat $PR.patch | grep "\bbridges/[A-Za-z0-9]*Bridge\.php\b" | sed "s=.*\bbridges/\([A-Za-z0-9]*\)Bridge\.php\b.*=\1=g" | sort | uniq > whitelist.txt
- name: Start Docker - Current
run: |
docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3000:80 ghcr.io/rss-bridge/rss-bridge:latest
@@ -31,9 +48,9 @@ jobs:
docker build -t prbuild .;
docker run -d -v $GITHUB_WORKSPACE/whitelist.txt:/app/whitelist.txt -v $GITHUB_WORKSPACE/DEBUG:/app/DEBUG -p 3001:80 prbuild
- name: Setup python
- uses: actions/setup-python@v4
+ uses: actions/setup-python@v5
with:
- python-version: '3.7'
+ python-version: '3.13'
cache: 'pip'
- name: Install requirements
run: |
@@ -49,9 +66,17 @@ jobs:
body="${body//$'\n'/'%0A'}";
body="${body//$'\r'/'%0D'}";
echo "bodylength=${#body}" >> $GITHUB_OUTPUT
+ env:
+ PR: ${{ github.event.number }}
+ - name: Upload generated tests
+ uses: actions/upload-artifact@v4
+ id: upload-generated-tests
+ with:
+ name: tests
+ path: '*.html'
- name: Find Comment
if: ${{ steps.testrun.outputs.bodylength > 130 }}
- uses: peter-evans/find-comment@v2
+ uses: peter-evans/find-comment@v3
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
@@ -59,9 +84,43 @@ jobs:
body-includes: Pull request artifacts
- name: Create or update comment
if: ${{ steps.testrun.outputs.bodylength > 130 }}
- uses: peter-evans/create-or-update-comment@v2
+ uses: peter-evans/create-or-update-comment@v4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body-file: comment.txt
edit-mode: replace
+ upload_tests:
+ name: Upload tests
+ runs-on: ubuntu-latest
+ needs: test-pr
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ repository: 'RSS-Bridge/rss-bridge-tests'
+ ref: 'main'
+ token: ${{ secrets.RSSTESTER_ACTION }}
+
+ - name: Setup git config
+ run: |
+ git config --global user.name "GitHub Actions"
+ git config --global user.email "<>"
+
+ - name: Download tests
+ uses: actions/download-artifact@v4
+ with:
+ name: tests
+
+ - name: Move tests
+ run: |
+ cd prs
+ mkdir -p ${{github.event.number}}
+ cd ${{github.event.number}}
+ mv -f $GITHUB_WORKSPACE/*.html .
+
+ - name: Commit and push generated tests
+ run: |
+ export COMMIT_MESSAGE="Added tests for PR ${{github.event.number}}"
+ git add .
+ git commit -m "$COMMIT_MESSAGE" || exit 0
+ git push
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index e7684e6b..435369fa 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,14 +8,16 @@ on:
jobs:
phpunit8:
- runs-on: ubuntu-20.04
+ runs-on: ubuntu-22.04
strategy:
matrix:
- php-versions: ['7.4', '8.0', '8.1']
+ php-versions: ['7.4', '8.0', '8.1', '8.2', '8.3', '8.4']
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
- uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php-versions }}
+ env:
+ update: true
- run: composer install
- run: composer test
diff --git a/.gitignore b/.gitignore
index f574992d..6ed95489 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,6 @@ data/
*.pydevproject
.project
.metadata
-bin/
tmp/
*.tmp
*.bak
@@ -230,6 +229,9 @@ pip-log.txt
DEBUG
config.ini.php
config/*
+!config/nginx.conf
+!config/php-fpm.conf
+!config/php.ini
######################
## VisualStudioCode ##
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index f1080743..d27421aa 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -15,7 +15,7 @@
* [Astalaseven](https://github.com/Astalaseven)
* [Astyan-42](https://github.com/Astyan-42)
* [austinhuang0131](https://github.com/austinhuang0131)
-* [AxorPL](https://github.com/AxorPL)
+* [axor-mst](https://github.com/axor-mst)
* [ayacoo](https://github.com/ayacoo)
* [az5he6ch](https://github.com/az5he6ch)
* [b1nj](https://github.com/b1nj)
@@ -23,6 +23,7 @@
* [Binnette](https://github.com/Binnette)
* [BoboTiG](https://github.com/BoboTiG)
* [Bockiii](https://github.com/Bockiii)
+* [brtsos](https://github.com/brtsos)
* [captn3m0](https://github.com/captn3m0)
* [chemel](https://github.com/chemel)
* [Chouchen](https://github.com/Chouchen)
@@ -144,6 +145,7 @@
* [Niehztog](https://github.com/Niehztog)
* [NikNikYkt](https://github.com/NikNikYkt)
* [Nono-m0le](https://github.com/Nono-m0le)
+* [NotsoanoNimus](https://github.com/NotsoanoNimus)
* [obsiwitch](https://github.com/obsiwitch)
* [Ololbu](https://github.com/Ololbu)
* [ORelio](https://github.com/ORelio)
diff --git a/Dockerfile b/Dockerfile
index 8157dc12..fb783344 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,36 +1,75 @@
-FROM lwthiker/curl-impersonate:0.5-ff-slim-buster AS curlimpersonate
-
-FROM php:8.0.27-fpm-buster AS rssbridge
+FROM debian:12-slim AS rssbridge
LABEL description="RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one."
LABEL repository="https://github.com/RSS-Bridge/rss-bridge"
LABEL website="https://github.com/RSS-Bridge/rss-bridge"
-RUN apt-get update && \
+ARG DEBIAN_FRONTEND=noninteractive
+RUN set -xe && \
+ apt-get update && \
apt-get install --yes --no-install-recommends \
+ ca-certificates \
nginx \
- zlib1g-dev \
- libzip-dev \
- libmemcached-dev \
nss-plugin-pem \
- libicu-dev && \
- docker-php-ext-install zip && \
- docker-php-ext-install intl && \
- pecl install memcached && \
- docker-php-ext-enable memcached && \
- docker-php-ext-enable opcache && \
- mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini"
+ php-curl \
+ php-fpm \
+ php-intl \
+ # php-json is enabled by default with PHP 8.2 in Debian 12
+ php-mbstring \
+ php-memcached \
+ # php-opcache is enabled by default with PHP 8.2 in Debian 12
+ # php-openssl is enabled by default with PHP 8.2 in Debian 12
+ php-sqlite3 \
+ php-xml \
+ php-zip \
+ # php-zlib is enabled by default with PHP 8.2 in Debian 12
+ # for downloading libcurl-impersonate
+ curl \
+ # for patching libcurl-impersonate
+ patchelf \
+ && \
+ # install curl-impersonate library
+ curlimpersonate_version=1.0.0rc2 && \
+ { \
+ { \
+ [ $(arch) = 'aarch64' ] && \
+ archive="libcurl-impersonate-v${curlimpersonate_version}.aarch64-linux-gnu.tar.gz" && \
+ sha512sum="c8add80e7a0430a074edea1a11f73d03044c48e848e164af2d6f362866623e29bede207a50f18f95b1bc5ab3d33f5c31408be60a6da66b74a0d176eebe299116" \
+ ; } \
+ || { \
+ [ $(arch) = 'armv7l' ] && \
+ archive="libcurl-impersonate-v${curlimpersonate_version}.arm-linux-gnueabihf.tar.gz" && \
+ sha512sum="d0403ca4ad55a8d499b120e5675c7b5a0dc4946af49c933e91fc24455ffe5e122aa21ee95554612ff5d1bd6faea1556e1e1b9c821918e2644cc9bcbddc05747a" \
+ ; } \
+ || { \
+ [ $(arch) = 'x86_64' ] && \
+ archive="libcurl-impersonate-v${curlimpersonate_version}.x86_64-linux-gnu.tar.gz" && \
+ sha512sum="35cafda2b96df3218a6d8545e0947a899837ede51c90f7ef2980bd2d99dbd67199bc620000df28b186727300b8c7046d506807fb48ee0fbc068dc8ae01986339" \
+ ; } \
+ } && \
+ curl -LO "https://github.com/lexiforest/curl-impersonate/releases/download/v${curlimpersonate_version}/${archive}" && \
+ echo "$sha512sum $archive" | sha512sum -c - && \
+ mkdir -p /usr/local/lib/curl-impersonate && \
+ tar xaf "$archive" -C /usr/local/lib/curl-impersonate && \
+ patchelf --set-soname libcurl.so.4 /usr/local/lib/curl-impersonate/libcurl-impersonate.so && \
+ rm "$archive" && \
+ apt-get purge --assume-yes curl patchelf && \
+ rm -rf /var/lib/apt/lists/*
-COPY ./config/nginx.conf /etc/nginx/sites-enabled/default
+ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate.so
+ENV CURL_IMPERSONATE chrome131
+
+# logs should go to stdout / stderr
+RUN ln -sfT /dev/stderr /var/log/nginx/error.log; \
+ ln -sfT /dev/stdout /var/log/nginx/access.log; \
+ chown -R --no-dereference www-data:adm /var/log/nginx/
+
+COPY ./config/nginx.conf /etc/nginx/sites-available/default
+COPY ./config/php-fpm.conf /etc/php/8.2/fpm/pool.d/rss-bridge.conf
+COPY ./config/php.ini /etc/php/8.2/fpm/conf.d/90-rss-bridge.ini
COPY --chown=www-data:www-data ./ /app/
-COPY --from=curlimpersonate /usr/local/lib/libcurl-impersonate-ff.so /usr/local/lib/curl-impersonate/
-
-ENV LD_PRELOAD /usr/local/lib/curl-impersonate/libcurl-impersonate-ff.so
-
-ENV CURL_IMPERSONATE ff91esr
-
EXPOSE 80
ENTRYPOINT ["/app/docker-entrypoint.sh"]
diff --git a/README.md b/README.md
index e9f7360d..05042630 100644
--- a/README.md
+++ b/README.md
@@ -2,12 +2,25 @@
-RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for websites that don't have one.
+RSS-Bridge is a PHP web application.
+
+It generates web feeds for websites that don't have one.
+
+Officially hosted instance: https://rss-bridge.org/bridge01/
+
+IRC channel #rssbridge at https://libera.chat/
+
+[Full documentation](https://rss-bridge.github.io/rss-bridge/index.html)
+
+Alternatively find another
+[public instance](https://rss-bridge.github.io/rss-bridge/General/Public_Hosts.html).
+
+Requires minimum PHP 7.4.
+
[](UNLICENSE)
[](https://github.com/rss-bridge/rss-bridge/releases/latest)
[](https://web.libera.chat/#rssbridge)
-[](https://matrix.to/#/#rssbridge:libera.chat)
[](https://github.com/RSS-Bridge/rss-bridge/actions)
|||
@@ -15,116 +28,219 @@ RSS-Bridge is a PHP project capable of generating RSS and Atom feeds for website
|||
|||
|||
-|||
-## A subset of bridges
+## A subset of bridges (15/447)
-* `YouTube` : YouTube user channel, playlist or search
-* `Twitter` : Return keyword/hashtag search or user timeline
-* `Telegram` : Return the latest posts from a public group
-* `Reddit` : Return the latest posts from a subreddit or user
-* `Filter` : Filter an existing feed url
-* `Vk` : Latest posts from a user or group
-* `FeedMerge` : Merge two or more existing feeds into one
-* `Twitch` : Fetch the latest videos from a channel
-* `ThePirateBay` : Returns the newest indexed torrents from [The Pirate Bay](https://thepiratebay.se/) with keywords
-
-And [many more](bridges/), thanks to the community!
-
-[Full documentation](https://rss-bridge.github.io/rss-bridge/index.html)
-
-Check out RSS-Bridge right now on https://rss-bridge.org/bridge01 or find another
-[public instance](https://rss-bridge.github.io/rss-bridge/General/Public_Hosts.html).
+* `CssSelectorBridge`: [Scrape out a feed using CSS selectors](https://rss-bridge.org/bridge01/#bridge-CssSelectorBridge)
+* `FeedMergeBridge`: [Combine multiple feeds into one](https://rss-bridge.org/bridge01/#bridge-FeedMergeBridge)
+* `FeedReducerBridge`: [Reduce a noisy feed by some percentage](https://rss-bridge.org/bridge01/#bridge-FeedReducerBridge)
+* `FilterBridge`: [Filter a feed by excluding/including items by keyword](https://rss-bridge.org/bridge01/#bridge-FilterBridge)
+* `GettrBridge`: [Fetches the latest posts from a GETTR user](https://rss-bridge.org/bridge01/#bridge-GettrBridge)
+* `MastodonBridge`: [Fetches statuses from a Mastodon (ActivityPub) instance](https://rss-bridge.org/bridge01/#bridge-MastodonBridge)
+* `RedditBridge`: [Fetches posts from a user/subredit (with filtering options)](https://rss-bridge.org/bridge01/#bridge-RedditBridge)
+* `RumbleBridge`: [Fetches channel/user videos](https://rss-bridge.org/bridge01/#bridge-RumbleBridge)
+* `SoundcloudBridge`: [Fetches music by username](https://rss-bridge.org/bridge01/#bridge-SoundcloudBridge)
+* `TelegramBridge`: [Fetches posts from a public channel](https://rss-bridge.org/bridge01/#bridge-TelegramBridge)
+* `ThePirateBayBridge:` [Fetches torrents by search/user/category](https://rss-bridge.org/bridge01/#bridge-ThePirateBayBridge)
+* `TikTokBridge`: [Fetches posts by username](https://rss-bridge.org/bridge01/#bridge-TikTokBridge)
+* `TwitchBridge`: [Fetches videos from channel](https://rss-bridge.org/bridge01/#bridge-TwitchBridge)
+* `XPathBridge`: [Scrape out a feed using XPath expressions](https://rss-bridge.org/bridge01/#bridge-XPathBridge)
+* `YoutubeBridge`: [Fetches videos by username/channel/playlist/search](https://rss-bridge.org/bridge01/#bridge-YoutubeBridge)
+* `YouTubeCommunityTabBridge`: [Fetches posts from a channel's Posts tab](https://rss-bridge.org/bridge01/#bridge-YouTubeCommunityTabBridge)
## Tutorial
-RSS-Bridge requires php 7.4 (or higher).
+### How to install on traditional shared web hosting
-### Install with composer or git
+RSS-Bridge can basically be unzipped into a web folder. Should be working instantly.
+
+Latest zip:
+https://github.com/RSS-Bridge/rss-bridge/archive/refs/heads/master.zip (2MB)
+
+### How to install on Debian 12 (nginx + php-fpm)
+
+These instructions have been tested on a fresh Debian 12 VM from Digital Ocean (1vcpu-512mb-10gb, 5 USD/month).
```shell
+timedatectl set-timezone Europe/Oslo
+
+apt install git nginx php8.2-fpm php-mbstring php-simplexml php-curl php-intl
+
+# Create a user account
+useradd --shell /bin/bash --create-home rss-bridge
+
cd /var/www
-composer create-project --no-dev rss-bridge/rss-bridge
+
+# Create folder and change its ownership to rss-bridge
+mkdir rss-bridge && chown rss-bridge:rss-bridge rss-bridge/
+
+# Become rss-bridge
+su rss-bridge
+
+# Clone master branch into existing folder
+git clone https://github.com/RSS-Bridge/rss-bridge.git rss-bridge/
+cd rss-bridge
+
+# Copy over the default config (OPTIONAL)
+cp -v config.default.ini.php config.ini.php
+
+# Recursively give full permissions to user/owner
+chmod 700 --recursive ./
+
+# Give read and execute to others on folder ./static
+chmod o+rx ./ ./static
+
+# Recursively give give read to others on folder ./static
+chmod o+r --recursive ./static
```
-```shell
-cd /var/www
-git clone https://github.com/RSS-Bridge/rss-bridge.git
-```
-
-Config:
-
-```shell
-# Give the http user write permission to the cache folder
-chown www-data:www-data /var/www/rss-bridge/cache
-
-# Optionally copy over the default config file
-cp config.default.ini.php config.ini.php
-```
-
-Example config for nginx:
+Nginx config:
```nginx
-# /etc/nginx/sites-enabled/rssbridge
+# /etc/nginx/sites-enabled/rss-bridge.conf
+
server {
listen 80;
- server_name example.com;
- root /var/www/rss-bridge;
- index index.php;
- location ~ \.php$ {
+ # TODO: change to your own server name
+ server_name example.com;
+
+ access_log /var/log/nginx/rss-bridge.access.log;
+ error_log /var/log/nginx/rss-bridge.error.log;
+ log_not_found off;
+
+ # Intentionally not setting a root folder
+
+ # Static content only served here
+ location /static/ {
+ alias /var/www/rss-bridge/static/;
+ }
+
+ # Pass off to php-fpm only when location is EXACTLY == /
+ location = / {
+ root /var/www/rss-bridge/;
include snippets/fastcgi-php.conf;
- fastcgi_read_timeout 60s;
- fastcgi_pass unix:/run/php/php-fpm.sock;
+ fastcgi_read_timeout 45s;
+ fastcgi_pass unix:/run/php/rss-bridge.sock;
+ }
+
+ # Reduce log noise
+ location = /favicon.ico {
+ access_log off;
+ }
+
+ # Reduce log noise
+ location = /robots.txt {
+ access_log off;
}
}
```
-### Install with Docker:
+PHP FPM pool config:
+```ini
+; /etc/php/8.2/fpm/pool.d/rss-bridge.conf
-Install by using docker image from Docker Hub:
+[rss-bridge]
+
+user = rss-bridge
+group = rss-bridge
+
+listen = /run/php/rss-bridge.sock
+
+listen.owner = www-data
+listen.group = www-data
+
+; Create 10 workers standing by to serve requests
+pm = static
+pm.max_children = 10
+
+; Respawn worker after 500 requests (workaround for memory leaks etc.)
+pm.max_requests = 500
+```
+
+PHP ini config:
+```ini
+; /etc/php/8.2/fpm/conf.d/30-rss-bridge.ini
+
+max_execution_time = 15
+memory_limit = 64M
+```
+
+Restart fpm and nginx:
+
+```shell
+# Lint and restart php-fpm
+php-fpm8.2 -t && systemctl restart php8.2-fpm
+
+# Lint and restart nginx
+nginx -t && systemctl restart nginx
+```
+
+### How to install from Composer
+
+Install the latest release.
+
+```shell
+cd /var/www
+composer create-project -v --no-dev --no-scripts rss-bridge/rss-bridge
+```
+
+### How to install with Caddy
+
+TODO. See https://github.com/RSS-Bridge/rss-bridge/issues/3785
+
+### Install from Docker Hub:
+
+Install by downloading the docker image from Docker Hub:
```bash
# Create container
-docker create --name=rss-bridge --publish 3000:80 rssbridge/rss-bridge
+docker create --name=rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rssbridge/rss-bridge
+```
+You can put custom `config.ini.php` and bridges into `./config`.
+
+**You must restart container for custom changes to take effect.**
+
+See `docker-entrypoint.sh` for details.
+
+```bash
# Start container
docker start rss-bridge
```
Browse http://localhost:3000/
-Install by locally building the image:
+### Install by locally building from Dockerfile
```bash
# Build image from Dockerfile
docker build -t rss-bridge .
# Create container
-docker create --name rss-bridge --publish 3000:80 rss-bridge
+docker create --name rss-bridge --publish 3000:80 --volume $(pwd)/config:/config rss-bridge
+```
-# Start the container
+You can put custom `config.ini.php` and bridges into `./config`.
+
+**You must restart container for custom changes to take effect.**
+
+See `docker-entrypoint.sh` for details.
+
+```bash
+# Start container
docker start rss-bridge
```
Browse http://localhost:3000/
-#### Install with docker-compose
+### Install with docker-compose (using Docker Hub)
-Create a `docker-compose.yml` file locally with with the following content:
-```yml
-version: '2'
-services:
- rss-bridge:
- image: rssbridge/rss-bridge:latest
- volumes:
- - :/config
- ports:
- - 3000:80
- restart: unless-stopped
-```
+You can put custom `config.ini.php` and bridges into `./config`.
-Then launch with `docker-compose`:
+**You must restart container for custom changes to take effect.**
+
+See `docker-entrypoint.sh` for details.
```bash
docker-compose up
@@ -132,13 +248,14 @@ docker-compose up
Browse http://localhost:3000/
-### Alternative installation methods
+### Other installation methods
[](https://my.scalingo.com/deploy?source=https://github.com/sebsauvage/rss-bridge)
[](https://heroku.com/deploy)
[](https://www.cloudron.io/store/com.rssbridgeapp.cloudronapp.html)
+[](https://www.pikapods.com/pods?run=rssbridge)
-The Heroku quick deploy currently does not work. It might possibly work if you fork this repo and
+The Heroku quick deploy currently does not work. It might work if you fork this repo and
modify the `repository` in `scalingo.json`. See https://github.com/RSS-Bridge/rss-bridge/issues/2688
Learn more in
@@ -146,13 +263,81 @@ Learn more in
## How-to
-### How to create a new bridge from scratch
+### How to fix "Access denied."
+
+Output is from php-fpm. It is unable to read index.php.
+
+ chown rss-bridge:rss-bridge /var/www/rss-bridge/index.php
+
+### How to password-protect the instance (token)
+
+Modify `config.ini.php`:
+
+ [authentication]
+
+ token = "hunter2"
+
+### How to remove all cache items
+
+As current user:
+
+ bin/cache-clear
+
+As user rss-bridge:
+
+ sudo -u rss-bridge bin/cache-clear
+
+As root:
+
+ sudo bin/cache-clear
+
+### How to remove all expired cache items
+
+ bin/cache-prune
+
+### How to fix "PHP Fatal error: Uncaught Exception: The FileCache path is not writable"
+
+```shell
+# Give rss-bridge ownership
+chown rss-bridge:rss-bridge -R /var/www/rss-bridge/cache
+
+# Or, give www-data ownership
+chown www-data:www-data -R /var/www/rss-bridge/cache
+
+# Or, give everyone write permission
+chmod 777 -R /var/www/rss-bridge/cache
+
+# Or last ditch effort (CAREFUL)
+rm -rf /var/www/rss-bridge/cache/ && mkdir /var/www/rss-bridge/cache/
+```
+
+### How to fix "attempt to write a readonly database"
+
+The sqlite files (db, wal and shm) are not writeable.
+
+ chown -v rss-bridge:rss-bridge cache/*
+
+### How to fix "Unable to prepare statement: 1, no such table: storage"
+
+ rm cache/*
+
+### How to create a completely new bridge
+
+New code files MUST have `declare(strict_types=1);` at the top of file:
+
+```php
+bridgeFactory = new BridgeFactory();
+ public function __construct(
+ BridgeFactory $bridgeFactory
+ ) {
+ $this->bridgeFactory = $bridgeFactory;
}
- public function execute(array $request)
+ public function __invoke(Request $request): Response
{
- if (!Debug::isEnabled()) {
- throw new \Exception('This action is only available in debug mode!');
+ if (Configuration::getConfig('system', 'env') !== 'dev') {
+ return new Response('This action is only available in dev environment!', 403);
}
- if (!isset($request['bridge'])) {
- return render_template('connectivity.html.php');
+ $bridgeName = $request->get('bridge');
+ if (!$bridgeName) {
+ return new Response(render_template('connectivity.html.php'));
+ }
+ $bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
+ if (!$bridgeClassName) {
+ return new Response('Bridge not found', 404);
}
-
- $bridgeClassName = $this->bridgeFactory->createBridgeClassName($request['bridge']);
-
return $this->reportBridgeConnectivity($bridgeClassName);
}
@@ -52,29 +43,25 @@ class ConnectivityAction implements ActionInterface
throw new \Exception('Bridge is not whitelisted!');
}
- $retVal = [
- 'bridge' => $bridgeClassName,
- 'successful' => false,
- 'http_code' => 200,
- ];
-
$bridge = $this->bridgeFactory->create($bridgeClassName);
$curl_opts = [
- CURLOPT_CONNECTTIMEOUT => 5
+ CURLOPT_CONNECTTIMEOUT => 5,
+ CURLOPT_FOLLOWLOCATION => true,
+ ];
+ $result = [
+ 'bridge' => $bridgeClassName,
+ 'successful' => false,
+ 'http_code' => null,
];
try {
- $reply = getContents($bridge::URI, [], $curl_opts, true);
-
- if ($reply['code'] === 200) {
- $retVal['successful'] = true;
- if (strpos(implode('', $reply['status_lines']), '301 Moved Permanently')) {
- $retVal['http_code'] = 301;
- }
+ $response = getContents($bridge::URI, [], $curl_opts, true);
+ $result['http_code'] = $response->getCode();
+ if (in_array($result['http_code'], [200])) {
+ $result['successful'] = true;
}
} catch (\Exception $e) {
- $retVal['successful'] = false;
}
- return new Response(Json::encode($retVal), 200, ['Content-Type' => 'text/json']);
+ return new Response(Json::encode($result), 200, ['content-type' => 'text/json']);
}
}
diff --git a/actions/DetectAction.php b/actions/DetectAction.php
index 6c9fa22d..8d3d6263 100644
--- a/actions/DetectAction.php
+++ b/actions/DetectAction.php
@@ -1,53 +1,51 @@
bridgeFactory = $bridgeFactory;
+ }
+
+ public function __invoke(Request $request): Response
+ {
+ $url = $request->get('url');
+ $format = $request->get('format');
+
+ if (!$url) {
+ return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a url']));
}
if (!$format) {
- throw new \Exception('You must specify a format!');
+ return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']));
}
- $bridgeFactory = new BridgeFactory();
-
- foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
- if (!$bridgeFactory->isEnabled($bridgeClassName)) {
+ foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
+ if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
continue;
}
- $bridge = $bridgeFactory->create($bridgeClassName);
+ $bridge = $this->bridgeFactory->create($bridgeClassName);
- $bridgeParams = $bridge->detectParameters($targetURL);
+ $bridgeParams = $bridge->detectParameters($url);
- if (is_null($bridgeParams)) {
+ if (!$bridgeParams) {
continue;
}
- $bridgeParams['bridge'] = $bridgeClassName;
- $bridgeParams['format'] = $format;
-
- $url = '?action=display&' . http_build_query($bridgeParams);
- return new Response('', 301, ['Location' => $url]);
+ $query = [
+ 'action' => 'display',
+ 'bridge' => $bridgeClassName,
+ 'format' => $format,
+ ];
+ $query = array_merge($query, $bridgeParams);
+ return new Response('', 301, ['location' => '?' . http_build_query($query)]);
}
- throw new \Exception('No bridge found for given URL: ' . $targetURL);
+ return new Response(render(__DIR__ . '/../templates/error.html.php', [
+ 'message' => 'No bridge found for given URL: ' . $url,
+ ]));
}
}
diff --git a/actions/DisplayAction.php b/actions/DisplayAction.php
index 66acd4cc..97f53caa 100644
--- a/actions/DisplayAction.php
+++ b/actions/DisplayAction.php
@@ -1,44 +1,43 @@
cache = $cache;
+ $this->logger = $logger;
+ $this->bridgeFactory = $bridgeFactory;
+ }
+
+ public function __invoke(Request $request): Response
{
- if (Configuration::getConfig('system', 'enable_maintenance_mode')) {
- return new Response('503 Service Unavailable', 503);
+ $bridgeName = $request->get('bridge');
+ $format = $request->get('format');
+ $noproxy = $request->get('_noproxy');
+
+ if (!$bridgeName) {
+ return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Missing bridge name parameter']), 400);
+ }
+ $bridgeClassName = $this->bridgeFactory->createBridgeClassName($bridgeName);
+ if (!$bridgeClassName) {
+ return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'Bridge not found']), 404);
}
- $bridgeFactory = new BridgeFactory();
-
- $bridgeClassName = $bridgeFactory->createBridgeClassName($request['bridge'] ?? '');
-
- $format = $request['format'] ?? null;
if (!$format) {
- throw new \Exception('You must specify a format!');
+ return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'You must specify a format']), 400);
}
- if (!$bridgeFactory->isEnabled($bridgeClassName)) {
- throw new \Exception('This bridge is not whitelisted');
+ if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
+ return new Response(render(__DIR__ . '/../templates/error.html.php', ['message' => 'This bridge is not whitelisted']), 400);
}
- $formatFactory = new FormatFactory();
- $format = $formatFactory->create($format);
-
- $bridge = $bridgeFactory->create($bridgeClassName);
- $bridge->loadConfiguration();
-
- $noproxy = $request['_noproxy'] ?? null;
+ // Disable proxy (if enabled and per user's request)
if (
Configuration::getConfig('proxy', 'url')
&& Configuration::getConfig('proxy', 'by_bridge')
@@ -48,176 +47,134 @@ class DisplayAction implements ActionInterface
define('NOPROXY', true);
}
- $cacheTimeout = $request['_cache_timeout'] ?? null;
- if (Configuration::getConfig('cache', 'custom_timeout') && $cacheTimeout) {
- $cacheTimeout = (int) $cacheTimeout;
- } else {
- // At this point the query argument might still be in the url but it won't be used
- $cacheTimeout = $bridge->getCacheTimeout();
+ $cacheKey = 'http_' . json_encode($request->toArray());
+
+ $bridge = $this->bridgeFactory->create($bridgeClassName);
+
+ $response = $this->createResponse($request, $bridge, $format);
+
+ if ($response->getCode() === 200) {
+ $ttl = $request->get('_cache_timeout');
+ if (Configuration::getConfig('cache', 'custom_timeout') && $ttl) {
+ $ttl = (int) $ttl;
+ } else {
+ $ttl = $bridge->getCacheTimeout();
+ }
+ $this->cache->set($cacheKey, $response, $ttl);
}
- // Remove parameters that don't concern bridges
- $bridge_params = array_diff_key(
- $request,
- array_fill_keys(
- [
- 'action',
- 'bridge',
- 'format',
- '_noproxy',
- '_cache_timeout',
- '_error_time'
- ],
- ''
- )
- );
-
- // Remove parameters that don't concern caches
- $cache_params = array_diff_key(
- $request,
- array_fill_keys(
- [
- 'action',
- 'format',
- '_noproxy',
- '_cache_timeout',
- '_error_time'
- ],
- ''
- )
- );
-
- $cache = RssBridge::getCache();
- $cache->setScope('');
- $cache->setKey($cache_params);
- // This cache purge will basically delete all cache items older than 24h, regardless of scope and key
- $cache->purgeCache(86400);
-
- $items = [];
- $infos = [];
- $mtime = $cache->getTime();
-
- if (
- $mtime
- && (time() - $cacheTimeout < $mtime)
- && !Debug::isEnabled()
- ) {
- // At this point we found the feed in the cache and debug mode is disabled
-
- if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) {
- // The client wants to know if the feed has changed since its last check
- $stime = strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE']);
- if ($mtime <= $stime) {
- $lastModified2 = gmdate('D, d M Y H:i:s ', $mtime) . 'GMT';
- return new Response('', 304, ['Last-Modified' => $lastModified2]);
- }
- }
-
- // Load the feed from cache and prepare it
- $cached = $cache->loadData();
- if (isset($cached['items']) && isset($cached['extraInfos'])) {
- foreach ($cached['items'] as $item) {
- $items[] = new FeedItem($item);
- }
- $infos = $cached['extraInfos'];
- }
- } else {
- // At this point we did NOT find the feed in the cache or debug mode is enabled.
- try {
- $bridge->setDatas($bridge_params);
- $bridge->collectData();
-
- $items = $bridge->getItems();
-
- if (isset($items[0]) && is_array($items[0])) {
- $feedItems = [];
- foreach ($items as $item) {
- $feedItems[] = new FeedItem($item);
- }
- $items = $feedItems;
- }
- $infos = [
- 'name' => $bridge->getName(),
- 'uri' => $bridge->getURI(),
- 'donationUri' => $bridge->getDonationURI(),
- 'icon' => $bridge->getIcon()
- ];
- } catch (\Throwable $e) {
- if ($e instanceof HttpException) {
- // Produce a smaller log record for http exceptions
- Logger::warning(sprintf('Exception in %s: %s', $bridgeClassName, create_sane_exception_message($e)));
- } else {
- // Log the exception
- Logger::error(sprintf('Exception in %s', $bridgeClassName), ['e' => $e]);
- }
-
- // Emit error only if we are passed the error report limit
- $errorCount = self::logBridgeError($bridge->getName(), $e->getCode());
- if ($errorCount >= Configuration::getConfig('error', 'report_limit')) {
- if (Configuration::getConfig('error', 'output') === 'feed') {
- // Emit the error as a feed item in a feed so that feed readers can pick it up
- $items[] = $this->createFeedItemFromException($e, $bridge);
- } elseif (Configuration::getConfig('error', 'output') === 'http') {
- // Emit as a regular web response
- throw $e;
- }
- }
- }
-
- // Unfortunately need to set scope and key again because they might be modified
- $cache->setScope('');
- $cache->setKey($cache_params);
- $cache->saveData([
- 'items' => array_map(function (FeedItem $item) {
- return $item->toArray();
- }, $items),
- 'extraInfos' => $infos
- ]);
- }
-
- $format->setItems($items);
- $format->setExtraInfos($infos);
- $lastModified = $cache->getTime();
- $format->setLastModified($lastModified);
- $headers = [];
- if ($lastModified) {
- $headers['Last-Modified'] = gmdate('D, d M Y H:i:s ', $lastModified) . 'GMT';
- }
- $headers['Content-Type'] = $format->getMimeType() . '; charset=' . $format->getCharset();
- return new Response($format->stringify(), 200, $headers);
+ return $response;
}
- private function createFeedItemFromException($e, BridgeInterface $bridge): FeedItem
+ private function createResponse(Request $request, BridgeAbstract $bridge, string $format)
{
- $item = new FeedItem();
+ $items = [];
+
+ try {
+ $bridge->loadConfiguration();
+ // Remove parameters that don't concern bridges
+ $remove = [
+ 'token',
+ 'action',
+ 'bridge',
+ 'format',
+ '_noproxy',
+ '_cache_timeout',
+ '_error_time',
+ '_', // Some RSS readers add a cache-busting parameter (_=) to feed URLs, detect and ignore them.
+ ];
+ $requestArray = $request->toArray();
+ $input = array_diff_key($requestArray, array_fill_keys($remove, ''));
+ $bridge->setInput($input);
+ $bridge->collectData();
+ $items = $bridge->getItems();
+ } catch (\Throwable $e) {
+ if ($e instanceof ClientException) {
+ $this->logger->debug(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
+ } elseif ($e instanceof RateLimitException) {
+ $this->logger->debug(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
+ return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 429);
+ } elseif ($e instanceof HttpException) {
+ if (in_array($e->getCode(), [429, 503])) {
+ // Log with debug, immediately reproduce and return
+ $this->logger->debug(sprintf('Exception in DisplayAction(%s): %s', $bridge->getShortName(), create_sane_exception_message($e)));
+ return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), $e->getCode());
+ }
+ // Some other status code which we let fail normally (but don't log it)
+ } else {
+ $this->logger->error(sprintf('Exception in DisplayAction(%s)', $bridge->getShortName()), ['e' => $e]);
+ }
+ $errorOutput = Configuration::getConfig('error', 'output');
+ $reportLimit = Configuration::getConfig('error', 'report_limit');
+ $errorCount = 1;
+ if ($reportLimit > 1) {
+ $errorCount = $this->logBridgeError($bridge->getName(), $e->getCode());
+ }
+ // Let clients know about the error if we are passed the report limit
+ if ($errorCount >= $reportLimit) {
+ if ($errorOutput === 'feed') {
+ // Render the exception as a feed item
+ $items = [$this->createFeedItemFromException($e, $bridge)];
+ } elseif ($errorOutput === 'http') {
+ return new Response(render(__DIR__ . '/../templates/exception.html.php', ['e' => $e]), 500);
+ } elseif ($errorOutput === 'none') {
+ // Do nothing (produces an empty feed)
+ }
+ }
+ }
+
+ $formatFactory = new FormatFactory();
+ $format = $formatFactory->create($format);
+
+ $format->setItems($items);
+ $format->setFeed($bridge->getFeed());
+ $now = time();
+ $format->setLastModified($now);
+ $headers = [
+ 'last-modified' => gmdate('D, d M Y H:i:s ', $now) . 'GMT',
+ 'content-type' => $format->getMimeType() . '; charset=UTF-8',
+ ];
+ $body = $format->render();
+
+ // This is supposed to remove non-utf8 byte sequences, but I'm unsure if it works
+ ini_set('mbstring.substitute_character', 'none');
+ $body = mb_convert_encoding($body, 'UTF-8', 'UTF-8');
+
+ return new Response($body, 200, $headers);
+ }
+
+ private function createFeedItemFromException($e, BridgeAbstract $bridge): array
+ {
+ $item = [];
// Create a unique identifier every 24 hours
$uniqueIdentifier = urlencode((int)(time() / 86400));
- $itemTitle = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier);
- $item->setTitle($itemTitle);
- $item->setURI(get_current_url());
- $item->setTimestamp(time());
+ $title = sprintf('Bridge returned error %s! (%s)', $e->getCode(), $uniqueIdentifier);
+
+ $item['title'] = $title;
+ $item['uri'] = get_current_url();
+ $item['timestamp'] = time();
// Create an item identifier for feed readers e.g. "staysafetv twitch videos_19389"
- $item->setUid($bridge->getName() . '_' . $uniqueIdentifier);
+ $item['uid'] = $bridge->getName() . '_' . $uniqueIdentifier;
$content = render_template(__DIR__ . '/../templates/bridge-error.html.php', [
- 'error' => render_template(__DIR__ . '/../templates/error.html.php', ['e' => $e]),
+ 'error' => render_template(__DIR__ . '/../templates/exception.html.php', ['e' => $e]),
'searchUrl' => self::createGithubSearchUrl($bridge),
- 'issueUrl' => self::createGithubIssueUrl($bridge, $e, create_sane_exception_message($e)),
+ 'issueUrl' => self::createGithubIssueUrl($bridge, $e),
'maintainer' => $bridge->getMaintainer(),
]);
- $item->setContent($content);
+ $item['content'] = $content;
+
return $item;
}
- private static function logBridgeError($bridgeName, $code)
+ private function logBridgeError($bridgeName, $code)
{
- $cache = RssBridge::getCache();
- $cache->setScope('error_reporting');
- $cache->setkey([$bridgeName . '_' . $code]);
-
- if ($report = $cache->loadData()) {
+ // todo: it's not really necessary to json encode $report
+ $cacheKey = 'error_reporting_' . $bridgeName . '_' . $code;
+ $report = $this->cache->get($cacheKey);
+ if ($report) {
$report = Json::decode($report);
$report['time'] = time();
$report['count']++;
@@ -228,26 +185,39 @@ class DisplayAction implements ActionInterface
'count' => 1,
];
}
- $cache->saveData(Json::encode($report));
+ $ttl = 86400 * 5;
+ $this->cache->set($cacheKey, Json::encode($report), $ttl);
return $report['count'];
}
- private static function createGithubIssueUrl($bridge, $e, string $message): string
+ private static function createGithubIssueUrl(BridgeAbstract $bridge, \Throwable $e): string
{
- return sprintf('https://github.com/RSS-Bridge/rss-bridge/issues/new?%s', http_build_query([
- 'title' => sprintf('%s failed with error %s', $bridge->getName(), $e->getCode()),
+ $maintainer = $bridge->getMaintainer();
+ if (str_contains($maintainer, ',')) {
+ $maintainers = explode(',', $maintainer);
+ } else {
+ $maintainers = [$maintainer];
+ }
+ $maintainers = array_map('trim', $maintainers);
+
+ $queryString = $_SERVER['QUERY_STRING'] ?? '';
+ $query = [
+ 'title' => $bridge->getName() . ' failed with: ' . $e->getMessage(),
'body' => sprintf(
- "```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```",
- $message,
+ "```\n%s\n\n%s\n\nQuery string: %s\nVersion: %s\nOs: %s\nPHP version: %s\n```\nMaintainer: @%s",
+ create_sane_exception_message($e),
implode("\n", trace_to_call_points(trace_from_exception($e))),
- $_SERVER['QUERY_STRING'] ?? '',
+ $queryString,
Configuration::getVersion(),
PHP_OS_FAMILY,
- phpversion() ?: 'Unknown'
+ phpversion() ?: 'Unknown',
+ implode(', @', $maintainers),
),
'labels' => 'Bridge-Broken',
- 'assignee' => $bridge->getMaintainer(),
- ]));
+ 'assignee' => $maintainer[0],
+ ];
+
+ return 'https://github.com/RSS-Bridge/rss-bridge/issues/new?' . http_build_query($query);
}
private static function createGithubSearchUrl($bridge): string
diff --git a/actions/FindfeedAction.php b/actions/FindfeedAction.php
new file mode 100644
index 00000000..e18c3e1d
--- /dev/null
+++ b/actions/FindfeedAction.php
@@ -0,0 +1,95 @@
+bridgeFactory = $bridgeFactory;
+ }
+
+ public function __invoke(Request $request): Response
+ {
+ $url = $request->get('url');
+ $format = $request->get('format');
+
+ if (!$url) {
+ return new Response('You must specify a url', 400);
+ }
+ if (!$format) {
+ return new Response('You must specify a format', 400);
+ }
+
+ $results = [];
+ foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
+ if (!$this->bridgeFactory->isEnabled($bridgeClassName)) {
+ continue;
+ }
+
+ $bridge = $this->bridgeFactory->create($bridgeClassName);
+
+ $bridgeParams = $bridge->detectParameters($url);
+
+ if ($bridgeParams === null) {
+ continue;
+ }
+
+ // It's allowed to have no 'context' in a bridge (only a default context without any name)
+ // In this case, the reference to the parameters are found in the first element of the PARAMETERS array
+
+ $context = $bridgeParams['context'] ?? 0;
+
+ $bridgeData = [];
+ // Construct the array of parameters
+ foreach ($bridgeParams as $key => $value) {
+ // 'context' is a special case : it's a bridge parameters, there is no "name" for this parameter
+ if ($key == 'context') {
+ $bridgeData[$key]['name'] = 'Context';
+ $bridgeData[$key]['value'] = $value;
+ } else {
+ $bridgeData[$key]['name'] = $this->getParameterName($bridge, $context, $key);
+ $bridgeData[$key]['value'] = $value;
+ }
+ }
+
+ $bridgeParams['bridge'] = $bridgeClassName;
+ $bridgeParams['format'] = $format;
+ $content = [
+ 'url' => './?action=display&' . http_build_query($bridgeParams),
+ 'bridgeParams' => $bridgeParams,
+ 'bridgeData' => $bridgeData,
+ 'bridgeMeta' => [
+ 'name' => $bridge::NAME,
+ 'description' => $bridge::DESCRIPTION,
+ 'parameters' => $bridge::PARAMETERS,
+ 'icon' => $bridge->getIcon(),
+ ],
+ ];
+ $results[] = $content;
+ }
+ if ($results === []) {
+ return new Response(Json::encode(['message' => 'No bridge found for given url']), 404, ['content-type' => 'application/json']);
+ }
+ return new Response(Json::encode($results), 200, ['content-type' => 'application/json']);
+ }
+
+ // Get parameter name in the actual context, or in the global parameter
+ private function getParameterName($bridge, $context, $key)
+ {
+ if (isset($bridge::PARAMETERS[$context][$key]['name'])) {
+ $name = $bridge::PARAMETERS[$context][$key]['name'];
+ } else if (isset($bridge::PARAMETERS['global'][$key]['name'])) {
+ $name = $bridge::PARAMETERS['global'][$key]['name'];
+ } else {
+ $name = 'Variable "' . $key . '" (No name provided)';
+ }
+ return $name;
+ }
+}
diff --git a/actions/FrontpageAction.php b/actions/FrontpageAction.php
index 40d25ea4..79ffb4f5 100644
--- a/actions/FrontpageAction.php
+++ b/actions/FrontpageAction.php
@@ -2,35 +2,48 @@
final class FrontpageAction implements ActionInterface
{
- public function execute(array $request)
+ private BridgeFactory $bridgeFactory;
+
+ public function __construct(
+ BridgeFactory $bridgeFactory
+ ) {
+ $this->bridgeFactory = $bridgeFactory;
+ }
+
+ public function __invoke(Request $request): Response
{
- $showInactive = (bool) ($request['show_inactive'] ?? null);
+ $token = $request->getAttribute('token');
+
+ $messages = [];
$activeBridges = 0;
- $bridgeFactory = new BridgeFactory();
- $bridgeClassNames = $bridgeFactory->getBridgeClassNames();
+ $bridgeClassNames = $this->bridgeFactory->getBridgeClassNames();
- $formatFactory = new FormatFactory();
- $formats = $formatFactory->getFormatNames();
+ foreach ($this->bridgeFactory->getMissingEnabledBridges() as $missingEnabledBridge) {
+ $messages[] = [
+ 'body' => sprintf('Warning : Bridge "%s" not found', $missingEnabledBridge),
+ 'level' => 'warning'
+ ];
+ }
$body = '';
foreach ($bridgeClassNames as $bridgeClassName) {
- if ($bridgeFactory->isEnabled($bridgeClassName)) {
- $body .= BridgeCard::displayBridgeCard($bridgeClassName, $formats);
+ if ($this->bridgeFactory->isEnabled($bridgeClassName)) {
+ $body .= BridgeCard::render($this->bridgeFactory, $bridgeClassName, $token);
$activeBridges++;
- } elseif ($showInactive) {
- $body .= BridgeCard::displayBridgeCard($bridgeClassName, $formats, false) . PHP_EOL;
}
}
- return render(__DIR__ . '/../templates/frontpage.html.php', [
- 'messages' => [],
- 'admin_email' => Configuration::getConfig('admin', 'email'),
- 'admin_telegram' => Configuration::getConfig('admin', 'telegram'),
- 'bridges' => $body,
- 'active_bridges' => $activeBridges,
- 'total_bridges' => count($bridgeClassNames),
- 'show_inactive' => $showInactive,
- ]);
+ $response = new Response(render(__DIR__ . '/../templates/frontpage.html.php', [
+ 'messages' => $messages,
+ 'admin_email' => Configuration::getConfig('admin', 'email'),
+ 'admin_telegram' => Configuration::getConfig('admin', 'telegram'),
+ 'bridges' => $body,
+ 'active_bridges' => $activeBridges,
+ 'total_bridges' => count($bridgeClassNames),
+ ]));
+
+ // TODO: The rendered template could be cached, but beware config changes that changes the html
+ return $response;
}
}
diff --git a/actions/HealthAction.php b/actions/HealthAction.php
index 8ae5df1b..13365a3c 100644
--- a/actions/HealthAction.php
+++ b/actions/HealthAction.php
@@ -4,7 +4,7 @@ declare(strict_types=1);
class HealthAction implements ActionInterface
{
- public function execute(array $request)
+ public function __invoke(Request $request): Response
{
$response = [
'code' => 200,
diff --git a/actions/ListAction.php b/actions/ListAction.php
index 6ce7e33e..f6347f9c 100644
--- a/actions/ListAction.php
+++ b/actions/ListAction.php
@@ -1,42 +1,36 @@
bridgeFactory = $bridgeFactory;
+ }
+
+ public function __invoke(Request $request): Response
{
$list = new \stdClass();
$list->bridges = [];
$list->total = 0;
- $bridgeFactory = new BridgeFactory();
-
- foreach ($bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
- $bridge = $bridgeFactory->create($bridgeClassName);
+ foreach ($this->bridgeFactory->getBridgeClassNames() as $bridgeClassName) {
+ $bridge = $this->bridgeFactory->create($bridgeClassName);
$list->bridges[$bridgeClassName] = [
- 'status' => $bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive',
- 'uri' => $bridge->getURI(),
- 'donationUri' => $bridge->getDonationURI(),
- 'name' => $bridge->getName(),
- 'icon' => $bridge->getIcon(),
- 'parameters' => $bridge->getParameters(),
- 'maintainer' => $bridge->getMaintainer(),
- 'description' => $bridge->getDescription()
+ 'status' => $this->bridgeFactory->isEnabled($bridgeClassName) ? 'active' : 'inactive',
+ 'uri' => $bridge->getURI(),
+ 'donationUri' => $bridge->getDonationURI(),
+ 'name' => $bridge->getName(),
+ 'icon' => $bridge->getIcon(),
+ 'parameters' => $bridge->getParameters(),
+ 'maintainer' => $bridge->getMaintainer(),
+ 'description' => $bridge->getDescription()
];
}
$list->total = count($list->bridges);
- return new Response(Json::encode($list), 200, ['Content-Type' => 'application/json']);
+ return new Response(Json::encode($list), 200, ['content-type' => 'application/json']);
}
}
diff --git a/actions/SetBridgeCacheAction.php b/actions/SetBridgeCacheAction.php
deleted file mode 100644
index a9a598bd..00000000
--- a/actions/SetBridgeCacheAction.php
+++ /dev/null
@@ -1,50 +0,0 @@
-createBridgeClassName($request['bridge'] ?? '');
-
- // whitelist control
- if (!$bridgeFactory->isEnabled($bridgeClassName)) {
- throw new \Exception('This bridge is not whitelisted', 401);
- die;
- }
-
- $bridge = $bridgeFactory->create($bridgeClassName);
- $bridge->loadConfiguration();
- $value = $request['value'];
-
- $cache = RssBridge::getCache();
- $cache->setScope(get_class($bridge));
- if (!is_array($key)) {
- // not sure if $key is an array when it comes in from request
- $key = [$key];
- }
- $cache->setKey($key);
- $cache->saveData($value);
-
- header('Content-Type: text/plain');
- echo 'done';
- }
-}
diff --git a/bin/cache-clear b/bin/cache-clear
new file mode 100755
index 00000000..2ca84ce6
--- /dev/null
+++ b/bin/cache-clear
@@ -0,0 +1,16 @@
+#!/usr/bin/env php
+clear();
diff --git a/bin/cache-prune b/bin/cache-prune
new file mode 100755
index 00000000..bb72c4ac
--- /dev/null
+++ b/bin/cache-prune
@@ -0,0 +1,24 @@
+#!/usr/bin/env php
+prune();
diff --git a/bin/test b/bin/test
new file mode 100755
index 00000000..74692410
--- /dev/null
+++ b/bin/test
@@ -0,0 +1,20 @@
+#!/usr/bin/env php
+debug('This is a test debug message');
+
+$logger->info('This is a test info message');
+
+$logger->error('This is a test error message');
diff --git a/bridges/ABCNewsBridge.php b/bridges/ABCNewsBridge.php
index c00fed1c..154eb489 100644
--- a/bridges/ABCNewsBridge.php
+++ b/bridges/ABCNewsBridge.php
@@ -31,17 +31,17 @@ class ABCNewsBridge extends BridgeAbstract
{
$url = sprintf('https://www.abc.net.au/news/%s', $this->getInput('topic'));
$dom = getSimpleHTMLDOM($url);
- $dom = $dom->find('div[data-component="CardList"]', 0);
+ $dom = $dom->find('div[data-component="PaginationList"]', 0);
if (!$dom) {
throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
}
$dom = defaultLinkTo($dom, $this->getURI());
- foreach ($dom->find('div[data-component="GenericCard"]') as $article) {
+ foreach ($dom->find('article[data-component="DetailCard"]') as $article) {
$a = $article->find('a', 0);
$this->items[] = [
'title' => $a->plaintext,
'uri' => $a->href,
- 'content' => $article->find('[data-component="CardDescription"]', 0)->plaintext,
+ 'content' => $article->find('p', 0)->plaintext,
'timestamp' => strtotime($article->find('time', 0)->datetime),
];
}
diff --git a/bridges/AO3Bridge.php b/bridges/AO3Bridge.php
index 6ca59cc5..2697dbc7 100644
--- a/bridges/AO3Bridge.php
+++ b/bridges/AO3Bridge.php
@@ -12,9 +12,29 @@ class AO3Bridge extends BridgeAbstract
'url' => [
'name' => 'url',
'required' => true,
- // Example: F/F tag, complete works only
- 'exampleValue' => 'https://archiveofourown.org/works?work_search[complete]=T&tag_id=F*s*F',
+ // Example: F/F tag
+ 'exampleValue' => 'https://archiveofourown.org/tags/F*s*F/works',
],
+ 'range' => [
+ 'name' => 'Chapter Content',
+ 'title' => 'Chapter(s) to include in each work\'s feed entry',
+ 'defaultValue' => null,
+ 'type' => 'list',
+ 'values' => [
+ 'None' => null,
+ 'First' => 'first',
+ 'Latest' => 'last',
+ 'Entire work' => 'all',
+ ],
+ ],
+ 'unique' => [
+ 'name' => 'Make separate entries for new fic chapters',
+ 'type' => 'checkbox',
+ 'required' => false,
+ 'title' => 'Make separate entries for new fic chapters',
+ 'defaultValue' => 'checked',
+ ],
+ 'limit' => self::LIMIT,
],
'Bookmarks' => [
'user' => [
@@ -33,23 +53,19 @@ class AO3Bridge extends BridgeAbstract
],
]
];
+ private $title;
public function collectData()
{
switch ($this->queriedContext) {
case 'Bookmarks':
- $user = $this->getInput('user');
- $this->title = $user;
- $url = self::URI
- . '/users/' . $user
- . '/bookmarks?bookmark_search[sort_column]=bookmarkable_date';
- $this->collectList($url);
+ $this->collectList($this->getURI());
break;
case 'List':
- $this->collectList($this->getInput('url'));
+ $this->collectList($this->getURI());
break;
case 'Work':
- $this->collectWork($this->getInput('id'));
+ $this->collectWork($this->getURI());
break;
}
}
@@ -60,9 +76,24 @@ class AO3Bridge extends BridgeAbstract
*/
private function collectList($url)
{
- $html = getSimpleHTMLDOM($url);
+ $version = 'v0.0.1';
+ $headers = [
+ "useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
+ ];
+ $response = getContents($url, $headers);
+
+ $html = \str_get_html($response);
$html = defaultLinkTo($html, self::URI);
+ // Get list title. Will include page range + count in some cases
+ $heading = ($html->find('#main h2', 0));
+ if ($heading->find('a.tag')) {
+ $heading = $heading->find('a.tag', 0);
+ }
+ $this->title = $heading->plaintext;
+
+ $limit = $this->getInput('limit') ?? 3;
+ $count = 0;
foreach ($html->find('.index.group > li') as $element) {
$item = [];
@@ -71,16 +102,70 @@ class AO3Bridge extends BridgeAbstract
continue; // discard deleted works
}
$item['title'] = $title->plaintext;
- $item['content'] = $element;
$item['uri'] = $title->href;
$strdate = $element->find('div p.datetime', 0)->plaintext;
$item['timestamp'] = strtotime($strdate);
+ // detach from rest of page because remove() is buggy
+ $element = str_get_html($element->outertext());
+ $tags = $element->find('ul.required-tags', 0);
+ foreach ($tags->childNodes() as $tag) {
+ $item['categories'][] = html_entity_decode($tag->plaintext);
+ }
+ $tags->remove();
+ $tags = $element->find('ul.tags', 0);
+ foreach ($tags->childNodes() as $tag) {
+ $item['categories'][] = html_entity_decode($tag->plaintext);
+ }
+ $tags->remove();
+
+ $item['content'] = implode('', $element->childNodes());
+
$chapters = $element->find('dl dd.chapters', 0);
// bookmarked series and external works do not have a chapters count
$chapters = (isset($chapters) ? $chapters->plaintext : 0);
- $item['uid'] = $item['uri'] . "/$strdate/$chapters";
+ if ($this->getInput('unique')) {
+ $item['uid'] = $item['uri'] . "/$strdate/$chapters";
+ } else {
+ $item['uid'] = $item['uri'];
+ }
+
+
+ // Fetch workskin of desired chapter(s) in list
+ if ($this->getInput('range') && ($limit == 0 || $count++ < $limit)) {
+ $url = $item['uri'];
+ switch ($this->getInput('range')) {
+ case ('all'):
+ $url .= '?view_full_work=true';
+ break;
+ case ('first'):
+ break;
+ case ('last'):
+ // only way to get this is using the navigate page unfortunately
+ $url .= '/navigate';
+ $response = getContents($url, $headers);
+ $html = \str_get_html($response);
+ $html = defaultLinkTo($html, self::URI);
+ $url = $html->find('ol.index.group > li > a', -1)->href;
+ break;
+ }
+ $response = getContents($url, $headers);
+
+ $html = \str_get_html($response);
+ $html = defaultLinkTo($html, self::URI);
+ // remove duplicate fic summary
+ if ($ficsum = $html->find('#workskin > .preface > .summary', 0)) {
+ $ficsum->remove();
+ }
+ $item['content'] .= $html->find('#workskin', 0);
+ }
+
+ // Use predictability of download links to generate enclosures
+ $wid = explode('/', $item['uri'])[4];
+ foreach (['azw3', 'epub', 'mobi', 'pdf', 'html'] as $ext) {
+ $item['enclosures'][] = 'https://archiveofourown.org/downloads/' . $wid . '/work.' . $ext;
+ }
$this->items[] = $item;
}
@@ -89,20 +174,31 @@ class AO3Bridge extends BridgeAbstract
/**
* Feed for recent chapters of a specific work.
*/
- private function collectWork($id)
+ private function collectWork($url)
{
- $url = self::URI . "/works/$id/navigate";
- $response = _http_request($url, ['useragent' => 'rss-bridge bot (https://github.com/RSS-Bridge/rss-bridge)']);
- $html = \str_get_html($response['body']);
+ $version = 'v0.0.1';
+ $headers = [
+ "useragent: rss-bridge $version (https://github.com/RSS-Bridge/rss-bridge)"
+ ];
+ $response = getContents($url . '/navigate', $headers);
+
+ $html = \str_get_html($response);
$html = defaultLinkTo($html, self::URI);
+ $response = getContents($url . '?view_full_work=true', $headers);
+
+ $workhtml = \str_get_html($response);
+ $workhtml = defaultLinkTo($workhtml, self::URI);
+
$this->title = $html->find('h2 a', 0)->plaintext;
- foreach ($html->find('ol.index.group > li') as $element) {
+ $nav = $html->find('ol.index.group > li');
+ for ($i = 0; $i < count($nav); $i++) {
$item = [];
+ $element = $nav[$i];
$item['title'] = $element->find('a', 0)->plaintext;
- $item['content'] = $element;
+ $item['content'] = $workhtml->find('#chapter-' . ($i + 1), 0);
$item['uri'] = $element->find('a', 0)->href;
$strdate = $element->find('span.datetime', 0)->plaintext;
@@ -131,4 +227,24 @@ class AO3Bridge extends BridgeAbstract
{
return self::URI . '/favicon.ico';
}
+
+ public function getURI()
+ {
+ $url = parent::getURI();
+ switch ($this->queriedContext) {
+ case 'Bookmarks':
+ $user = $this->getInput('user');
+ $url = self::URI
+ . '/users/' . $user
+ . '/bookmarks?bookmark_search[sort_column]=bookmarkable_date';
+ break;
+ case 'List':
+ $url = $this->getInput('url');
+ break;
+ case 'Work':
+ $url = self::URI . '/works/' . $this->getInput('id');
+ break;
+ }
+ return $url;
+ }
}
diff --git a/bridges/ARDAudiothekBridge.php b/bridges/ARDAudiothekBridge.php
index 07a3cc68..e6b82775 100644
--- a/bridges/ARDAudiothekBridge.php
+++ b/bridges/ARDAudiothekBridge.php
@@ -63,13 +63,15 @@ class ARDAudiothekBridge extends BridgeAbstract
public function collectData()
{
- $oldTz = date_default_timezone_get();
+ $path = $this->getInput('path');
+ $limit = $this->getInput('limit');
+ $oldTz = date_default_timezone_get();
date_default_timezone_set('Europe/Berlin');
- $pathComponents = explode('/', $this->getInput('path'));
+ $pathComponents = explode('/', $path);
if (empty($pathComponents)) {
- returnClientError('Path may not be empty');
+ throwClientException('Path may not be empty');
}
if (count($pathComponents) < 2) {
$showID = $pathComponents[0];
@@ -82,17 +84,21 @@ class ARDAudiothekBridge extends BridgeAbstract
}
$url = self::APIENDPOINT . 'programsets/' . $showID . '/';
- $rawJSON = getContents($url);
- $processedJSON = json_decode($rawJSON)->data->programSet;
+ $json1 = getContents($url);
+ $data1 = Json::decode($json1, false);
+ $processedJSON = $data1->data->programSet;
+ if (!$processedJSON) {
+ throw new \Exception('Unable to find show id: ' . $showID);
+ }
- $limit = $this->getInput('limit');
$answerLength = 1;
$offset = 0;
$numberOfElements = 1;
while ($answerLength != 0 && $offset < $numberOfElements && (is_null($limit) || $offset < $limit)) {
- $rawJSON = getContents($url . '?offset=' . $offset);
- $processedJSON = json_decode($rawJSON)->data->programSet;
+ $json2 = getContents($url . '?offset=' . $offset);
+ $data2 = Json::decode($json2, false);
+ $processedJSON = $data2->data->programSet;
$answerLength = count($processedJSON->items->nodes);
$offset = $offset + $answerLength;
@@ -113,7 +119,16 @@ class ARDAudiothekBridge extends BridgeAbstract
$item['timestamp'] = $audio->publicationStartDateAndTime;
$item['uid'] = $audio->id;
$item['author'] = $audio->programSet->publicationService->title;
- $item['categories'] = [ $audio->programSet->editorialCategories->title ];
+
+ $category = $audio->programSet->editorialCategories->title ?? null;
+ if ($category) {
+ $item['categories'] = [$category];
+ }
+
+ $item['itunes'] = [
+ 'duration' => $audio->duration,
+ ];
+
$this->items[] = $item;
}
}
diff --git a/bridges/ARDMediathekBridge.php b/bridges/ARDMediathekBridge.php
index 6de8dad7..6f7272dc 100644
--- a/bridges/ARDMediathekBridge.php
+++ b/bridges/ARDMediathekBridge.php
@@ -40,6 +40,11 @@ class ARDMediathekBridge extends BridgeAbstract
* @const IMAGEWIDTHPLACEHOLDER
*/
const IMAGEWIDTHPLACEHOLDER = '{width}';
+ /**
+ * Title of the current show
+ * @var string
+ */
+ private $title;
const PARAMETERS = [
[
@@ -60,7 +65,7 @@ class ARDMediathekBridge extends BridgeAbstract
$pathComponents = explode('/', $this->getInput('path'));
if (empty($pathComponents)) {
- returnClientError('Path may not be empty');
+ throwClientException('Path may not be empty');
}
if (count($pathComponents) < 2) {
$showID = $pathComponents[0];
@@ -72,7 +77,7 @@ class ARDMediathekBridge extends BridgeAbstract
}
}
- $url = self::APIENDPOINT . $showID . '/?pageSize=' . self::PAGESIZE;
+ $url = self::APIENDPOINT . $showID . '?pageSize=' . self::PAGESIZE;
$rawJSON = getContents($url);
$processedJSON = json_decode($rawJSON);
@@ -93,6 +98,17 @@ class ARDMediathekBridge extends BridgeAbstract
$this->items[] = $item;
}
+ $this->title = $processedJSON->title;
+
date_default_timezone_set($oldTz);
}
+
+ /** {@inheritdoc} */
+ public function getName()
+ {
+ if (!empty($this->title)) {
+ return $this->title;
+ }
+ return parent::getName();
+ }
}
diff --git a/bridges/AcrimedBridge.php b/bridges/AcrimedBridge.php
index d37f3ce4..f7bbd58e 100644
--- a/bridges/AcrimedBridge.php
+++ b/bridges/AcrimedBridge.php
@@ -20,17 +20,14 @@ class AcrimedBridge extends FeedExpander
public function collectData()
{
- $this->collectExpandableDatas(
- static::URI . 'spip.php?page=backend',
- $this->getInput('limit')
- );
+ $url = 'https://www.acrimed.org/spip.php?page=backend';
+ $limit = $this->getInput('limit');
+ $this->collectExpandableDatas($url, $limit);
}
- protected function parseItem($newsItem)
+ protected function parseItem(array $item)
{
- $item = parent::parseItem($newsItem);
-
- $articlePage = getSimpleHTMLDOM($newsItem->link);
+ $articlePage = getSimpleHTMLDOM($item['uri']);
$article = sanitize($articlePage->find('article.article1', 0)->innertext);
$article = defaultLinkTo($article, static::URI);
$item['content'] = $article;
diff --git a/bridges/ActivisionResearchBridge.php b/bridges/ActivisionResearchBridge.php
new file mode 100644
index 00000000..88af4b46
--- /dev/null
+++ b/bridges/ActivisionResearchBridge.php
@@ -0,0 +1,45 @@
+find('div[id="home-blog-feed"]', 0);
+ if (!$dom) {
+ throw new \Exception(sprintf('Unable to find css selector on `%s`', $url));
+ }
+ $dom = defaultLinkTo($dom, $this->getURI());
+ foreach ($dom->find('div[class="blog-entry"]') as $article) {
+ $a = $article->find('a', 0);
+
+ $blogimg = extractFromDelimiters($article->find('div[class="blog-img"]', 0)->style, 'url(', ')');
+
+ $title = htmlspecialchars_decode($article->find('div[class="title"]', 0)->plaintext);
+ $author = htmlspecialchars_decode($article->find('div[class="author]', 0)->plaintext);
+ $date = $article->find('div[class="pubdate"]', 0)->plaintext;
+
+ $entry = getSimpleHTMLDOMCached($a->href, static::CACHE_TIMEOUT * 7 * 4);
+ $entry = defaultLinkTo($entry, $this->getURI());
+
+ $content = $entry->find('div[class="blog-body"]', 0);
+ $tagsremove = ['script', 'iframe', 'input', 'form'];
+ $content = sanitize($content, $tagsremove);
+ $content = '' . $content;
+
+ $this->items[] = [
+ 'title' => $title,
+ 'author' => $author,
+ 'uri' => $a->href,
+ 'content' => $content,
+ 'timestamp' => strtotime($date),
+ ];
+ }
+ }
+}
diff --git a/bridges/AirBreizhBridge.php b/bridges/AirBreizhBridge.php
index a822625f..272c74ee 100644
--- a/bridges/AirBreizhBridge.php
+++ b/bridges/AirBreizhBridge.php
@@ -32,8 +32,7 @@ class AirBreizhBridge extends BridgeAbstract
public function collectData()
{
$html = '';
- $html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'))
- or returnClientError('No results for this query.');
+ $html = getSimpleHTMLDOM(static::URI . 'publications/?fwp_publications_thematiques=' . $this->getInput('theme'));
foreach ($html->find('article') as $article) {
$item = [];
diff --git a/bridges/AllegroBridge.php b/bridges/AllegroBridge.php
index 55457416..3509afa1 100644
--- a/bridges/AllegroBridge.php
+++ b/bridges/AllegroBridge.php
@@ -13,26 +13,42 @@ class AllegroBridge extends BridgeAbstract
'exampleValue' => 'https://allegro.pl/kategoria/swieze-warzywa-cebula-318660',
'required' => true,
],
- 'sessioncookie' => [
- 'name' => 'The \'wdctx\' session cookie',
- 'title' => 'Paste the value of the \'wdctx\' cookie from your browser if you want to prevent Allegro imposing rate limits',
- 'pattern' => '^.{250,};?$',
- // phpcs:ignore
- 'exampleValue' => 'v4.1-oCrmXTMqv2ppC21GTUCKLmUwRPP1ssQVALKuqwsZ1VXjcKgL2vO5TTRM5xMxS9GiyqxF1gAeyc-63dl0coUoBKXCXi_nAmr95yyqGpq2RAFoneZ4L399E8n6iYyemcuGARjAoSfjvLHJCEwvvHHynSgaxlFBu7hUnKfuy39zo9sSQdyTUjotJg3CAZ53q9v2raAnPCyGOAR4ytRILd9p24EJnxp7_oR0XbVPIo1hDa4WmjXFOxph8rHaO5tWd',
- 'required' => false,
+ 'cookie' => [
+ 'name' => 'The complete cookie value',
+ 'title' => 'Paste the cookie value from your browser, otherwise 403 gets returned',
+ 'required' => true,
],
'includeSponsoredOffers' => [
'type' => 'checkbox',
- 'name' => 'Include Sponsored Offers'
+ 'name' => 'Include Sponsored Offers',
+ 'defaultValue' => 'checked'
+ ],
+ 'includePromotedOffers' => [
+ 'type' => 'checkbox',
+ 'name' => 'Include Promoted Offers',
+ 'defaultValue' => 'checked'
]
]];
public function getName()
{
- parse_str(parse_url($this->getInput('url'), PHP_URL_QUERY), $fields);
+ $url = $this->getInput('url');
+ if (!$url) {
+ return parent::getName();
+ }
+ $parsedUrl = parse_url($url, PHP_URL_QUERY);
+ if (!$parsedUrl) {
+ return parent::getName();
+ }
+ parse_str($parsedUrl, $fields);
- if ($query = array_key_exists('string', $fields) ? urldecode($fields['string']) : false) {
- return $query;
+ if (array_key_exists('string', $fields)) {
+ $f = urldecode($fields['string']);
+ } else {
+ $f = false;
+ }
+ if ($f) {
+ return $f;
}
return parent::getName();
@@ -49,96 +65,56 @@ class AllegroBridge extends BridgeAbstract
$url = preg_replace('/([?&])order=[^&]+(&|$)/', '$1', $this->getInput('url'));
$url .= (parse_url($url, PHP_URL_QUERY) ? '&' : '?') . 'order=n';
- $opts = [];
+ $html = getContents($url, [], [CURLOPT_COOKIE => $this->getInput('cookie')]);
- // If a session cookie is provided
- if ($sessioncookie = $this->getInput('sessioncookie')) {
- $opts[CURLOPT_COOKIE] = 'wdctx=' . $sessioncookie;
+ $storeData = null;
+ if (preg_match('/]*>\s*(\{\s*?"__listing_StoreState".*\})\s*<\/script>/i', $html, $match)) {
+ $data = json_decode($match[1], true);
+ $storeData = $data['__listing_StoreState'] ?? null;
}
- $html = getSimpleHTMLDOM($url, [], $opts);
+ foreach ($storeData['items']['elements'] as $elements) {
+ if (!array_key_exists('offerId', $elements)) {
+ continue;
+ }
+ if (!$this->getInput('includeSponsoredOffers') && $elements['isSponsored']) {
+ continue;
+ }
+ if (!$this->getInput('includePromotedOffers') && $elements['promoted']) {
+ continue;
+ }
- # if no results found
- if ($html->find('.mzmg_6m.m9qz_yo._6a66d_-fJr5')) {
- return;
- }
-
- $results = $html->find('._6a66d_V7Lel article');
-
- if (!$this->getInput('includeSponsoredOffers')) {
- $results = array_filter($results, function ($node) {
- return $node->{'data-analytics-view-label'} != 'showSponsoredItems';
- });
- }
-
- foreach ($results as $post) {
$item = [];
+ $item['uid'] = $elements['offerId'];
+ $item['uri'] = $elements['url'];
+ $item['title'] = $elements['alt'];
- $item['uri'] = $post->find('._6a66d_LX75-', 0)->href;
-
-//TODO: port this over, whatever it does, from https://github.com/MK-PL/AllegroRSS
-// if (arrayLinks.includes('events/clicks?')) {
-// let sponsoredLink = new URL(arrayLinks).searchParams.get('redirect')
-// arrayLinks = sponsoredLink.slice(0, sponsoredLink.indexOf('?'))
-// }
-
- $item['title'] = $post->find('._6a66d_LX75-', 0)->innertext;
-
- $item['uid'] = $post->{'data-analytics-view-value'};
-
- $descriptionPatterns = ['/<\s*dt[^>]*>\b/', '/<\/dt>/', '/<\s*dd[^>]*>\b/', '/<\/dd>/'];
- $descriptionReplacements = ['', ': ', '', ' '];
- $description = $post->find('.m7er_k4.mpof_5r.mpof_z0_s', 0)->innertext;
- $descriptionPretty = preg_replace($descriptionPatterns, $descriptionReplacements, $description);
-
- $buyNowAuction = $post->find('.mqu1_g3.mvrt_0.mgn2_12', 0)->innertext ?? '';
- $buyNowAuction = str_replace('find('._6a66d_ImOzU', 0)->innertext ?? '';
-
- $price = $post->find('._6a66d_6R3iN', 0)->plaintext;
- $price = empty($auctionTimeLeft) ? $price : $price . '- kwota licytacji';
-
- $image = $post->find('._6a66d_44ioA img', 0)->{'data-src'} ?: $post->find('._6a66d_44ioA img', 0)->src ?? false;
+ $image = $elements['photos'][0]['medium'];
if ($image) {
$item['enclosures'] = [$image . '#.image'];
}
- $offerExtraInfo = array_filter($post->find('.mqu1_g3.mgn2_12'), function ($node) {
- return empty($node->find('.mvrt_0'));
- });
+ $price = $elements['price']['mainPrice']['amount'];
+ $currency = $elements['price']['mainPrice']['currency'];
+ $sellerType = $elements['seller']['title'];
- $offerExtraInfo = $offerExtraInfo[0]->plaintext ?? '';
+ $item['categories'] = [$sellerType];
- $isSmart = $post->find('._6a66d_TC2Zk', 0)->innertext ?? '';
- if (str_contains($isSmart, 'z kurierem')) {
- $offerExtraInfo .= ', Smart z kurierem';
- } else {
- $offerExtraInfo .= ', Smart';
+ $description = '';
+ foreach ($elements['parameters'] as $parameter) {
+ $item['categories'] = array_merge($item['categories'], $parameter['values']);
+ $description .= '
+EOD;
+ }
+ }
}
}
@@ -267,6 +300,7 @@ EOD;
{
$length = 100;
+ $text = strip_tags($text);
if (strlen($text) > $length) {
$text = explode(' ', wordwrap($text, $length, ' '));
return $text[0] . '...';
@@ -275,12 +309,26 @@ EOD;
return $text;
}
- private function formatUrls($content)
+ private function formatUrls($content, $url)
{
- return preg_replace(
- '/(http[s]{0,1}\:\/\/[a-zA-Z0-9.\/\?\&=\-_]{4,})/ims',
- '$1 ',
- $content
- );
+ if (substr(strval($url), 0, 1) == '/') {
+ // fix relative URL
+ $url = 'https://www.youtube.com' . $url;
+ } elseif (substr(strval($url), 0, 33) == 'https://www.youtube.com/redirect?') {
+ // extract actual URL from YouTube redirect
+ parse_str(substr($url, 33), $params);
+ if (strpos(($params['q'] ?? ''), rtrim($content, '.')) === 0) {
+ $url = $params['q'];
+ }
+ }
+
+ // ensure all URLs are made clickable
+ $url = $url ?? $content;
+
+ if (filter_var($url, FILTER_VALIDATE_URL)) {
+ return '' . $content . '';
+ }
+
+ return $content;
}
}
diff --git a/bridges/YouTubeFeedExpanderBridge.php b/bridges/YouTubeFeedExpanderBridge.php
new file mode 100644
index 00000000..3811d960
--- /dev/null
+++ b/bridges/YouTubeFeedExpanderBridge.php
@@ -0,0 +1,81 @@
+ [
+ 'name' => 'Channel ID',
+ 'required' => true,
+ // Example: vinesauce
+ 'exampleValue' => 'UCzORJV8l3FWY4cFO8ot-F2w',
+ ],
+ 'embed' => [
+ 'name' => 'Add embed to entry',
+ 'type' => 'checkbox',
+ 'required' => false,
+ 'title' => 'Add embed to entry',
+ 'defaultValue' => 'checked',
+ ],
+ 'embedurl' => [
+ 'name' => 'Use embed page as entry url',
+ 'type' => 'checkbox',
+ 'required' => false,
+ 'title' => 'Use embed page as entry url',
+ ],
+ 'nocookie' => [
+ 'name' => 'Use nocookie embed page',
+ 'type' => 'checkbox',
+ 'required' => false,
+ 'title' => 'Use nocookie embed page'
+ ],
+ ]];
+
+ public function getIcon()
+ {
+ if ($this->getInput('channel') != null) {
+ $html = getSimpleHTMLDOMCached($this->getURI());
+ return $html->find('[itemprop="thumbnailUrl"]', 0)->href;
+ }
+ return parent::getIcon();
+ }
+
+ public function collectData()
+ {
+ $url = 'https://www.youtube.com/feeds/videos.xml?channel_id=' . $this->getInput('channel');
+ $this->collectExpandableDatas($url);
+ }
+
+ protected function parseItem(array $item)
+ {
+ $id = $item['yt']['videoId'];
+ $item['comments'] = $item['uri'] . '#comments';
+ $item['uid'] = $item['id'];
+
+ $thumbnail = sprintf('https://img.youtube.com/vi/%s/maxresdefault.jpg', $id);
+ $item['enclosures'] = [$thumbnail];
+
+ $item['content'] = $item['media']['group']['description'];
+ $item['content'] = str_replace("\n", ' ', $item['content']);
+ unset($item['media']);
+
+ $embedURI = self::URI;
+ if ($this->getInput('nocookie')) {
+ $embedURI = 'https://www.youtube-nocookie.com/';
+ }
+ $embed = $embedURI . 'embed/' . $id;
+ if ($this->getInput('embed')) {
+ $iframe_fmt = ''; //phpcs:ignore
+ $iframe = sprintf($iframe_fmt, $embed, $item['title']) . ' ';
+ $item['content'] = $iframe . $item['content'];
+ }
+ if ($this->getInput('embedurl')) {
+ $item['uri'] = $embed;
+ }
+
+ return $item;
+ }
+}
diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php
index 54a38d98..0c37be0c 100644
--- a/bridges/YoutubeBridge.php
+++ b/bridges/YoutubeBridge.php
@@ -1,17 +1,10 @@
ytGetSimpleHTMLDOM(self::URI . "watch?v=$vid", true);
+ $cacheKey = 'youtube_rate_limit';
+ if ($this->cache->get($cacheKey)) {
+ throwRateLimitException();
+ }
+ try {
+ $this->collectDataInternal();
+ } catch (HttpException $e) {
+ if ($e->getCode() === 429) {
+ $this->cache->set($cacheKey, true, 60 * 16);
+ throwRateLimitException();
+ }
+ throw $e;
+ }
+ }
+
+ private function collectDataInternal()
+ {
+ $html = '';
+ $url_feed = '';
+ $url_listing = '';
+
+ $username = $this->getInput('u');
+ $channel = $this->getInput('c');
+ $custom = $this->getInput('custom');
+ $playlist = $this->getInput('p');
+ $search = $this->getInput('s');
+
+ $durationMin = $this->getInput('duration_min');
+ $durationMax = $this->getInput('duration_max');
+
+ // Whether to discriminate videos by duration
+ $filterByDuration = $durationMin || $durationMax;
+
+ if ($username) {
+ // user and channel
+ $url_feed = self::URI . '/feeds/videos.xml?user=' . urlencode($username);
+ $url_listing = self::URI . '/user/' . urlencode($username) . '/videos';
+ } elseif ($channel) {
+ $url_feed = self::URI . '/feeds/videos.xml?channel_id=' . urlencode($channel);
+ $url_listing = self::URI . '/channel/' . urlencode($channel) . '/videos';
+ } elseif ($custom) {
+ $url_listing = self::URI . '/' . urlencode($custom) . '/videos';
+ }
+
+ if ($url_feed || $url_listing) {
+ // user, channel or custom
+ $this->feeduri = $url_listing;
+ if ($custom) {
+ // Extract the feed url for the custom name
+ $html = $this->fetch($url_listing);
+ $jsonData = $this->extractJsonFromHtml($html);
+ // Pluck out the rss feed url
+ $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl;
+ $this->feedIconUrl = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url;
+ }
+ if ($filterByDuration) {
+ if (!$custom) {
+ // Fetch the html page
+ $html = $this->fetch($url_listing);
+ $jsonData = $this->extractJsonFromHtml($html);
+ }
+ $channel_id = '';
+ if (isset($jsonData->contents)) {
+ $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId;
+ $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1];
+ $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents;
+ // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items;
+ $this->fetchItemsFromFromJsonData($jsonData);
+ } else {
+ throwServerException('Unable to get data from YouTube');
+ }
+ } else {
+ // Fetch the xml feed
+ $html = $this->fetch($url_feed);
+ $this->extractItemsFromXmlFeed($html);
+ }
+ $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext);
+ } elseif ($playlist) {
+ // playlist
+ $url_feed = self::URI . '/feeds/videos.xml?playlist_id=' . urlencode($playlist);
+ $url_listing = self::URI . '/playlist?list=' . urlencode($playlist);
+ $html = $this->fetch($url_listing);
+ $jsonData = $this->extractJsonFromHtml($html);
+ // TODO: this method returns only first 100 video items
+ // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element
+ $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0] ?? null;
+ if (!$jsonData) {
+ // playlist probably doesnt exists
+ throw new \Exception('Unable to find playlist: ' . $url_listing);
+ }
+ $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer;
+ $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents;
+ $item_count = count($jsonData);
+
+ if ($item_count > 15 || $filterByDuration) {
+ $this->fetchItemsFromFromJsonData($jsonData);
+ } else {
+ $xml = $this->fetch($url_feed);
+ $this->extractItemsFromXmlFeed($xml);
+ }
+ $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext);
+ usort($this->items, function ($item1, $item2) {
+ if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) {
+ $item1['timestamp'] = strtotime($item1['timestamp']);
+ $item2['timestamp'] = strtotime($item2['timestamp']);
+ }
+ return $item2['timestamp'] - $item1['timestamp'];
+ });
+ } elseif ($search) {
+ // search
+ $url_listing = self::URI . '/results?search_query=' . urlencode($search) . '&sp=CAI%253D';
+ $html = $this->fetch($url_listing);
+ $jsonData = $this->extractJsonFromHtml($html);
+ $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents;
+ $jsonData = $jsonData->sectionListRenderer->contents[0]->itemSectionRenderer->contents;
+ $this->fetchItemsFromFromJsonData($jsonData);
+ $this->feeduri = $url_listing;
+ $this->feedName = 'Search: ' . $search;
+ } else {
+ throwClientException("You must either specify either:\n - YouTube username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)");
+ }
+ }
+
+ private function fetchVideoDetails($videoId, &$author, &$description, &$timestamp)
+ {
+ $url = self::URI . "/watch?v=$videoId";
+ $html = $this->fetch($url, true);
// Skip unavailable videos
if (strpos($html->innertext, 'IS_UNAVAILABLE_PAGE') !== false) {
@@ -94,15 +213,18 @@ class YoutubeBridge extends BridgeAbstract
$elDatePublished = $html->find('meta[itemprop=datePublished]', 0);
if (!is_null($elDatePublished)) {
- $time = strtotime($elDatePublished->getAttribute('content'));
+ $timestamp = strtotime($elDatePublished->getAttribute('content'));
}
- $jsonData = $this->getJSONData($html);
- if (! isset($jsonData->contents)) {
+ $jsonData = $this->extractJsonFromHtml($html);
+ if (!isset($jsonData->contents)) {
return;
}
- $jsonData = $jsonData->contents->twoColumnWatchNextResults->results->results->contents;
+ $jsonData = $jsonData->contents->twoColumnWatchNextResults->results->results->contents ?? null;
+ if (!$jsonData) {
+ throw new \Exception('Unable to find json data');
+ }
$videoSecondaryInfo = null;
foreach ($jsonData as $item) {
if (isset($item->videoSecondaryInfoRenderer)) {
@@ -111,149 +233,199 @@ class YoutubeBridge extends BridgeAbstract
}
}
if (!$videoSecondaryInfo) {
- returnServerError('Could not find videoSecondaryInfoRenderer. Error at: ' . $vid);
+ throwServerException('Could not find videoSecondaryInfoRenderer. Error at: ' . $videoId);
}
- if (isset($videoSecondaryInfo->description)) {
- foreach ($videoSecondaryInfo->description->runs as $description) {
- if (isset($description->navigationEndpoint)) {
- $metadata = $description->navigationEndpoint->commandMetadata->webCommandMetadata;
- $web_type = $metadata->webPageType;
- $url = $metadata->url;
- $text = '';
- switch ($web_type) {
- case 'WEB_PAGE_TYPE_UNKNOWN':
- $url_components = parse_url($url);
- if (isset($url_components['query']) && strpos($url_components['query'], '&q=') !== false) {
- parse_str($url_components['query'], $params);
- $url = urldecode($params['q']);
- }
- $text = $url;
- break;
- case 'WEB_PAGE_TYPE_WATCH':
- case 'WEB_PAGE_TYPE_BROWSE':
- $url = 'https://www.youtube.com' . $url;
- $text = $description->text;
- break;
- }
- $desc .= "$text";
- } else {
- $desc .= nl2br($description->text);
- }
+ $description = $videoSecondaryInfo->attributedDescription->content ?? '';
+
+ // Default whitespace chars used by trim + non-breaking spaces (https://en.wikipedia.org/wiki/Non-breaking_space)
+ $whitespaceChars = " \t\n\r\0\x0B\u{A0}\u{2060}\u{202F}\u{2007}";
+ $descEnhancements = $this->ytBridgeGetVideoDescriptionEnhancements($videoSecondaryInfo, $description, self::URI, $whitespaceChars);
+ foreach ($descEnhancements as $descEnhancement) {
+ if (isset($descEnhancement['url'])) {
+ $descBefore = mb_substr($description, 0, $descEnhancement['pos']);
+ $descValue = mb_substr($description, $descEnhancement['pos'], $descEnhancement['len']);
+ $descAfter = mb_substr($description, $descEnhancement['pos'] + $descEnhancement['len'], null);
+
+ // Extended trim for the display value of internal links, e.g.:
+ // FAVICON • Video Name
+ // FAVICON / @ChannelName
+ $descValue = trim($descValue, $whitespaceChars . '•/');
+
+ $description = sprintf('%s%s%s', $descBefore, $descEnhancement['url'], $descValue, $descAfter);
}
}
}
- private function ytBridgeAddItem($vid, $title, $author, $desc, $time, $thumbnail = '')
- {
- $item = [];
- $item['id'] = $vid;
- $item['title'] = $title;
- $item['author'] = $author;
- $item['timestamp'] = $time;
- $item['uri'] = self::URI . 'watch?v=' . $vid;
- if (!$thumbnail) {
- $thumbnail = '0'; // Fallback to default thumbnail if there aren't any provided.
+ private function ytBridgeGetVideoDescriptionEnhancements(
+ object $videoSecondaryInfo,
+ string $descriptionContent,
+ string $baseUrl,
+ string $whitespaceChars
+ ): array {
+ $commandRuns = $videoSecondaryInfo->attributedDescription->commandRuns ?? [];
+ if (count($commandRuns) <= 0) {
+ return [];
}
- $thumbnailUri = str_replace('/www.', '/img.', self::URI) . 'vi/' . $vid . '/' . $thumbnail . '.jpg';
- $item['content'] = ' ' . $desc;
- $this->items[] = $item;
+
+ $enhancements = [];
+
+ $boundaryWhitespaceChars = mb_str_split($whitespaceChars);
+ $boundaryStartChars = array_merge($boundaryWhitespaceChars, [':', '-', '(']);
+ $boundaryEndChars = array_merge($boundaryWhitespaceChars, [',', '.', "'", ')']);
+ $hashtagBoundaryEndChars = array_merge($boundaryEndChars, ['#', '-']);
+
+ $descriptionContentLength = mb_strlen($descriptionContent);
+
+ $minPositionOffset = 0;
+
+ $prevStartPosition = 0;
+ $totalLength = 0;
+ $maxPositionByStartIndex = [];
+ foreach (array_reverse($commandRuns) as $commandRun) {
+ $endPosition = $commandRun->startIndex + $commandRun->length;
+ if ($endPosition < $prevStartPosition) {
+ $totalLength += 1;
+ }
+ $totalLength += $commandRun->length;
+ $maxPositionByStartIndex[$commandRun->startIndex] = $totalLength;
+ $prevStartPosition = $commandRun->startIndex;
+ }
+
+ foreach ($commandRuns as $commandRun) {
+ $commandMetadata = $commandRun->onTap->innertubeCommand->commandMetadata->webCommandMetadata ?? null;
+ if (!isset($commandMetadata)) {
+ continue;
+ }
+
+ $enhancement = null;
+
+ /*
+ $commandRun->startIndex can be offset by few positions in the positive direction
+ when some multibyte characters (e.g. emojis, but maybe also others) are used in the plain text video description.
+ (probably some difference between php and javascript in handling multibyte characters)
+ This loop should correct the position in most cases. It searches for the next word (determined by a set of boundary chars) with the expected length.
+ Several safeguards ensure that the correct word is chosen. When a link can not be matched,
+ everything will be discarded to prevent corrupting the description.
+ Hashtags require a different set of boundary chars.
+ */
+ $isHashtag = $commandMetadata->webPageType === 'WEB_PAGE_TYPE_BROWSE';
+ $prevEnhancement = end($enhancements);
+ $minPosition = $prevEnhancement === false ? 0 : $prevEnhancement['pos'] + $prevEnhancement['len'];
+ $maxPosition = $descriptionContentLength - $maxPositionByStartIndex[$commandRun->startIndex];
+ $position = min($commandRun->startIndex - $minPositionOffset, $maxPosition);
+ while ($position >= $minPosition) {
+ // The link display value can only ever include a new line at the end (which will be removed further below), never in between.
+ $newLinePosition = mb_strpos($descriptionContent, "\n", $position);
+ if ($newLinePosition !== false && $newLinePosition < $position + ($commandRun->length - 1)) {
+ $position = $newLinePosition - ($commandRun->length - 1);
+ continue;
+ }
+
+ $firstChar = mb_substr($descriptionContent, $position, 1);
+ $boundaryStart = mb_substr($descriptionContent, $position - 1, 1);
+ $boundaryEndIndex = $position + $commandRun->length;
+ $boundaryEnd = mb_substr($descriptionContent, $boundaryEndIndex, 1);
+
+ $boundaryStartIsValid = $position === 0 ||
+ in_array($boundaryStart, $boundaryStartChars) ||
+ ($isHashtag && $firstChar === '#');
+ $boundaryEndIsValid = $boundaryEndIndex === $descriptionContentLength ||
+ in_array($boundaryEnd, $isHashtag ? $hashtagBoundaryEndChars : $boundaryEndChars);
+
+ if ($boundaryStartIsValid && $boundaryEndIsValid) {
+ $minPositionOffset = $commandRun->startIndex - $position;
+ $enhancement = [
+ 'pos' => $position,
+ 'len' => $commandRun->length,
+ ];
+ break;
+ }
+
+ $position--;
+ }
+
+ if (!isset($enhancement)) {
+ $this->logger->debug(sprintf('Position %d cannot be corrected in "%s"', $commandRun->startIndex, substr($descriptionContent, 0, 50) . '...'));
+ // Skip to prevent the description from becoming corrupted
+ continue;
+ }
+
+ // $commandRun->length sometimes incorrectly includes the newline as last char
+ $lastChar = mb_substr($descriptionContent, $enhancement['pos'] + $enhancement['len'] - 1, 1);
+ if ($lastChar === "\n") {
+ $enhancement['len'] -= 1;
+ }
+
+ $commandUrl = parse_url($commandMetadata->url);
+ if ($commandUrl['path'] === '/redirect') {
+ parse_str($commandUrl['query'], $commandUrlQuery);
+ $enhancement['url'] = urldecode($commandUrlQuery['q']);
+ } elseif (isset($commandUrl['host'])) {
+ $enhancement['url'] = $commandMetadata->url;
+ } else {
+ $enhancement['url'] = $baseUrl . $commandMetadata->url;
+ }
+
+ $enhancements[] = $enhancement;
+ }
+
+ if (count($enhancements) !== count($commandRuns)) {
+ // At least one link can not be matched. Discard everything to prevent corrupting the description.
+ return [];
+ }
+
+ // Sort by position in descending order to be able to safely replace values
+ return array_reverse($enhancements);
}
- private function ytBridgeParseXmlFeed($xml)
+ private function extractItemsFromXmlFeed($xml)
{
+ $this->feedName = $this->decodeTitle($xml->find('feed > title', 0)->plaintext);
+
foreach ($xml->find('entry') as $element) {
- $title = $this->ytBridgeFixTitle($element->find('title', 0)->plaintext);
+ $videoId = str_replace('yt:video:', '', $element->find('id', 0)->plaintext);
+ if (strpos($videoId, 'googleads') !== false) {
+ continue;
+ }
+ $title = $this->decodeTitle($element->find('title', 0)->plaintext);
$author = $element->find('name', 0)->plaintext;
$desc = $element->find('media:description', 0)->innertext;
-
- // Make sure the description is easy on the eye :)
$desc = htmlspecialchars($desc);
$desc = nl2br($desc);
- $desc = preg_replace(
- self::URI_REGEX,
- '$1 ',
- $desc
- );
-
- $vid = str_replace('yt:video:', '', $element->find('id', 0)->plaintext);
+ $desc = preg_replace(self::URI_REGEX, '$1 ', $desc);
$time = strtotime($element->find('published', 0)->plaintext);
- if (strpos($vid, 'googleads') === false) {
- $this->ytBridgeAddItem($vid, $title, $author, $desc, $time);
- }
+ $this->addItem($videoId, $title, $author, $desc, $time);
}
- $this->feedName = $this->ytBridgeFixTitle($xml->find('feed > title', 0)->plaintext); // feedName will be used by getName()
}
- private function ytBridgeFixTitle($title)
+ private function fetch($url, bool $cache = false)
{
- // convert both Ӓ and " to UTF-8
- return html_entity_decode($title, ENT_QUOTES, 'UTF-8');
- }
-
- private function ytGetSimpleHTMLDOM($url, $cached = false)
- {
- $header = [
- 'Accept-Language: en-US'
- ];
- $opts = [];
- $lowercase = true;
- $forceTagsClosed = true;
- $target_charset = DEFAULT_TARGET_CHARSET;
- $stripRN = false;
- $defaultBRText = DEFAULT_BR_TEXT;
- $defaultSpanText = DEFAULT_SPAN_TEXT;
- if ($cached) {
- return getSimpleHTMLDOMCached(
- $url,
- 86400,
- $header,
- $opts,
- $lowercase,
- $forceTagsClosed,
- $target_charset,
- $stripRN,
- $defaultBRText,
- $defaultSpanText
- );
+ $header = ['Accept-Language: en-US'];
+ $ttl = 86400 * 3; // 3d
+ $stripNewlines = false;
+ if ($cache) {
+ return getSimpleHTMLDOMCached($url, $ttl, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines);
}
- return getSimpleHTMLDOM(
- $url,
- $header,
- $opts,
- $lowercase,
- $forceTagsClosed,
- $target_charset,
- $stripRN,
- $defaultBRText,
- $defaultSpanText
- );
+ return getSimpleHTMLDOM($url, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines);
}
- private function getJSONData($html)
+ private function extractJsonFromHtml($html)
{
$scriptRegex = '/var ytInitialData = (.*?);<\/script>/';
$result = preg_match($scriptRegex, $html, $matches);
if (! $result) {
- Logger::debug('Could not find ytInitialData');
+ $this->logger->debug('Could not find ytInitialData');
return null;
}
- return json_decode($matches[1]);
+ $data = json_decode($matches[1]);
+ return $data;
}
- private function parseJSONListing($jsonData)
+ private function fetchItemsFromFromJsonData($jsonData)
{
- $duration_min = $this->getInput('duration_min') ?: -1;
- $duration_min = $duration_min * 60;
-
- $duration_max = $this->getInput('duration_max') ?: INF;
- $duration_max = $duration_max * 60;
-
- if ($duration_max < $duration_min) {
- returnClientError('Max duration must be greater than min duration!');
- }
-
- // $vid_list = '';
+ $minimumDurationSeconds = ($this->getInput('duration_min') ?: -1) * 60;
+ $maximumDurationSeconds = ($this->getInput('duration_max') ?: INF) * 60;
foreach ($jsonData as $item) {
$wrapper = null;
@@ -269,24 +441,32 @@ class YoutubeBridge extends BridgeAbstract
continue;
}
- $vid = $wrapper->videoId;
- $title = $wrapper->title->runs[0]->text;
- if (isset($wrapper->ownerText)) {
- $this->channel_name = $wrapper->ownerText->runs[0]->text;
- } elseif (isset($wrapper->shortBylineText)) {
- $this->channel_name = $wrapper->shortBylineText->runs[0]->text;
+ // 01:03:30 | 15:06 | 1:24
+ $lengthText = $wrapper->lengthText->simpleText ?? null;
+ // 6,875 views
+ $viewCount = $wrapper->viewCountText->simpleText ?? null;
+ // Dc645M8Het8
+ $videoId = $wrapper->videoId;
+ // Jumbo frames - transfer more data faster!
+ $title = $wrapper->title->runs[0]->text ?? $wrapper->title->accessibility->accessibilityData->label ?? null;
+ $author = null;
+ $description = $wrapper->descriptionSnippet->runs[0]->text ?? null;
+ // 5 days ago | 1 month ago
+ $publishedTimeText = $wrapper->publishedTimeText->simpleText ?? $wrapper->videoInfo->runs[2]->text ?? null;
+ $timestamp = null;
+ if ($publishedTimeText) {
+ try {
+ $publicationDate = new \DateTimeImmutable($publishedTimeText);
+ // Hard-code hour, minute and second
+ $publicationDate = $publicationDate->setTime(0, 0, 0);
+ $timestamp = $publicationDate->getTimestamp();
+ } catch (\Exception $e) {
+ }
}
- $author = '';
- $desc = '';
- $time = '';
-
- // The duration comes in one of the formats:
- // hh:mm:ss / mm:ss / m:ss
- // 01:03:30 / 15:06 / 1:24
$durationText = 0;
- if (isset($wrapper->lengthText)) {
- $durationText = $wrapper->lengthText->simpleText;
+ if ($lengthText) {
+ $durationText = $lengthText;
} else {
foreach ($wrapper->thumbnailOverlays as $overlay) {
if (isset($overlay->thumbnailOverlayTimeStatusRenderer)) {
@@ -295,7 +475,6 @@ class YoutubeBridge extends BridgeAbstract
}
}
}
-
if (is_string($durationText)) {
if (preg_match('/([\d]{1,2})\:([\d]{1,2})\:([\d]{2})/', $durationText)) {
$durationText = preg_replace('/([\d]{1,2})\:([\d]{1,2})\:([\d]{2})/', '$1:$2:$3', $durationText);
@@ -304,131 +483,50 @@ class YoutubeBridge extends BridgeAbstract
}
sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds);
$duration = $hours * 3600 + $minutes * 60 + $seconds;
- if ($duration < $duration_min || $duration > $duration_max) {
+ if ($duration < $minimumDurationSeconds || $duration > $maximumDurationSeconds) {
continue;
}
}
-
- // $vid_list .= $vid . ',';
- $this->ytBridgeQueryVideoInfo($vid, $author, $desc, $time);
- $this->ytBridgeAddItem($vid, $title, $author, $desc, $time);
+ if (!$description || !$timestamp) {
+ $this->fetchVideoDetails($videoId, $author, $description, $timestamp);
+ }
+ $this->addItem($videoId, $title, $author, $description, $timestamp);
+ if (count($this->items) >= 99) {
+ break;
+ }
}
}
- public function collectData()
+ private function addItem($videoId, $title, $author, $description, $timestamp, $thumbnail = '')
{
- $xml = '';
- $html = '';
- $url_feed = '';
- $url_listing = '';
+ $description = nl2br($description);
- if ($this->getInput('u')) { /* User and Channel modes */
- $this->request = $this->getInput('u');
- $url_feed = self::URI . 'feeds/videos.xml?user=' . urlencode($this->request);
- $url_listing = self::URI . 'user/' . urlencode($this->request) . '/videos';
- } elseif ($this->getInput('c')) {
- $this->request = $this->getInput('c');
- $url_feed = self::URI . 'feeds/videos.xml?channel_id=' . urlencode($this->request);
- $url_listing = self::URI . 'channel/' . urlencode($this->request) . '/videos';
- } elseif ($this->getInput('custom')) {
- $this->request = $this->getInput('custom');
- $url_listing = self::URI . urlencode($this->request) . '/videos';
- }
-
- if (!empty($url_feed) || !empty($url_listing)) {
- $this->feeduri = $url_listing;
- if (!empty($this->getInput('custom'))) {
- $html = $this->ytGetSimpleHTMLDOM($url_listing);
- $jsonData = $this->getJSONData($html);
- $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl;
- $this->iconURL = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url;
- }
- if (!$this->skipFeeds()) {
- $html = $this->ytGetSimpleHTMLDOM($url_feed);
- $this->ytBridgeParseXmlFeed($html);
- } else {
- if (empty($this->getInput('custom'))) {
- $html = $this->ytGetSimpleHTMLDOM($url_listing);
- $jsonData = $this->getJSONData($html);
- }
- $channel_id = '';
- if (isset($jsonData->contents)) {
- $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId;
- $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1];
- $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents;
- // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items;
- $this->parseJSONListing($jsonData);
- } else {
- returnServerError('Unable to get data from YouTube. Username/Channel: ' . $this->request);
- }
- }
- $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext);
- } elseif ($this->getInput('p')) { /* playlist mode */
- // TODO: this mode makes a lot of excess video query requests.
- // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration"
- // This cache will be used to find out, which videos to fetch
- // to make feed of 15 items or more, if there a lot of videos published on that date.
- $this->request = $this->getInput('p');
- $url_feed = self::URI . 'feeds/videos.xml?playlist_id=' . urlencode($this->request);
- $url_listing = self::URI . 'playlist?list=' . urlencode($this->request);
- $html = $this->ytGetSimpleHTMLDOM($url_listing);
- $jsonData = $this->getJSONData($html);
- // TODO: this method returns only first 100 video items
- // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element
- $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0];
- $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer;
- $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents;
- $item_count = count($jsonData);
-
- if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) {
- $this->ytBridgeParseXmlFeed($xml);
- } else {
- $this->parseJSONListing($jsonData);
- }
- $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); // feedName will be used by getName()
- usort($this->items, function ($item1, $item2) {
- if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) {
- $item1['timestamp'] = strtotime($item1['timestamp']);
- $item2['timestamp'] = strtotime($item2['timestamp']);
- }
- return $item2['timestamp'] - $item1['timestamp'];
- });
- } elseif ($this->getInput('s')) { /* search mode */
- $this->request = $this->getInput('s');
- $url_listing = self::URI
- . 'results?search_query='
- . urlencode($this->request)
- . '&sp=CAI%253D';
-
- $html = $this->ytGetSimpleHTMLDOM($url_listing);
-
- $jsonData = $this->getJSONData($html);
- $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents;
- $jsonData = $jsonData->sectionListRenderer->contents;
- foreach ($jsonData as $data) { // Search result includes some ads, have to filter them
- if (isset($data->itemSectionRenderer->contents[0]->videoRenderer)) {
- $jsonData = $data->itemSectionRenderer->contents;
- break;
- }
- }
- $this->parseJSONListing($jsonData);
- $this->feeduri = $url_listing;
- $this->feedName = 'Search: ' . $this->request; // feedName will be used by getName()
- } else { /* no valid mode */
- returnClientError("You must either specify either:\n - YouTube
- username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)");
+ $item = [];
+ // This should probably be uid?
+ $item['id'] = $videoId;
+ $item['title'] = $title;
+ $item['author'] = $author ?? '';
+ $item['timestamp'] = $timestamp;
+ $item['uri'] = self::URI . '/watch?v=' . $videoId;
+ if (!$thumbnail) {
+ // Fallback to default thumbnail if there aren't any provided.
+ $thumbnail = '0';
}
+ $thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $videoId . '/' . $thumbnail . '.jpg';
+ $item['content'] = sprintf(' %s', $item['uri'], $thumbnailUri, $description);
+ $this->items[] = $item;
}
- private function skipFeeds()
+ private function decodeTitle($title)
{
- return ($this->getInput('duration_min') || $this->getInput('duration_max'));
+ // convert both Ӓ and " to UTF-8
+ return html_entity_decode($title, ENT_QUOTES, 'UTF-8');
}
public function getURI()
{
if (!is_null($this->getInput('p'))) {
- return static::URI . 'playlist?list=' . $this->getInput('p');
+ return static::URI . '/playlist?list=' . $this->getInput('p');
} elseif ($this->feeduri) {
return $this->feeduri;
}
@@ -438,14 +536,13 @@ class YoutubeBridge extends BridgeAbstract
public function getName()
{
- // Name depends on queriedContext:
switch ($this->queriedContext) {
case 'By username':
case 'By channel id':
case 'By custom name':
case 'By playlist Id':
case 'Search result':
- return htmlspecialchars_decode($this->feedName) . ' - YouTube'; // We already know it's a bridge, right?
+ return htmlspecialchars_decode($this->feedName) . ' - YouTube';
default:
return parent::getName();
}
@@ -453,10 +550,10 @@ class YoutubeBridge extends BridgeAbstract
public function getIcon()
{
- if (empty($this->iconURL)) {
+ if (empty($this->feedIconUrl)) {
return parent::getIcon();
} else {
- return $this->iconURL;
+ return $this->feedIconUrl;
}
}
}
diff --git a/bridges/ZDNetBridge.php b/bridges/ZDNetBridge.php
index 693f542c..e3b659a8 100644
--- a/bridges/ZDNetBridge.php
+++ b/bridges/ZDNetBridge.php
@@ -174,19 +174,17 @@ class ZDNetBridge extends FeedExpander
$this->collectExpandableDatas($url, $limit);
}
- protected function parseItem($item)
+ protected function parseItem(array $item)
{
- $item = parent::parseItem($item);
-
$article = getSimpleHTMLDOMCached($item['uri']);
if (!$article) {
- Logger::info('Unable to parse the dom from ' . $item['uri']);
+ $this->logger->info('Unable to parse the dom from ' . $item['uri']);
return $item;
}
$articleTag = $article->find('article', 0) ?? $article->find('.c-articleContent', 0);
if (!$articleTag) {
- Logger::info('Unable to parse tag in ' . $item['uri']);
+ $this->logger->info('Unable to parse tag in ' . $item['uri']);
return $item;
}
$contents = $articleTag->innertext;
diff --git a/bridges/ZeitBridge.php b/bridges/ZeitBridge.php
index a9e4bcf2..3f96d8d6 100644
--- a/bridges/ZeitBridge.php
+++ b/bridges/ZeitBridge.php
@@ -50,24 +50,20 @@ class ZeitBridge extends FeedExpander
'defaultValue' => 5
]
]];
- const LIMIT = 5;
public function collectData()
{
- $this->collectExpandableDatas(
- $this->getInput('category'),
- $this->getInput('limit') ?: static::LIMIT
- );
+ $url = $this->getInput('category');
+ $limit = $this->getInput('limit') ?: 5;
+
+ $this->collectExpandableDatas($url, $limit);
}
- protected function parseItem($item)
+ protected function parseItem(array $item)
{
- $item = parent::parseItem($item);
$item['enclosures'] = [];
$headers = [
- 'User-Agent: Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
- 'X-Forwarded-For: 66.249.66.1',
'Cookie: zonconsent=' . date('Y-m-d\TH:i:s.v\Z'),
];
@@ -91,7 +87,10 @@ class ZeitBridge extends FeedExpander
// remove known bad elements
foreach (
$article->find(
- 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge, .article-heading__container--podcast'
+ 'aside, .visually-hidden, .carousel-container, #tickaroo-liveblog, .zplus-badge,
+ .article-heading__container--podcast, .podcast-player__image, div[data-paywall],
+ .js-embed-consent, script, nav, .article-flexible-toc__subheading-link, .faq-link,
+ .zoner-article-magazinbox'
) as $bad
) {
$bad->remove();
@@ -112,16 +111,28 @@ class ZeitBridge extends FeedExpander
}
// authors
- $authors = $article->find('*[itemtype*="schema.org/Person"]');
- if (!$authors) {
- $authors = $article->find('.metadata__source');
- }
+ $authors = $article->find('*[itemtype*="schema.org/Person"]') ?? $article->find('.metadata__source');
if ($authors) {
- $item['author'] = implode(', ', $authors);
+ $item['author'] = implode(', ', array_map(function ($e) {
+ return trim($e->plaintext);
+ }, $authors));
+ }
+
+ $item['content'] = '';
+ // advertorial marker
+ $advert = $article->find('.advertorial-marker', 0);
+ if ($advert) {
+ $item['content'] .= $advert;
+ }
+
+ // summary
+ $summary = $article->find('.summary');
+ if ($summary) {
+ $item['content'] .= implode('', $summary);
}
// header image
- $headerimg = $article->find('*[data-ct-row="headerimage"]', 0) ?? $article->find('header', 0);
+ $headerimg = $article->find('*[data-ct-row="headerimage"]', 0) ?? $article->find('.article-header', 0) ?? $article->find('header', 0);
if ($headerimg) {
$item['content'] .= implode('', $headerimg->find('img[src], figcaption'));
}
@@ -131,7 +142,7 @@ class ZeitBridge extends FeedExpander
if ($pages) {
foreach ($pages as $page) {
- $elements = $page->find('p, h2, figcaption, img[src]');
+ $elements = $page->find('p, ul, ol, h2, figure.article__media img[src], figure.article__media figcaption, figure.quote');
$item['content'] .= implode('', $elements);
}
}
diff --git a/bridges/ZonebourseBridge.php b/bridges/ZonebourseBridge.php
new file mode 100644
index 00000000..e351b9dd
--- /dev/null
+++ b/bridges/ZonebourseBridge.php
@@ -0,0 +1,85 @@
+ [
+ 'name' => 'topic',
+ 'type' => 'list',
+ 'values' => [
+ 'toute-l-actualite' => [
+ 'monde' => '/actualite-bourse/',
+ 'france' => '/actualite-bourse/regions/locales/',
+ 'europe' => '/actualite-bourse/regions/europe/',
+ 'amerique-du-nord' => '/actualite-bourse/regions/amerique-du-nord/',
+ 'amerique-du-sud' => '/actualite-bourse/regions/amerique-du-sud/',
+ 'asie' => '/actualite-bourse/regions/asie/',
+ 'afrique' => '/actualite-bourse/regions/afrique/',
+ 'moyen-orient' => '/actualite-bourse/regions/moyenorient/',
+ 'emergents' => '/actualite-bourse/regions/emergents/',
+ ],
+ 'societes' => [
+ 'toute-l-actualite' => '/actualite-bourse/societes/',
+ 'reco-analystes' => '/actualite-bourse/societes/recommandations/',
+ 'rumeurs' => '/actualite-bourse/societes/rumeur/',
+ 'introductions' => '/actualite-bourse/societes/introductions/',
+ 'operations-capitalistiques' => '/actualite-bourse/societes/operations/',
+ 'nouveaux-contrats' => '/actualite-bourse/societes/nouveaux-contrats/',
+ 'profits-warnings' => '/actualite-bourse/societes/profits-warnings/',
+ 'nominations' => '/actualite-bourse/societes/nominations/',
+ 'communiques' => '/actualite-bourse/societes/communique/',
+ 'operations-sur-titre' => '/actualite-bourse/societes/operations_titre/',
+ 'publications-de-resultats' => '/actualite-bourse/societes/publications/',
+ 'nouveaux-marches' => '/actualite-bourse/societes/nouveaux-marches/',
+ 'nouveaux-produits' => '/actualite-bourse/societes/nouveaux-produits/',
+ 'strategies-societes' => '/actualite-bourse/societes/strategies-societes/',
+ 'risques-juridiques' => '/actualite-bourse/societes/risques-juridiques/',
+ 'rachats-d-actions' => '/actualite-bourse/societes/rachats-actions/',
+ 'fusions-et-acquisitions' => '/actualite-bourse/societes/fusions-acquisitions/',
+ 'call-transcripts' => '/actualite-bourse/societes/call-transcripts/',
+ 'guidance' => '/actualite-bourse/societes/guidance/',
+ ],
+ ],
+ ],
+ ],
+ ];
+
+ public function getName()
+ {
+ $topic = $this->getKey('topic');
+ return self::NAME . ($topic ? ': ' . $topic : '');
+ }
+
+ public function collectData()
+ {
+ $dom = getSimpleHTMLDOM(self::URI . $this->getInput('topic'));
+ $articles = $dom->find('table#newsScreener tbody tr');
+
+ if (!$articles) {
+ throwServerException('Failed to retrieve news content');
+ }
+
+ foreach ($articles as $article) {
+ $element = $article->find('.grid a', 0);
+
+ if (!$element || empty($element->plaintext) || empty($element->href)) {
+ continue;
+ }
+
+ $date = $article->find('span.js-date-relative.txt-muted.h-100', 0);
+ $timestamp = $date->{'data-utc-date'} ?? '';
+
+ $this->items[] = [
+ 'timestamp' => $timestamp,
+ 'title' => trim($element->plaintext),
+ 'uid' => $element->href,
+ 'uri' => self::URI . $element->href,
+ ];
+ }
+ }
+}
diff --git a/caches/ArrayCache.php b/caches/ArrayCache.php
new file mode 100644
index 00000000..ddbd77d5
--- /dev/null
+++ b/caches/ArrayCache.php
@@ -0,0 +1,55 @@
+data[$key] ?? null;
+ if (!$item) {
+ return $default;
+ }
+ $expiration = $item['expiration'];
+ if ($expiration === 0 || $expiration > time()) {
+ return $item['value'];
+ }
+ $this->delete($key);
+ return $default;
+ }
+
+ public function set(string $key, $value, ?int $ttl = null): void
+ {
+ $this->data[$key] = [
+ 'key' => $key,
+ 'value' => $value,
+ 'expiration' => $ttl === null ? 0 : time() + $ttl,
+ ];
+ }
+
+ public function delete(string $key): void
+ {
+ unset($this->data[$key]);
+ }
+
+ public function clear(): void
+ {
+ $this->data = [];
+ }
+
+ public function prune(): void
+ {
+ foreach ($this->data as $key => $item) {
+ $expiration = $item['expiration'];
+ if ($expiration === 0 || $expiration > time()) {
+ continue;
+ }
+ $this->delete($key);
+ }
+ }
+}
diff --git a/caches/FileCache.php b/caches/FileCache.php
index a4f48962..5b2f4b37 100644
--- a/caches/FileCache.php
+++ b/caches/FileCache.php
@@ -1,13 +1,17 @@
logger = $logger;
$default = [
'path' => null,
'enable_purge' => true,
@@ -20,120 +24,97 @@ class FileCache implements CacheInterface
$this->config['path'] = rtrim($this->config['path'], '/') . '/';
}
- public function getConfig()
+ public function get(string $key, $default = null)
{
- return $this->config;
+ $cacheFile = $this->createCacheFile($key);
+ if (!file_exists($cacheFile)) {
+ return $default;
+ }
+ $data = file_get_contents($cacheFile);
+ $item = unserialize($data);
+ if ($item === false) {
+ $this->logger->warning(sprintf('Failed to unserialize: %s', $cacheFile));
+ $this->delete($key);
+ return $default;
+ }
+ $expiration = $item['expiration'] ?? time();
+ if ($expiration === 0 || $expiration > time()) {
+ return $item['value'];
+ }
+ $this->delete($key);
+ return $default;
}
- public function loadData()
+ public function set($key, $value, ?int $ttl = null): void
{
- if (!file_exists($this->getCacheFile())) {
- return null;
- }
- $data = unserialize(file_get_contents($this->getCacheFile()));
- if ($data === false) {
- // Intentionally not throwing an exception
- Logger::warning(sprintf('Failed to unserialize: %s', $this->getCacheFile()));
- return null;
- }
- return $data;
- }
+ $item = [
+ 'key' => $key,
+ 'expiration' => $ttl === null ? 0 : time() + $ttl,
+ 'value' => $value,
+ ];
+ $cacheFile = $this->createCacheFile($key);
+ $bytes = file_put_contents($cacheFile, serialize($item));
+
+ // TODO: Consider tightening the permissions of the created file.
+ // It usually allow others to read, depending on umask
- public function saveData($data): void
- {
- $bytes = file_put_contents($this->getCacheFile(), serialize($data), LOCK_EX);
if ($bytes === false) {
- throw new \Exception(sprintf('Failed to write to: %s', $this->getCacheFile()));
+ // Typically means no disk space remaining
+ $this->logger->warning(sprintf('Failed to write to: %s', $cacheFile));
}
}
- public function getTime(): ?int
+ public function delete(string $key): void
{
- // https://www.php.net/manual/en/function.clearstatcache.php
- clearstatcache();
-
- $cacheFile = $this->getCacheFile();
- if (file_exists($cacheFile)) {
- $time = filemtime($cacheFile);
- if ($time !== false) {
- return $time;
- }
- return null;
- }
-
- return null;
+ unlink($this->createCacheFile($key));
}
- public function purgeCache(int $seconds): void
+ public function clear(): void
+ {
+ foreach (scandir($this->config['path']) as $filename) {
+ $cacheFile = $this->config['path'] . $filename;
+ $excluded = ['.' => true, '..' => true, '.gitkeep' => true];
+ if (isset($excluded[$filename]) || !is_file($cacheFile)) {
+ continue;
+ }
+ unlink($cacheFile);
+ }
+ }
+
+ public function prune(): void
{
if (! $this->config['enable_purge']) {
return;
}
-
- $cachePath = $this->getScope();
- if (!file_exists($cachePath)) {
- return;
- }
- $cacheIterator = new \RecursiveIteratorIterator(
- new \RecursiveDirectoryIterator($cachePath),
- \RecursiveIteratorIterator::CHILD_FIRST
- );
-
- foreach ($cacheIterator as $cacheFile) {
- $basename = $cacheFile->getBasename();
- $excluded = [
- '.' => true,
- '..' => true,
- '.gitkeep' => true,
- ];
- if (isset($excluded[$basename])) {
+ foreach (scandir($this->config['path']) as $filename) {
+ $cacheFile = $this->config['path'] . $filename;
+ $excluded = ['.' => true, '..' => true, '.gitkeep' => true];
+ if (isset($excluded[$filename]) || !is_file($cacheFile)) {
continue;
- } elseif ($cacheFile->isFile()) {
- $filepath = $cacheFile->getPathname();
- if (filemtime($filepath) < time() - $seconds) {
- // todo: sometimes this file doesn't exists
- unlink($filepath);
- }
}
- }
- }
-
- public function setScope(string $scope): void
- {
- $this->scope = $this->config['path'] . trim($scope, " \t\n\r\0\x0B\\\/") . '/';
- }
-
- public function setKey(array $key): void
- {
- $this->key = json_encode($key);
- }
-
- private function getScope()
- {
- if (is_null($this->scope)) {
- throw new \Exception('Call "setScope" first!');
- }
-
- if (!is_dir($this->scope)) {
- if (mkdir($this->scope, 0755, true) !== true) {
- throw new \Exception('mkdir: Unable to create file cache folder');
+ $data = file_get_contents($cacheFile);
+ $item = unserialize($data);
+ if ($item === false) {
+ unlink($cacheFile);
+ continue;
}
+ $expiration = $item['expiration'] ?? time();
+ if ($expiration === 0 || $expiration > time()) {
+ // Cached forever, or not expired yet
+ continue;
+ }
+ // Expired, so delete file
+ unlink($cacheFile);
}
-
- return $this->scope;
}
- private function getCacheFile()
+ private function createCacheFile(string $key): string
{
- return $this->getScope() . $this->getCacheName();
+ return $this->config['path'] . hash('md5', $key) . '.cache';
}
- private function getCacheName()
+ public function getConfig()
{
- if (is_null($this->key)) {
- throw new \Exception('Call "setKey" first!');
- }
-
- return hash('md5', $this->key) . '.cache';
+ return $this->config;
}
}
diff --git a/caches/MemcachedCache.php b/caches/MemcachedCache.php
index 8cd1932b..f994c1ae 100644
--- a/caches/MemcachedCache.php
+++ b/caches/MemcachedCache.php
@@ -1,110 +1,61 @@
logger = $logger;
+ $this->conn = new \Memcached();
+ // This call does not actually connect to server yet
+ if (!$this->conn->addServer($host, $port)) {
+ throw new \Exception('Unable to add memcached server');
}
-
- $section = 'MemcachedCache';
- $host = Configuration::getConfig($section, 'host');
- $port = Configuration::getConfig($section, 'port');
-
- if (empty($host) && empty($port)) {
- throw new \Exception('Configuration for ' . $section . ' missing.');
- }
- if (empty($host)) {
- throw new \Exception('"host" param is not set for ' . $section);
- }
- if (empty($port)) {
- throw new \Exception('"port" param is not set for ' . $section);
- }
- if (!ctype_digit($port)) {
- throw new \Exception('"port" param is invalid for ' . $section);
- }
-
- $port = intval($port);
-
- if ($port < 1 || $port > 65535) {
- throw new \Exception('"port" param is invalid for ' . $section);
- }
-
- $conn = new \Memcached();
- $conn->addServer($host, $port) or returnServerError('Could not connect to memcached server');
- $this->conn = $conn;
}
- public function loadData()
+ public function get(string $key, $default = null)
{
- if ($this->data) {
- return $this->data;
+ $value = $this->conn->get($key);
+ if ($value === false) {
+ return $default;
}
- $result = $this->conn->get($this->getCacheKey());
+ return $value;
+ }
+
+ public function set(string $key, $value, $ttl = null): void
+ {
+ $expiration = $ttl === null ? 0 : time() + $ttl;
+ $result = $this->conn->set($key, $value, $expiration);
if ($result === false) {
- return null;
+ $this->logger->warning('Failed to store an item in memcached', [
+ 'key' => $key,
+ 'code' => $this->conn->getLastErrorCode(),
+ 'message' => $this->conn->getLastErrorMessage(),
+ 'number' => $this->conn->getLastErrorErrno(),
+ ]);
+ // Intentionally not throwing an exception
}
-
- $this->time = $result['time'];
- $this->data = $result['data'];
- return $result['data'];
}
- public function saveData($data): void
+ public function delete(string $key): void
{
- $time = time();
- $object_to_save = [
- 'data' => $data,
- 'time' => $time,
- ];
- $result = $this->conn->set($this->getCacheKey(), $object_to_save, $this->expiration);
-
- if ($result === false) {
- throw new \Exception('Cannot write the cache to memcached server');
- }
-
- $this->time = $time;
+ $this->conn->delete($key);
}
- public function getTime(): ?int
+ public function clear(): void
{
- if ($this->time === null) {
- $this->loadData();
- }
- return $this->time;
+ $this->conn->flush();
}
- public function purgeCache(int $seconds): void
+ public function prune(): void
{
- // Note: does not purges cache right now
- // Just sets cache expiration and leave cache purging for memcached itself
- $this->expiration = $seconds;
- }
-
- public function setScope(string $scope): void
- {
- $this->scope = $scope;
- }
-
- public function setKey(array $key): void
- {
- $this->key = json_encode($key);
- }
-
- private function getCacheKey()
- {
- if (is_null($this->key)) {
- throw new \Exception('Call "setKey" first!');
- }
-
- return 'rss_bridge_cache_' . hash('md5', $this->scope . $this->key . 'A');
+ // memcached manages pruning on its own
}
}
diff --git a/caches/NullCache.php b/caches/NullCache.php
index 1a01df2f..9756b0aa 100644
--- a/caches/NullCache.php
+++ b/caches/NullCache.php
@@ -4,28 +4,24 @@ declare(strict_types=1);
class NullCache implements CacheInterface
{
- public function setScope(string $scope): void
+ public function get(string $key, $default = null)
+ {
+ return $default;
+ }
+
+ public function set(string $key, $value, ?int $ttl = null): void
{
}
- public function setKey(array $key): void
+ public function delete(string $key): void
{
}
- public function loadData()
+ public function clear(): void
{
}
- public function saveData($data): void
- {
- }
-
- public function getTime(): ?int
- {
- return null;
- }
-
- public function purgeCache(int $seconds): void
+ public function prune(): void
{
}
}
diff --git a/caches/SQLiteCache.php b/caches/SQLiteCache.php
index 309b86d1..a8d51c74 100644
--- a/caches/SQLiteCache.php
+++ b/caches/SQLiteCache.php
@@ -1,17 +1,22 @@
+ * The storage table has a column `updated` which is incorrectly named.
+ * It should have been named `expiration` and the code treats it as an expiration date (in unix timestamp)
*/
class SQLiteCache implements CacheInterface
{
- private \SQLite3 $db;
- private string $scope;
- private string $key;
+ private Logger $logger;
private array $config;
+ private \SQLite3 $db;
- public function __construct(array $config)
- {
+ public function __construct(
+ Logger $logger,
+ array $config
+ ) {
+ $this->logger = $logger;
$default = [
'file' => null,
'timeout' => 5000,
@@ -32,71 +37,89 @@ class SQLiteCache implements CacheInterface
$this->db = new \SQLite3($config['file']);
$this->db->enableExceptions(true);
$this->db->exec("CREATE TABLE storage ('key' BLOB PRIMARY KEY, 'value' BLOB, 'updated' INTEGER)");
+ // Consider uncommenting this to add an index on expiration
+ //$this->db->exec('CREATE INDEX idx_storage_updated ON storage (updated)');
}
$this->db->busyTimeout($config['timeout']);
+
+ // https://www.sqlite.org/pragma.html#pragma_journal_mode
+ $this->db->exec('PRAGMA journal_mode = wal');
+
+ // https://www.sqlite.org/pragma.html#pragma_synchronous
+ $this->db->exec('PRAGMA synchronous = NORMAL');
}
- public function loadData()
+ public function get(string $key, $default = null)
{
- $stmt = $this->db->prepare('SELECT value FROM storage WHERE key = :key');
- $stmt->bindValue(':key', $this->getCacheKey());
+ $cacheKey = $this->createCacheKey($key);
+ $stmt = $this->db->prepare('SELECT value, updated FROM storage WHERE key = :key');
+ $stmt->bindValue(':key', $cacheKey);
$result = $stmt->execute();
- if ($result) {
- $data = $result->fetchArray(\SQLITE3_ASSOC);
- if (isset($data['value'])) {
- return unserialize($data['value']);
- }
+ if (!$result) {
+ return $default;
}
-
- return null;
+ $row = $result->fetchArray(\SQLITE3_ASSOC);
+ if ($row === false) {
+ return $default;
+ }
+ $expiration = $row['updated'];
+ if ($expiration === 0 || $expiration > time()) {
+ $blob = $row['value'];
+ $value = unserialize($blob);
+ if ($value === false) {
+ $this->logger->error(sprintf("Failed to unserialize: '%s'", mb_substr($blob, 0, 100)));
+ // delete?
+ return $default;
+ }
+ return $value;
+ }
+ // delete?
+ return $default;
}
- public function saveData($data): void
+ public function set(string $key, $value, ?int $ttl = null): void
{
+ $cacheKey = $this->createCacheKey($key);
+ $blob = serialize($value);
+ $expiration = $ttl === null ? 0 : time() + $ttl;
$stmt = $this->db->prepare('INSERT OR REPLACE INTO storage (key, value, updated) VALUES (:key, :value, :updated)');
- $stmt->bindValue(':key', $this->getCacheKey());
- $stmt->bindValue(':value', serialize($data));
- $stmt->bindValue(':updated', time());
- $stmt->execute();
- }
-
- public function getTime(): ?int
- {
- $stmt = $this->db->prepare('SELECT updated FROM storage WHERE key = :key');
- $stmt->bindValue(':key', $this->getCacheKey());
- $result = $stmt->execute();
- if ($result) {
- $data = $result->fetchArray(\SQLITE3_ASSOC);
- if (isset($data['updated'])) {
- return $data['updated'];
- }
+ $stmt->bindValue(':key', $cacheKey);
+ $stmt->bindValue(':value', $blob, \SQLITE3_BLOB);
+ $stmt->bindValue(':updated', $expiration);
+ try {
+ $result = $stmt->execute();
+ // Should $result->finalize() be called here?
+ } catch (\Exception $e) {
+ $this->logger->warning(create_sane_exception_message($e));
+ // Intentionally not rethrowing exception
}
-
- return null;
}
- public function purgeCache(int $seconds): void
+ public function delete(string $key): void
+ {
+ $key = $this->createCacheKey($key);
+ $stmt = $this->db->prepare('DELETE FROM storage WHERE key = :key');
+ $stmt->bindValue(':key', $key);
+ $result = $stmt->execute();
+ }
+
+ public function prune(): void
{
if (!$this->config['enable_purge']) {
return;
}
- $stmt = $this->db->prepare('DELETE FROM storage WHERE updated < :expired');
- $stmt->bindValue(':expired', time() - $seconds);
- $stmt->execute();
+ $stmt = $this->db->prepare('DELETE FROM storage WHERE updated <= :now');
+ $stmt->bindValue(':now', time());
+ $result = $stmt->execute();
}
- public function setScope(string $scope): void
+ public function clear(): void
{
- $this->scope = $scope;
+ $this->db->query('DELETE FROM storage');
}
- public function setKey(array $key): void
+ private function createCacheKey($key)
{
- $this->key = json_encode($key);
- }
-
- private function getCacheKey()
- {
- return hash('sha1', $this->scope . $this->key, true);
+ return hash('sha1', $key, true);
}
}
diff --git a/composer.json b/composer.json
index a08c9666..8d41c51b 100644
--- a/composer.json
+++ b/composer.json
@@ -28,18 +28,20 @@
"ext-openssl": "*",
"ext-libxml": "*",
"ext-simplexml": "*",
+ "ext-dom": "*",
"ext-json": "*",
- "ext-intl": "*"
+ "ext-filter": "*"
},
"require-dev": {
"phpunit/phpunit": "^9",
"squizlabs/php_codesniffer": "^3.6"
},
"suggest": {
+ "php-webdriver/webdriver": "Required for Selenium usage",
"ext-memcached": "Allows to use memcached as cache type",
"ext-sqlite3": "Allows to use an SQLite database for caching",
"ext-zip": "Required for FDroidRepoBridge",
- "ext-dom": "Allows to use some bridges based on XPath expressions"
+ "ext-intl": "Required for OLXBridge"
},
"autoload-dev": {
"psr-4": {
@@ -48,7 +50,7 @@
},
"scripts": {
"test": "./vendor/bin/phpunit",
- "lint": "./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./",
+ "lint": "./vendor/bin/phpcs --parallel=2 --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./",
"compat": "./vendor/bin/phpcs --standard=phpcompatibility.xml --warning-severity=0 --extensions=php -p ./"
}
}
diff --git a/composer.lock b/composer.lock
index 39e9cfc4..94a71227 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
- "content-hash": "24083060ddb8be9a95e75f6596e3bb83",
+ "content-hash": "6103a05bc4ac2c33281ef349fd8ff968",
"packages": [],
"packages-dev": [
{
@@ -623,16 +623,16 @@
},
{
"name": "phpunit/phpunit",
- "version": "9.6.9",
+ "version": "9.6.11",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/phpunit.git",
- "reference": "a9aceaf20a682aeacf28d582654a1670d8826778"
+ "reference": "810500e92855eba8a7a5319ae913be2da6f957b0"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/a9aceaf20a682aeacf28d582654a1670d8826778",
- "reference": "a9aceaf20a682aeacf28d582654a1670d8826778",
+ "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/810500e92855eba8a7a5319ae913be2da6f957b0",
+ "reference": "810500e92855eba8a7a5319ae913be2da6f957b0",
"shasum": ""
},
"require": {
@@ -706,7 +706,7 @@
"support": {
"issues": "https://github.com/sebastianbergmann/phpunit/issues",
"security": "https://github.com/sebastianbergmann/phpunit/security/policy",
- "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.9"
+ "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.11"
},
"funding": [
{
@@ -722,7 +722,7 @@
"type": "tidelift"
}
],
- "time": "2023-06-11T06:13:56+00:00"
+ "time": "2023-08-19T07:10:56+00:00"
},
{
"name": "sebastian/cli-parser",
@@ -1690,16 +1690,16 @@
},
{
"name": "squizlabs/php_codesniffer",
- "version": "3.7.2",
+ "version": "3.8.1",
"source": {
"type": "git",
- "url": "https://github.com/squizlabs/PHP_CodeSniffer.git",
- "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879"
+ "url": "https://github.com/PHPCSStandards/PHP_CodeSniffer.git",
+ "reference": "14f5fff1e64118595db5408e946f3a22c75807f7"
},
"dist": {
"type": "zip",
- "url": "https://api.github.com/repos/squizlabs/PHP_CodeSniffer/zipball/ed8e00df0a83aa96acf703f8c2979ff33341f879",
- "reference": "ed8e00df0a83aa96acf703f8c2979ff33341f879",
+ "url": "https://api.github.com/repos/PHPCSStandards/PHP_CodeSniffer/zipball/14f5fff1e64118595db5408e946f3a22c75807f7",
+ "reference": "14f5fff1e64118595db5408e946f3a22c75807f7",
"shasum": ""
},
"require": {
@@ -1709,11 +1709,11 @@
"php": ">=5.4.0"
},
"require-dev": {
- "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0"
+ "phpunit/phpunit": "^4.0 || ^5.0 || ^6.0 || ^7.0 || ^8.0 || ^9.3.4"
},
"bin": [
- "bin/phpcs",
- "bin/phpcbf"
+ "bin/phpcbf",
+ "bin/phpcs"
],
"type": "library",
"extra": {
@@ -1728,22 +1728,45 @@
"authors": [
{
"name": "Greg Sherwood",
- "role": "lead"
+ "role": "Former lead"
+ },
+ {
+ "name": "Juliette Reinders Folmer",
+ "role": "Current lead"
+ },
+ {
+ "name": "Contributors",
+ "homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer/graphs/contributors"
}
],
"description": "PHP_CodeSniffer tokenizes PHP, JavaScript and CSS files and detects violations of a defined set of coding standards.",
- "homepage": "https://github.com/squizlabs/PHP_CodeSniffer",
+ "homepage": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
"keywords": [
"phpcs",
"standards",
"static analysis"
],
"support": {
- "issues": "https://github.com/squizlabs/PHP_CodeSniffer/issues",
- "source": "https://github.com/squizlabs/PHP_CodeSniffer",
- "wiki": "https://github.com/squizlabs/PHP_CodeSniffer/wiki"
+ "issues": "https://github.com/PHPCSStandards/PHP_CodeSniffer/issues",
+ "security": "https://github.com/PHPCSStandards/PHP_CodeSniffer/security/policy",
+ "source": "https://github.com/PHPCSStandards/PHP_CodeSniffer",
+ "wiki": "https://github.com/PHPCSStandards/PHP_CodeSniffer/wiki"
},
- "time": "2023-02-22T23:07:41+00:00"
+ "funding": [
+ {
+ "url": "https://github.com/PHPCSStandards",
+ "type": "github"
+ },
+ {
+ "url": "https://github.com/jrfnl",
+ "type": "github"
+ },
+ {
+ "url": "https://opencollective.com/php_codesniffer",
+ "type": "open_collective"
+ }
+ ],
+ "time": "2024-01-11T20:47:48+00:00"
},
{
"name": "theseer/tokenizer",
@@ -1808,9 +1831,9 @@
"ext-openssl": "*",
"ext-libxml": "*",
"ext-simplexml": "*",
- "ext-json": "*",
- "ext-intl": "*"
+ "ext-dom": "*",
+ "ext-json": "*"
},
"platform-dev": [],
- "plugin-api-version": "2.0.0"
+ "plugin-api-version": "2.6.0"
}
diff --git a/config.default.ini.php b/config.default.ini.php
index 1c9a20ae..39329a2a 100644
--- a/config.default.ini.php
+++ b/config.default.ini.php
@@ -6,55 +6,60 @@
[system]
+; System environment: "dev" or "prod"
+env = "prod"
+
; Only these bridges are available for feed production
; How to enable all bridges: enabled_bridges[] = *
-enabled_bridges[] = FeedMerge
-enabled_bridges[] = FeedReducerBridge
-enabled_bridges[] = Filter
-enabled_bridges[] = GettrBridge
-enabled_bridges[] = MastodonBridge
-enabled_bridges[] = Reddit
-enabled_bridges[] = RumbleBridge
-enabled_bridges[] = SoundcloudBridge
-enabled_bridges[] = Telegram
-enabled_bridges[] = ThePirateBay
-enabled_bridges[] = TikTokBridge
-enabled_bridges[] = Twitch
-enabled_bridges[] = Twitter
-enabled_bridges[] = Vk
-enabled_bridges[] = XPathBridge
-enabled_bridges[] = Youtube
-enabled_bridges[] = YouTubeCommunityTabBridge
+;enabled_bridges[] = CssSelectorBridge
+;enabled_bridges[] = FeedMerge
+;enabled_bridges[] = FeedReducerBridge
+;enabled_bridges[] = Filter
+;enabled_bridges[] = GettrBridge
+;enabled_bridges[] = MastodonBridge
+;enabled_bridges[] = Reddit
+;enabled_bridges[] = RumbleBridge
+;enabled_bridges[] = SoundcloudBridge
+;enabled_bridges[] = Telegram
+;enabled_bridges[] = ThePirateBay
+;enabled_bridges[] = TikTokBridge
+;enabled_bridges[] = Twitch
+;enabled_bridges[] = XPathBridge
+;enabled_bridges[] = Youtube
+;enabled_bridges[] = YouTubeCommunityTabBridge
+enabled_bridges[] = *
-; Defines the timezone used by RSS-Bridge
-; Find a list of supported timezones at
-; https://www.php.net/manual/en/timezones.php
-; timezone = "UTC" (default)
timezone = "UTC"
; Display a system message to users.
-message = ""
-
-; Whether to enable debug mode.
-enable_debug_mode = false
-
-; Enable debug mode only for these permitted ip addresses
-; debug_mode_whitelist[] = 127.0.0.1
-; debug_mode_whitelist[] = 192.168.1.10
+;message = "Hello world"
; Whether to enable maintenance mode. If enabled, feed requests receive 503 Service Unavailable
enable_maintenance_mode = false
+; Max file size for simple_html_dom in bytes (10000000 => 10 MB)
+max_file_size = 10000000
+
[http]
-timeout = 60
-useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0"
+
+; Operation timeout in seconds
+timeout = 5
+
+; Operation retry count in case of curl error
+retries = 1
+
+; Curl user agent
+; This is already set by curl-impersonate, which comes included as default
+; in RSS-Bridge docker container. Use only if you know what you're doing.
+; For reference, see https://github.com/lexiforest/curl-impersonate/tree/main/docs
+;useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0"
; Max http response size in MB
max_filesize = 20
[cache]
-; Cache type: file, sqlite, memcached, null
+; Cache type: file, sqlite, memcached, array, null
type = "file"
; Allow users to specify custom timeout for specific requests.
@@ -63,12 +68,13 @@ type = "file"
custom_timeout = false
[admin]
+
; Advertise an email address where people can reach the administrator.
; This address is displayed on the main page, visible to everyone!
; "" = Disabled (default)
email = ""
-; Advertise a contact Telegram url e.g. "https://t.me/elegantobjects"
+; Advertise a contact URL (can be any URL!) e.g. "https://t.me/elegantobjects"
telegram = ""
; Show Donation information for bridges if available.
@@ -80,7 +86,8 @@ donations = true
[proxy]
-; Sets the proxy url (i.e. "tcp://192.168.0.0:32")
+; The HTTP proxy to tunnel requests through
+; https://curl.se/libcurl/c/CURLOPT_PROXY.html
; "" = Proxy disabled (default)
url = ""
@@ -93,23 +100,25 @@ name = "Hidden proxy name"
; false = disabled (default)
by_bridge = false
+[webdriver]
+
+; Sets the url of the webdriver or selenium server
+selenium_server_url = "http://localhost:4444"
+
+; Sets whether the browser should run in headless mode (no visible ui)
+; true = enabled
+; false = disabled (default)
+headless = false
+
[authentication]
-; Enables basic authentication for all requests to this RSS-Bridge instance.
-;
-; Warning: You'll have to upgrade existing feeds after enabling this option!
-;
-; true = enabled
-; false = disabled (default)
+; HTTP basic authentication
enable = false
-
username = "admin"
-
-; The password cannot be the empty string if authentication is enabled.
password = ""
-; This will be used only for actions that require privileged access
-access_token = ""
+; Token authentication (URL)
+token = ""
[error]
@@ -126,6 +135,7 @@ report_limit = 1
; --- Cache specific configuration ---------------------------------------------
[FileCache]
+
; The root folder to store files in.
; "" = Use the cache folder in the repository (default)
path = ""
@@ -133,6 +143,7 @@ path = ""
enable_purge = true
[SQLiteCache]
+
; Filepath of the sqlite db file
file = "cache.sqlite"
; Whether to actually delete data when purging
@@ -141,11 +152,17 @@ enable_purge = true
timeout = 5000
[MemcachedCache]
+
host = "localhost"
port = 11211
; --- Bridge specific configuration ------
+[TelegramBridge]
+
+; Max pages to fetch (1 page => 20 messages), min=1 max=100
+max_pages = 1
+
[DiscogsBridge]
; Sets the personal access token for interactions with Discogs. When
diff --git a/config/nginx.conf b/config/nginx.conf
index bb7b1dcb..c65f8e00 100644
--- a/config/nginx.conf
+++ b/config/nginx.conf
@@ -2,8 +2,8 @@ server {
listen 80 default_server;
listen [::]:80 default_server;
root /app;
- access_log /dev/stdout;
- error_log /dev/stderr;
+ access_log /var/log/nginx/access.log;
+ error_log /var/log/nginx/error.log;
index index.php;
location ~ /(\.|vendor|tests) {
@@ -13,6 +13,7 @@ server {
location ~ \.php$ {
include snippets/fastcgi-php.conf;
- fastcgi_pass 127.0.0.1:9000;
+ fastcgi_read_timeout 45s;
+ fastcgi_pass unix:/var/run/php/php8.2-fpm.sock;
}
}
diff --git a/config/php-fpm.conf b/config/php-fpm.conf
new file mode 100644
index 00000000..a508a0f6
--- /dev/null
+++ b/config/php-fpm.conf
@@ -0,0 +1,18 @@
+; Inspired by https://github.com/docker-library/php/blob/master/8.2/bookworm/fpm/Dockerfile
+
+[global]
+error_log = /proc/self/fd/2
+
+; https://github.com/docker-library/php/pull/725#issuecomment-443540114
+log_limit = 8192
+
+[www]
+; php-fpm closes STDOUT on startup, so sending logs to /proc/self/fd/1 does not work.
+; https://bugs.php.net/bug.php?id=73886
+access.log = /proc/self/fd/2
+
+clear_env = no
+
+; Ensure worker stdout and stderr are sent to the main error log.
+catch_workers_output = yes
+decorate_workers_output = no
diff --git a/config/php.ini b/config/php.ini
new file mode 100644
index 00000000..383afffb
--- /dev/null
+++ b/config/php.ini
@@ -0,0 +1,4 @@
+; Inspired by https://github.com/docker-library/php/blob/master/8.2/bookworm/fpm/Dockerfile
+
+; https://github.com/docker-library/php/issues/878#issuecomment-938595965
+fastcgi.logging = Off
diff --git a/contrib/prepare_release/fetch_contributors.php b/contrib/prepare_release/fetch_contributors.php
deleted file mode 100644
index ad04458a..00000000
--- a/contrib/prepare_release/fetch_contributors.php
+++ /dev/null
@@ -1,49 +0,0 @@
- 'application/json',
- 'Content-Type' => 'application/json',
- 'User-Agent' => 'RSS-Bridge',
- ];
- $result = _http_request($url, ['headers' => $headers]);
-
- foreach (json_decode($result['body']) as $contributor) {
- $contributors[] = $contributor;
- }
-
- // Extract links to "next", "last", etc...
- $links = explode(',', $result['headers']['link'][0]);
- $next = false;
-
- // Check if there is a link with 'rel="next"'
- foreach ($links as $link) {
- [$url, $type] = explode(';', $link, 2);
-
- if (trim($type) === 'rel="next"') {
- $url = trim(preg_replace('/([<>])/', '', $url));
- $next = true;
- break;
- }
- }
-}
-
-/* Example JSON data: https://api.github.com/repos/rss-bridge/rss-bridge/contributors */
-
-// We want contributors sorted by name
-usort($contributors, function ($a, $b) {
- return strcasecmp($a->login, $b->login);
-});
-
-// Export as Markdown list
-foreach ($contributors as $contributor) {
- echo " * [{$contributor->login}]({$contributor->html_url})\n";
-}
diff --git a/contrib/prepare_release/rssbridge-log-helper.el b/contrib/prepare_release/rssbridge-log-helper.el
deleted file mode 100644
index a4b28226..00000000
--- a/contrib/prepare_release/rssbridge-log-helper.el
+++ /dev/null
@@ -1,151 +0,0 @@
-;;; rssbridge-log-helper.el --- A helper for preparing RSS-Bridge releases -*- lexical-binding:t; coding:utf-8 -*-
-
-;;; Commentary:
-
-;; Keyboard abbreviations used below:
-;; C-x == Ctrl + x
-;; M-x == Alt + x
-
-;; How to use this helper?
-;; 1. Run "git log --reverse 2021-04-25..master > tmp.md" (2021-04-25 is an example tag of a previous version)
-;; 2. Copy the contents of template.md to the start of tmp.md
-;; 3. Open Emacs. Type M-x load-file
-;; 4. Enter in the path to rssbridge-log-helper.el then
-;; 5. Type M-x find-file
-;; 6. Enter the path to tmp.md then
-;; 7. Type M-x rssbridge-log-transient-state
-;; 8. You can now use the following shortcuts to organize the commits:
-;; x: Delete commit
-;; g: Copy as general change
-;; n: Copy as new bridge
-;; m: Copy as modified bridge
-;; r: Copy as removed bridge
-;; : Quit
-;; 9. Once you are done with all the commits, type C-x then C-s
-;; 10. Exit Emacs with C-x then C-c
-
-;;; Code:
-
-(defun rssbridge-log--get-commit-block ()
- "Select a commit block that begins before the cursor."
- (re-search-backward "^commit ") ;; (move-beginning-of-line nil)
- (set-mark-command nil)
- (right-char)
- (re-search-forward "^commit ")
- (move-end-of-line 1))
-
-(defun rssbridge-log--goto-first-commit ()
- "Find the first commit in the file."
- (goto-char (point-min))
- (re-search-forward "^commit "))
-
-(defun rssbridge-log--remove-until-prev-commit-block ()
- "Remove from start of current line to previous commit block."
- (move-beginning-of-line nil)
- (set-mark-command nil)
- (re-search-backward "^commit ")
- (delete-region (region-beginning) (region-end)))
-
-(defun rssbridge-log--remove-until-next-commit-block ()
- "Remove from start of current line to next commit block."
- (move-beginning-of-line nil)
- (set-mark-command nil)
- (re-search-forward "^commit ")
- (move-beginning-of-line nil)
- (delete-region (region-beginning) (region-end)))
-
-(defun rssbridge-log--cut-paste (arg)
- "Copy current line to header that matches ARG."
- (kill-whole-line 0)
- (rssbridge-log--remove-until-next-commit-block)
- (goto-char (point-min))
- (re-search-forward arg)
- (move-end-of-line 1)
- (newline)
- (yank)
- (set-mark-command 1)
- (re-search-forward "^commit ")
- (recenter))
-
-(defun rssbridge-log-remove ()
- "Remove the current commit block.
-
-You can bind this function or use `rssbridge-log-transient-state'
-to access the function."
- (interactive)
- (rssbridge-log--get-commit-block)
- (rssbridge-log--remove-until-prev-commit-block)
- (set-mark-command 1)
- (re-search-forward "^commit "))
-
-(defun rssbridge-log-copy-as-new ()
- "Copy the current commit block as a new bridge.
-
-You can bind this function or use `rssbridge-log-transient-state'
-to access the function."
- (interactive)
- (rssbridge-log--get-commit-block)
- (re-search-backward "^.*\\[\\(.*\\)\\].*\\((.*)\\)" (region-beginning))
- (replace-match "* \\1 () \\2")
- (rssbridge-log--remove-until-prev-commit-block)
- (rssbridge-log--cut-paste "## New bridges"))
-
-(defun rssbridge-log-copy-as-mod ()
- "Copy the current commit block as a modified bridge.
-
-You can bind this function or use `rssbridge-log-transient-state'
-to access the function."
- (interactive)
- (rssbridge-log--get-commit-block)
- (re-search-backward "^.*\\[\\(.*\\)\\]" (region-beginning))
- (replace-match "* \\1:")
- (rssbridge-log--remove-until-prev-commit-block)
- (rssbridge-log--cut-paste "## Modified bridges"))
-
-(defun rssbridge-log-copy-as-gen ()
- "Copy the current commit block as a general change.
-
-You can bind this function or use `rssbridge-log-transient-state'
-to access the function."
- (interactive)
- (rssbridge-log--get-commit-block)
- (re-search-backward "^.*\\[\\(.*\\)\\]" (region-beginning))
- (replace-match "* \\1:")
- (rssbridge-log--remove-until-prev-commit-block)
- (rssbridge-log--cut-paste "## General changes"))
-
-(defun rssbridge-log-copy-as-rem ()
- "Copy the current commit block as a removed bridge.
-
-You can bind this function or use `rssbridge-log-transient-state'
-to access the function."
- (interactive)
- (rssbridge-log--get-commit-block)
- (re-search-backward "^.*\\[\\(.*\\)\\]" (region-beginning))
- (replace-match "* \\1:")
- (rssbridge-log--remove-until-prev-commit-block)
- (rssbridge-log--cut-paste "## Removed bridges"))
-
-
-(defun rssbridge-log-transient-state ()
- "Create a transient map for convienience.
-x: Delete commit
-g: Copy as general change
-n: Copy as new bridge
-m: Copy as modified bridge
-r: Copy as removed bridge
-: Quit"
- (interactive)
- (rssbridge-log--goto-first-commit)
- (set-transient-map
- (let ((map (make-sparse-keymap)))
- (define-key map "x" 'rssbridge-log-remove)
- (define-key map "g" 'rssbridge-log-copy-as-gen)
- (define-key map "n" 'rssbridge-log-copy-as-new)
- (define-key map "m" 'rssbridge-log-copy-as-mod)
- (define-key map "r" 'rssbridge-log-copy-as-rem)
- map)
- t))
-
-(provide 'rssbridge-log-helper)
-;;; rssbridge-log-helper.el ends here
diff --git a/contrib/prepare_release/template.md b/contrib/prepare_release/template.md
deleted file mode 100644
index 7314113d..00000000
--- a/contrib/prepare_release/template.md
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
-## General changes
-
-
-## New bridges
-
-
-## Modified bridges
-
-
-## Removed bridges
-
-
-
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..9f178049
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,9 @@
+version: '2'
+services:
+ rss-bridge:
+ image: rssbridge/rss-bridge:latest
+ volumes:
+ - ./config:/config
+ ports:
+ - 3000:80
+ restart: unless-stopped
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index aa95fa87..f92d8b21 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -41,5 +41,5 @@ fi
# nginx will daemonize
nginx
-# php-fpm will not
-php-fpm
+# php-fpm should not daemonize
+exec php-fpm8.2 --nodaemonize
diff --git a/docs/01_General/03_Requirements.md b/docs/01_General/03_Requirements.md
index 1ae5aa26..c9c91a52 100644
--- a/docs/01_General/03_Requirements.md
+++ b/docs/01_General/03_Requirements.md
@@ -1,24 +1,3 @@
-**RSS-Bridge** requires either of the following:
-## A Web server* with:
- PHP 7.4 (or higher)
- - [`openssl`](https://secure.php.net/manual/en/book.openssl.php) extension
- - [`libxml`](https://secure.php.net/manual/en/book.libxml.php) extension (enabled by default, see [PHP Manual](http://php.net/manual/en/libxml.installation.php))
- - [`mbstring`](https://secure.php.net/manual/en/book.mbstring.php) extension
- - [`simplexml`](https://secure.php.net/manual/en/book.simplexml.php) extension
- - [`curl`](https://secure.php.net/manual/en/book.curl.php) extension
- - [`json`](https://secure.php.net/manual/en/book.json.php) extension
- - [`filter`](https://secure.php.net/manual/en/book.filter.php) extension
- - [`zip`](https://secure.php.net/manual/en/book.zip.php) (for some bridges)
- - [`sqlite3`](http://php.net/manual/en/book.sqlite3.php) extension (only when using SQLiteCache)
-
-Enable extensions by un-commenting the corresponding line in your PHP configuration (`php.ini`).
-
-
-## A Linux server with:
-
- - Docker server configured (Any recent version should do)
- - 100MB of disk space
-
-To setup RSS Bridge using Docker, see the [Docker Guide](../03_For_Hosts/03_Docker_Installation.md) on installing RSS Bridge.
\ No newline at end of file
diff --git a/docs/01_General/05_FAQ.md b/docs/01_General/05_FAQ.md
index ade746d7..19cfae4b 100644
--- a/docs/01_General/05_FAQ.md
+++ b/docs/01_General/05_FAQ.md
@@ -1,30 +1,33 @@
-This page provides a collection of frequently asked questions and their answers. Please check this page before opening a new Issue :revolving_hearts:
-
-* [Why doesn't my bridge show new contents?](#why-doesnt-my-bridge-show-new-contents)
-* [How can I make a bridge update more frequently?](#how-can-i-make-a-bridge-update-more-frequently)
-* [Firefox doesn't show feeds anymore, what can I do?](#firefox-doesnt-show-feeds-anymore-what-can-i-do)
-
## Why doesn't my bridge show new contents?
-RSS-Bridge creates a cached version of your feed in order to reduce traffic and respond faster. The cached version is created on the first request and served for all subsequent requests. On every request RSS-Bridge checks if the cache timeout has elapsed. If the timeout has elapsed, it loads new contents and updates the cached version.
+RSS-Bridge creates a cached version of your feed in order to reduce traffic and respond faster.
+The cached version is created on the first request and served for all subsequent requests.
+On every request RSS-Bridge checks if the cache timeout has elapsed.
+If the timeout has elapsed, it loads new contents and updates the cached version.
-_Notice_: RSS-Bridge only updates feeds if you actively request it, for example by pressing F5 in your browser or using a feed reader.
+_Notice_: RSS-Bridge only updates feeds if you actively request it,
+for example by pressing F5 in your browser or using a feed reader.
-The cache duration is bridge specific and can last anywhere between five minutes and 24 hours. You can specify a custom cache timeout for each bridge if [this option](#how-can-i-make-a-bridge-update-more-frequently) has been enabled on the server.
+The cache duration is bridge specific (usually `1h`)
+You can specify a custom cache timeout for each bridge if
+[this option](#how-can-i-make-a-bridge-update-more-frequently) has been enabled on the server.
## How can I make a bridge update more frequently?
You can only do that if you are hosting the RSS-Bridge instance:
+- Lower the bridge ttl: `CACHE_TIMEOUT` constant
- Enable [`custom_timeout`](../03_For_Hosts/08_Custom_Configuration.md#customtimeout)
-- Alternatively, change the default timeout for your bridge by modifying the `CACHE_TIMEOUT` constant in the relevant bridge file (e.g [here](https://github.com/RSS-Bridge/rss-bridge/blob/master/bridges/FilterBridge.php#L7) for the Filter Bridge).
## Firefox doesn't show feeds anymore, what can I do?
-As of version 64, Firefox removed support for viewing Atom and RSS feeds in the browser. This results in the browser downloading the pages instead of showing contents.
+As of version 64, Firefox removed support for viewing Atom and RSS feeds in the browser.
+This results in the browser downloading the pages instead of showing contents.
Further reading:
- https://support.mozilla.org/en-US/kb/feed-reader-replacements-firefox
- https://bugzilla.mozilla.org/show_bug.cgi?id=1477667
-To restore the original behavior in Firefox 64 or higher you can use following Add-on which attempts to recreate the original behavior (with some sugar on top):
-- https://addons.mozilla.org/en-US/firefox/addon/rsspreview/
\ No newline at end of file
+To restore the original behavior in Firefox 64 or higher you can use following Add-on
+which attempts to recreate the original behavior (with some sugar on top):
+
+- https://addons.mozilla.org/en-US/firefox/addon/rsspreview/
diff --git a/docs/01_General/06_Public_Hosts.md b/docs/01_General/06_Public_Hosts.md
index 531e5383..71f6fdd6 100644
--- a/docs/01_General/06_Public_Hosts.md
+++ b/docs/01_General/06_Public_Hosts.md
@@ -3,9 +3,11 @@
| Country | Address | Status | Contact | Comment |
|:-------:|---------|--------|----------|---------|
|  | https://rss-bridge.org/bridge01 |  | [@dvikan](https://github.com/dvikan) | London, Digital Ocean|
+|  | https://rssbridge.flossboxin.org.in |  | [@vdbhb59](https://github.com/vdbhb59) | Hosted with Netcup Germany (Maintained in India) |
+|  | https://rss-bridge.cheredeprince.net |  | [@La_Bécasse](https://cheredeprince.net/contact) | Self-Hosted at home in France |
+|  | https://rss-bridge.sans-nuage.fr |  | [@Alsace Réseau Neutre](https://arn-fai.net/contact) | Hosted in Alsace, France |
|  | https://rss-bridge.lewd.tech |  | [@Erisa](https://github.com/Erisa) | Hosted in London, protected by Cloudflare Rate Limiting |
-|  | https://bridge.easter.fr |  | [@chatainsim](https://github.com/chatainsim) | Hosted in Isère, France |
-|  | https://wtf.roflcopter.fr/rss-bridge/ |  | [roflcopter.fr](https://wtf.roflcopter.fr/) | Hosted in France |
+|  | https://wtf.roflcopter.fr/rss-bridge |  | [roflcopter.fr](https://wtf.roflcopter.fr/) | Hosted in France |
|  | https://rss.nixnet.services |  | [@amolith](https://nixnet.services/contact) | Hosted in Wunstorf, Germany |
|  | https://rss-bridge.ggc-project.de |  | [@ggc-project.de](https://social.dev-wiki.de/@ggc_project) | Hosted in Steyr, Austria |
|  | https://rssbridge.bus-hit.me | | [@austinhuang0131](https://austinhuang.me/) | Hosted with Oracle in Québec, Canada |
@@ -13,13 +15,17 @@
|  | https://rssbridge.boldair.dev |  | [@Boldairdev](https://github.com/Boldairdev) | Latest Github release, Hosted on PHP 8.0 in Roubaix, France |
|  | https://rss-bridge.bb8.fun |  | [@captn3m0](https://github.com/captn3m0) | Hosted in Bengaluru, India |
|  | https://ololbu.ru/rss-bridge |  | [@Ololbu](https://github.com/Ololbu) | Hosted in Moscow, Russia |
-|  | https://tools.bheil.net/rss-bridge/ |  | [@bheil](https://www.bheil.net) | Hosted in Germany |
+|  | https://tools.bheil.net/rss-bridge |  | [@bheil](https://www.bheil.net) | Hosted in Germany |
|  | https://bridge.suumitsu.eu |  | [@mitsukarenai](https://github.com/mitsukarenai) | Hosted in Paris, France |
|  | https://feed.eugenemolotov.ru |  | [@em92](https://github.com/em92) | Hosted in Amsterdam, Netherlands |
|  | https://rss-bridge.mediani.de |  | [@sokai](https://github.com/sokai) | Hosted with Netcup, Germany |
-|  | http://rb.vern.cc/ |  | [@vern.cc](https://vern.cc/en/admin) | Hosted with Hetzner, US |
-|  | https://rssbridge.flossboxin.org.in/ |  | [@vdbhb59](https://github.com/vdbhb59) | Hosted with OVH SAS (Maintained in India)
-|  | https://rss.foxhaven.cyou|  | [@Aysilu](https://foxhaven.cyou) | Hosted with Timeweb (Maintained in Poland)
+|  | https://rb.ash.fail |  | [@ash](https://ash.fail/contact.html) | Hosted with Hostaris, Germany
+|  | https://rss.noleron.com |  | [@ihor](https://noleron.com/about) | Hosted with Hosting Ukraine, Ukraine
+|  | https://rssbridge.projectsegfau.lt |  | [@gi-yt](https://aryak.me) | Self-Hosted at Mumbai, India with Airtel (ISP) |
+|  | https://rb.vern.cc |  | [@vern.cc](https://vern.cc/en/admin) | Hosted with Hetzner, US |
+|  | https://rss.bloat.cat |  | [@vlnst](https://bloat.cat/contact) | Hosted with Datalix, Germany |
+|  | https://rssbridge.prenghy.org |  | [@pprenghy](https://github.com/pprenghy) | Hosted with vpsFree, The Czech Republic |
+
## Inactive instances
diff --git a/docs/02_CLI/index.md b/docs/02_CLI/index.md
index 9292746a..727e59cb 100644
--- a/docs/02_CLI/index.md
+++ b/docs/02_CLI/index.md
@@ -1,10 +1,12 @@
-RSS-Bridge supports calls via CLI. You can use the same parameters as you would normally use via the URI. Example:
+RSS-Bridge supports calls via CLI.
+You can use the same parameters as you would normally use via the URI. Example:
`php index.php action=display bridge=DansTonChat format=Json`
## Required parameters
-RSS-Bridge requires a few parameters that must be specified on every call. Omitting these parameters will result in error messages:
+RSS-Bridge requires a few parameters that must be specified on every call.
+Omitting these parameters will result in error messages:
### action
@@ -17,20 +19,26 @@ Value | Description
### bridge
-This parameter specifies the name of the bridge RSS-Bridge should return feeds from. The name of the bridge equals the class name of the bridges in the ./bridges/ folder without the 'Bridge' prefix. For example: DansTonChatBridge => DansTonChat.
+This parameter specifies the name of the bridge RSS-Bridge should return feeds from.
+The name of the bridge equals the class name of the bridges in the ./bridges/ folder without the 'Bridge' prefix.
+For example: DansTonChatBridge => DansTonChat.
### format
-This parameter specifies the format in which RSS-Bridge returns the contents. RSS-Bridge currently supports five formats: `Atom`, `Html`, `Json`, `Mrss`and `Plaintext`.
+This parameter specifies the format in which RSS-Bridge returns the contents.
## Optional parameters
-RSS-Bridge supports optional parameters. These parameters are only valid if the options have been enabled in the index.php script.
+RSS-Bridge supports optional parameters.
+These parameters are only valid if the options have been enabled in the index.php script.
### \_noproxy
-This parameter is only available if a proxy server has been specified via `proxy.url` and `proxy.by_bridge` has been enabled. This is a Boolean parameter that can be set to `true` or `false`.
+This parameter is only available if a proxy server has been specified via `proxy.url` and `proxy.by_bridge`
+has been enabled. This is a Boolean parameter that can be set to `true` or `false`.
## Bridge parameters
-Each bridge can specify its own set of parameters. As in the example above, some bridges don't specify any parameters or only optional parameters that can be neglected. For more details read the `PARAMETERS` definition for your bridge.
\ No newline at end of file
+Each bridge can specify its own set of parameters.
+As in the example above, some bridges don't specify any parameters or only optional parameters that can be neglected.
+For more details read the `PARAMETERS` definition for your bridge.
diff --git a/docs/03_For_Hosts/01_Installation.md b/docs/03_For_Hosts/01_Installation.md
index 39df7918..3312230d 100644
--- a/docs/03_For_Hosts/01_Installation.md
+++ b/docs/03_For_Hosts/01_Installation.md
@@ -1,12 +1 @@
-In order to install RSS-Bridge on your own web server* do as follows:
-
-* Make sure your web server meets all [requirements](../01_General/03_Requirements.md)
-* Download the ZIP file of the [last stable release](https://github.com/RSS-Bridge/rss-bridge/releases)
-* Place all files on your web server
-
-For linux hosts:
-* Grant read-write-access for `www-data` to the `./cache` directory (`chown -R www-data ./cache`)
-
-You have successfully installed RSS-Bridge.
-
-Instructions for Docker setups are at [Docker Installation](../03_For_Hosts/03_Docker_Installation.md)
\ No newline at end of file
+https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md
diff --git a/docs/03_For_Hosts/02_Updating.md b/docs/03_For_Hosts/02_Updating.md
index 3ec98049..e69de29b 100644
--- a/docs/03_For_Hosts/02_Updating.md
+++ b/docs/03_For_Hosts/02_Updating.md
@@ -1,42 +0,0 @@
-Updating an existing installation is very simple, depending on your type of installation.
-
-## Release Build
-
-* Download latest version
-* Extract all files
-* Replace existing files
-
-This will update all core files to the latest version. Your custom configuration and bridges are left untouched. Keep in mind that changes to any core file of RSS-Bridge will be replaced.
-
-## Docker
-
-Simply get the latest Docker build via `:latest` or specific builds via `:`.
-
-## Heroku
-
-### If you didn't fork the repo before
-
-Fork the repo by clicking the `Fork` button at the top right of this page (must be on desktop site). Then on your Heroku account, go to the application. Click on the `Deploy` tab and connect the repo named `yourusername/rss-bridge`. Do a manual deploy of the `master` branch.
-
-### If you forked the repo before
-
-[Click here to create a new pull request to your fork](https://github.com/RSS-Bridge/rss-bridge/pull/new/master). Select `compare across forks`, make the base repository `yourusername/rss-bridge` and ensure the branch is set to master. Put any title you want and create the pull request. On the page that comes after this, merge the pull request.
-
-You then want to go to your application in Heroku, connect your fork via the `Deploy` tab and deploy the `master` branch.
-
-You can turn on auto-deploy for the master branch if you don't want to go through the process of logging into Heroku and deploying the branch every time changes to the repo are made in the future.
-
-## Git
-
-To get the latest changes from the master branch
-
-```
-git pull
-```
-
-To use a specific tag
-
-```
-git fetch --all
-git checkout tags/
-```
\ No newline at end of file
diff --git a/docs/03_For_Hosts/03_Docker_Installation.md b/docs/03_For_Hosts/03_Docker_Installation.md
deleted file mode 100644
index d895e748..00000000
--- a/docs/03_For_Hosts/03_Docker_Installation.md
+++ /dev/null
@@ -1,49 +0,0 @@
-This guide is for people who want to run RSS Bridge using Docker. If you want to run it a simple PHP Webhost environment, see [Installation](../03_For_Hosts/01_Installation.md) instead.
-
-## Setup
-
-### Create the container
-
-```bash
-docker create \
---name=rss-bridge \
---volume :/config \
---publish 3000:80 \
-rssbridge/rss-bridge:latest
-```
-### Run it
-```bash
-docker start rss-bridge
-```
-
-Access it using `http://IP_Address:3000`. If you'd like to run a specific version, you can run it by changing the ':latest' on the image to a tag listed [here](https://hub.docker.com/r/rssbridge/rss-bridge/tags/)
-
-The server runs on port 80 internally, map any port of your choice (in this example 3000).
-
-You can run it using a `docker-compose.yml` as well:
-
-```yml
-version: '2'
-services:
- rss-bridge:
- image: rssbridge/rss-bridge:latest
- volumes:
- - :/config
- ports:
- - 3000:80
- restart: unless-stopped
-```
-
-# Container access and information
-
-|Function|Command|
-|----|----|
-|Shell access (live container)|`docker exec -it rss-bridge /bin/sh`|
-|Realtime container logs|`docker logs -f rss-bridge`|
-
-# Adding custom bridges and configurations
-If you want to add a bridge that is not part of [`/bridges`](https://github.com/RSS-Bridge/rss-bridge/tree/master/bridges), you can map a folder to the `/config` folder of the `rss-bridge` container.
-
-1. Create a folder in the location of your docker-compose.yml or your general docker working area (in this example it will be `/home/docker/rssbridge/config` ).
-2. Copy your [custom bridges](../05_Bridge_API/01_How_to_create_a_new_bridge.md) to the `/home/docker/rssbridge/config` folder. Applies also to [config.ini.php](../03_For_Hosts/08_Custom_Configuration.md).
-3. Map the folder to `/config` inside the container. To do that, replace the `` from the previous examples with `/home/docker/rssbridge/config`
\ No newline at end of file
diff --git a/docs/03_For_Hosts/05_Whitelisting.md b/docs/03_For_Hosts/05_Whitelisting.md
index 113c4e3d..156174f0 100644
--- a/docs/03_For_Hosts/05_Whitelisting.md
+++ b/docs/03_For_Hosts/05_Whitelisting.md
@@ -1,14 +1,18 @@
-Modify `config.ini.php` to limit available bridges.
+Modify `config.ini.php` to limit available bridges. Those changes should be applied in the `[system]` section.
## Enable all bridges
```
+[system]
+
enabled_bridges[] = *
```
## Enable some bridges
```
+[system]
+
enabled_bridges[] = TwitchBridge
enabled_bridges[] = GettrBridge
```
diff --git a/docs/03_For_Hosts/06_Authentication.md b/docs/03_For_Hosts/06_Authentication.md
index bb9c6656..f505f5a6 100644
--- a/docs/03_For_Hosts/06_Authentication.md
+++ b/docs/03_For_Hosts/06_Authentication.md
@@ -1,101 +1,6 @@
-Depending on your servers abilities you can choose between two types of authentication:
-* [.htaccess](#htaccess)
-* [RSS-Bridge Authentication](#rss-bridge-authentication)
-
-**General advice**:
-
-- Make sure to use a strong password, no matter which solution you choose!
-- Enable HTTPS on your server to ensure your connection is encrypted and secure!
-
-## .htaccess
-
-.htaccess files are commonly used to restrict access to files on a web server. One of the features of .htaccess files is the ability to password protect specific (or all) directories. If setup correctly, a password is required to access the files.
-
-The usage of .htaccess files requires three basic steps:
-
-1) [Enable .htaccess](#enable-htaccess)
-2) [Create a .htpasswd file](#create-a-htpasswd-file)
-3) [Create a .htaccess file](#create-a-htaccess-file)
-
-### Enable .htaccess
-
-This process depends on the server you are using. Some providers may require you to change some settings, or place/change some file. Here are some helpful links for your server (please add your own if missing :sparkling_heart:)
-
-- Apache: http://ask.xmodulo.com/enable-htaccess-apache.html
-
-### Create a .htpasswd file
-
-The `.htpasswd` file contains the user name and password used for login to your web server. Please notice that the password is stored in encrypted form, which requires you to encrypt your password before creating the `.htpasswd` file!
-
-Here are three ways of creating your own `.htpasswd` file:
-
-**1) Example file**
-
-Example `.htpasswd` file (user name: "test", password: "test"):
-
-```.htpasswd
-test:$apr1$a52u9ILP$XTNG8qMJiEXSm1zD0lQcR0
-```
-
-Just copy and paste the contents to your `.htpasswd` file.
-
-**2) Online generator (read warning!)**
-
-You can create your own `.htpasswd` file online using a `.htpasswd` generator like this: https://www.htaccesstools.com/htpasswd-generator/
-
-**WARNING!**
-- Never insert real passwords to an online generator!
-
-**3) Generate your own password**
-
-Another way to create your own `.htpasswd` file is to run this script on your server (it'll output the data for you, you just have to paste it int a `.htpasswd` file):
-
-```PHP
-
-```
-
->source: https://www.htaccesstools.com/articles/create-password-for-htpasswd-file-using-php/
-
-### Create a .htaccess file
-
-The `.htaccess` file is used to specify which directories are password protected. For that purpose you should place the file in whatever directory you want to restrict access. If you want to restrict access to RSS-Bridge in general, you should place the file in the root directory (where `index.php` is located).
-
-Two parameters must be specified in the `.htaccess` file:
-
-* AuthName
-* AuthUserFile
-
-`AuthName` specifies the name of the authentication (i.e. "RSS-Bridge"). `AuthUserFile` defines the **absolute** path to a `.htpasswd` file.
-
-Here are two ways of creating your own `.htaccess` file:
-
-**1) Example file**
-
-```.htaccess
-AuthType Basic
-AuthName "My Protected Area"
-AuthUserFile /path/to/.htpasswd
-Require valid-user
-```
-
-Notice: You must change the `AuthUserFile` location to fit your own server (i.e. `/var/www/html/rss-bridge/.htpasswd`)
-
-**2) Online generator**
-
-You can use an online generator to create the file for you and copy-paste it to your `.htaccess` file: https://www.htaccesstools.com/htaccess-authentication/
-
-## RSS-Bridge Authentication
-
-RSS-Bridge ships with an authentication module designed for single user environments. You can enable authentication and specify the username & password in the [configuration file](../03_For_Hosts/08_Custom_Configuration.md#authentication).
-
-Please notice that the password is stored in plain text and thus is readable to anyone who can access the file. Make sure to restrict access to the file, so that it cannot be read remotely!
\ No newline at end of file
+* http basic auth
+* token
+* Access control via webserver (see nginx/caddy/apache docs)
+
+https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md
diff --git a/docs/03_For_Hosts/07_Customizations.md b/docs/03_For_Hosts/07_Customizations.md
index be4c7f85..380f5f3a 100644
--- a/docs/03_For_Hosts/07_Customizations.md
+++ b/docs/03_For_Hosts/07_Customizations.md
@@ -1,9 +1,14 @@
-RSS-Bridge ships a few options the host may or may not activate. All options are listed in the [config.default.ini.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/config.default.ini.php) file, see [Custom Configuration](08_Custom_Configuration.md) section for more information.
+RSS-Bridge ships a few options the host may or may not activate.
+All options are listed in the [config.default.ini.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/config.default.ini.php) file,
+see [Custom Configuration](08_Custom_Configuration.md) section for more information.
## Customizable cache timeout
-Sometimes it is necessary to specify custom timeouts to update contents more frequently than the bridge maintainer intended. In these cases the client may specify a custom cache timeout to prevent loading contents from cache earlier (or later).
+Sometimes it is necessary to specify custom timeouts to update contents more frequently
+than the bridge maintainer intended.
+In these cases the client may specify a custom cache timeout to prevent loading contents
+from cache earlier (or later).
-This option can be activated by setting the [`cache.custom_timeout`](08_Custom_Configuration.md#custom_timeout) option to `true`. When enabled each bridge receives an additional parameter `Cache timeout in seconds` that can be set to any value between 1 and 86400 (24 hours). If the value is not within the limits the default settings apply (as specified by the bridge maintainer).
-
-The cache timeout is send to RSS-Bridge using the `_cache_timeout` parameter. RSS-Bridge will return an error message if the parameter is received and the option is disabled.
+This option can be activated by setting the [`cache.custom_timeout`](08_Custom_Configuration.md#custom_timeout) option to `true`.
+When enabled each bridge receives an additional parameter `Cache timeout in seconds`
+that can be set to any value.
diff --git a/docs/03_For_Hosts/08_Custom_Configuration.md b/docs/03_For_Hosts/08_Custom_Configuration.md
index 9a1f78f2..ebb53cb3 100644
--- a/docs/03_For_Hosts/08_Custom_Configuration.md
+++ b/docs/03_For_Hosts/08_Custom_Configuration.md
@@ -1,16 +1,21 @@
RSS-Bridge supports custom configurations for common parameters on the server side!
-A default configuration file (`config.default.ini.php`) is shipped with RSS-Bridge. Please do not edit this file, as it gets replaced when upgrading RSS-Bridge!
+A default configuration file (`config.default.ini.php`) is shipped with RSS-Bridge.
+Please do not edit this file, as it gets replaced when upgrading RSS-Bridge!
-You should, however, use this file as template to create your own configuration (or leave it as is, to keep the default settings). In order to create your own configuration perform following actions:
+You should, however, use this file as template to create your own configuration
+(or leave it as is, to keep the default settings).
+In order to create your own configuration perform following actions:
* Create the file `config.ini.php` in the RSS-Bridge root folder (next to `config.default.ini.php`)
* Copy the contents from `config.default.ini.php` to your configuration file
* Change the parameters to satisfy your requirements
-RSS-Bridge will automatically detect the `config.ini.php` and use it. If the file doesn't exist it will default to `config.default.ini.php` automatically.
+RSS-Bridge will automatically detect the `config.ini.php` and use it.
+If the file doesn't exist it will default to `config.default.ini.php` automatically.
-__Notice__: If a parameter is not specified in your `config.ini.php` RSS-Bridge will automatically use the default settings from `config.default.ini.php`.
+__Notice__: If a parameter is not specified in your `config.ini.php` RSS-Bridge will
+automatically use the default settings from `config.default.ini.php`.
# Available parameters
@@ -114,7 +119,9 @@ Default network timeout.
### useragent
-Default user agent.
+Overrides the user agent value. Note that the default value, together with a set of other detection-preventing options is set
+automatically by the [libcurl-impersonate](https://github.com/lexiforest/curl-impersonate), which is used by the default Docker container distributed together with RSS-Bridge. Use only if you know what you're doing, otherwise you may stop libcurl-impersonate
+from doing its job impersonating real browser.
## Authentication
diff --git a/docs/03_For_Hosts/index.md b/docs/03_For_Hosts/index.md
index 1529cb37..5ecbd4b7 100644
--- a/docs/03_For_Hosts/index.md
+++ b/docs/03_For_Hosts/index.md
@@ -1,11 +1,12 @@
This section is directed at **hosts** and **server administrators**.
-To install RSS-Bridge, please follow the [installation instructions](../03_For_Hosts/01_Installation.md). You must have access to a web server with a working PHP environment!
+RSS-Bridge comes with a large amount of bridges.
-RSS-Bridge comes with a large amount of bridges. Only few bridges are enabled by default. Unlock more bridges by adding them to the [whitelist](../03_For_Hosts/05_Whitelisting.md).
+Some bridges could be implemented more efficiently by actually using proprietary APIs,
+but there are reasons against it:
-Some bridges could be implemented more efficiently by actually using proprietary APIs, but there are reasons against it:
+- RSS-Bridge exists in the first place to NOT use APIs.
+ See [the rant](https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md#Rant).
-- RSS-Bridge exists in the first place to NOT use APIs. See [the rant](https://github.com/RSS-Bridge/rss-bridge/blob/master/README.md#Rant)
-
-- APIs require private keys that could be stored on servers running RSS-Bridge, which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution.
\ No newline at end of file
+- APIs require private keys that could be stored on servers running RSS-Bridge,
+ which is a security concern, involves complex authorizations for inexperienced users and could cause harm (when using paid services for example). In a closed environment (a server only you use for yourself) however you might be interested in using them anyway. So, check [this](https://github.com/RSS-Bridge/rss-bridge/pull/478/files) possible implementation of an anti-captcha solution.
diff --git a/docs/04_For_Developers/01_Coding_style_policy.md b/docs/04_For_Developers/01_Coding_style_policy.md
index 4e796c32..9fb0eb10 100644
--- a/docs/04_For_Developers/01_Coding_style_policy.md
+++ b/docs/04_For_Developers/01_Coding_style_policy.md
@@ -1,8 +1,13 @@
-This section explains the coding style policy for RSS-Bridge with examples and references to external resources. Please make sure your code is compliant before opening a pull request.
+This section explains the coding style policy for RSS-Bridge with examples and references to external resources.
+Please make sure your code is compliant before opening a pull request.
-You will automatically be notified if issues were found in your pull request. You must fix those issues before the pull request will be merged. Refer to [phpcs.xml](https://github.com/RSS-Bridge/rss-bridge/blob/master/phpcs.xml) for a complete list of policies enforced by Travis-CI.
+You will automatically be notified if issues were found in your pull request.
+You must fix those issues before the pull request will be merged.
+Refer to [phpcs.xml](https://github.com/RSS-Bridge/rss-bridge/blob/master/phpcs.xml) for a complete list of policies enforced by Travis-CI.
-If you want to run the checks locally, make sure you have [`phpcs`](https://github.com/squizlabs/PHP_CodeSniffer) and [`phpunit`](https://phpunit.de/) installed on your machine and run following commands in the root directory of RSS-Bridge (tested on Debian):
+If you want to run the checks locally, make sure you have
+[`phpcs`](https://github.com/squizlabs/PHP_CodeSniffer) and
+[`phpunit`](https://phpunit.de/) installed on your machine and run following commands in the root directory of RSS-Bridge (tested on Debian):
```console
./vendor/bin/phpcs --standard=phpcs.xml --warning-severity=0 --extensions=php -p ./
@@ -73,7 +78,7 @@ _Reference_: [`Squiz.WhiteSpace.SuperfluousWhitespace`](https://github.com/squiz
# Maximum Line Length
-There is no maximum line length.
+180
# Strings
diff --git a/docs/04_For_Developers/04_Actions.md b/docs/04_For_Developers/04_Actions.md
index c346731a..74c8cbfb 100644
--- a/docs/04_For_Developers/04_Actions.md
+++ b/docs/04_For_Developers/04_Actions.md
@@ -1,8 +1,9 @@
-RSS-Bridge currently supports three 'actions' which it can operate:
+RSS-Bridge currently supports four 'actions' which it can operate:
1) [Display](#display) (`?action=display`)
2) [Detect](#detect) (`?action=detect`)
3) [List](#list) (`?action=list`)
+3) [FindFeed](#findfeed) (`?action=findfeed`)
## Display
@@ -19,7 +20,7 @@ The `detect` action attempts to redirect the user to an appropriate `display` ac
If an appropriate bridge is found, a `301 Moved Permanently` HTTP status code is returned with a relative location for a `display` action. If no appropriate bridge is found or a required parameter is missing, a `400 Bad Request` status code is returned.
-The parameters for this action are listed bellow:
+The parameters for this action are listed below:
Parameter | Required | Description
----------|----------|------------
@@ -75,4 +76,91 @@ Parameter | Optional | Description
### `total`
-This parameter represents the total number of bridges available to the current instance of RSS-Bridge.
\ No newline at end of file
+This parameter represents the total number of bridges available to the current instance of RSS-Bridge.
+
+## FindFeed
+
+The `findfeed` action attempts to list all available feeds based on a supplied URL for the active bridges of this instance. As bridges have to individually implement `detectParameters`, this it may not work for every bridge.
+
+If one or more bridges return a feed, a JSON data array structure is returned. If no feeds were found, a `404 Not Found` status code is returned. If a required parameter is missing, a `400 Bad Request` status code is returned.
+
+For each feed, the whole feed URL is sent in the `url` member, the feed specific bridge parameters metadata in the `bridgeData` member and the Bridge metadata in the `bridgeMeta` member.
+
+This example shows JSON data for the NASA Instagram account URL (`https://www.instagram.com/nasa/`) using the `Html` format :
+
+```JSON
+[
+ {
+ "url": "https://rssbridge.host/?action=display&context=Username&u=nasa&bridge=InstagramBridge&format=Html",
+ "bridgeParams": {
+ "context": "Username",
+ "u": "nasa",
+ "bridge": "InstagramBridge",
+ "format": "Html"
+ },
+ "bridgeData": {
+ "context": {
+ "name": "Context",
+ "value": "Username"
+ },
+ "u": {
+ "name": "username",
+ "value": "nasa"
+ }
+ },
+ "bridgeMeta": {
+ "name": "Instagram Bridge",
+ "description": "Returns the newest images",
+ "parameters": {
+ "Username": {
+ "u": {
+ "name": "username",
+ "exampleValue": "aesoprockwins",
+ "required": true
+ }
+ },
+ "Hashtag": {
+ "h": {
+ "name": "hashtag",
+ "exampleValue": "beautifulday",
+ "required": true
+ }
+ },
+ "Location": {
+ "l": {
+ "name": "location",
+ "exampleValue": "london",
+ "required": true
+ }
+ },
+ "global": {
+ "media_type": {
+ "name": "Media type",
+ "type": "list",
+ "required": false,
+ "values": {
+ "All": "all",
+ "Video": "video",
+ "Picture": "picture",
+ "Multiple": "multiple"
+ },
+ "defaultValue": "all"
+ },
+ "direct_links": {
+ "name": "Use direct media links",
+ "type": "checkbox"
+ }
+ }
+ },
+ "icon": "https://www.instagram.com//favicon.ico"
+ }
+ }
+]
+```
+
+The parameters for this action are listed below:
+
+Parameter | Required | Description
+----------|----------|------------
+`url` | yes | Specifies the URL to attempt to find a feed from. The value of this should be URL encoded.
+`format` | yes | Specifies the name of the format to use for the URL of the feeds. This is passed to the detected `display` action. Possible values are determined from the formats available to the current instance of RSS-Bridge.
diff --git a/docs/04_For_Developers/05_Debug_mode.md b/docs/04_For_Developers/05_Debug_mode.md
index 6bdb1d48..cccf2768 100644
--- a/docs/04_For_Developers/05_Debug_mode.md
+++ b/docs/04_For_Developers/05_Debug_mode.md
@@ -1,33 +1,28 @@
-
Warning!
+Debug mode has been removed.
-Enabling debug mode on a public server may result in malicious clients retrieving sensitive data about your server and possibly gaining access to it. Do not enable debug mode on a public server, unless you understand the implications of your doing!
+If you want to disable caching you can set cache type to array (in-memory cache):
-***
+```ini
+[cache]
-Debug mode enables error reporting and prevents loading data from the cache (data is still written to the cache).
-To enable debug mode, set in `config.ini.php`:
+; Cache type: file, sqlite, memcached, array, null
+type = "array"
+```
- enable_debug_mode = true
+Alternatively, you can comment out the cache middleware in `lib/RssBridge.php`:
-Allow only explicit ip addresses:
+```diff
+diff --git a/lib/RssBridge.php b/lib/RssBridge.php
+index d16f1d89..da3df8be 100644
+--- a/lib/RssBridge.php
++++ b/lib/RssBridge.php
+@@ -24,7 +24,7 @@ final class RssBridge
- debug_mode_whitelist[] = 127.0.0.1
- debug_mode_whitelist[] = 192.168.1.10
-
-_Notice_:
-
-* An empty file enables debug mode for anyone!
-* The bridge whitelist still applies! (debug mode does **not** enable all bridges)
-
-RSS-Bridge will give you a visual feedback when debug mode is enabled.
-
-While debug mode is active, RSS-Bridge will write additional data to your servers `error.log`.
-
-Debug mode is controlled by the static class `Debug`. It provides three core functions:
-
-* `Debug::isEnabled()`: Returns `true` if debug mode is enabled.
-* `Debug::log($message)`: Adds a message to `error.log`. It takes one parameter, which can be anything.
-
-Example: `Debug::log('Hello World!');`
-
-**Notice**: `Debug::log($message)` calls `Debug::isEnabled()` internally. You don't have to do that manually.
\ No newline at end of file
+ $middlewares = [
+ new BasicAuthMiddleware(),
+- new CacheMiddleware($this->container['cache']),
++ //new CacheMiddleware($this->container['cache']),
+ new ExceptionMiddleware($this->container['logger']),
+ new SecurityMiddleware(),
+ new MaintenanceMiddleware(),
+```
\ No newline at end of file
diff --git a/docs/04_For_Developers/07_Development_Environment_Setup.md b/docs/04_For_Developers/07_Development_Environment_Setup.md
index 23a4b101..d3a5ee8d 100644
--- a/docs/04_For_Developers/07_Development_Environment_Setup.md
+++ b/docs/04_For_Developers/07_Development_Environment_Setup.md
@@ -1,39 +1,5 @@
-These are examples of how to setup a local development environment to add bridges, improve the docs, etc.
-## Docker
-
-The following can serve as an example for using docker:
-
-```
-# create a new directory
-mkdir rss-bridge-contribution
-cd rss-bridge-contribution
-
-# clone the project into a subfolder
-git clone https://github.com/RSS-Bridge/rss-bridge
-```
-
-Then add a `docker-compose.yml` file:
-
-```yml
-version: '3'
-
-services:
- rss-bridge:
- build:
- context: ./rss-bridge
- ports:
- - 3000:80
- volumes:
- - ./config:/config
- - ./rss-bridge/bridges:/app/bridges
-```
-
-You can then access RSS-Bridge at `localhost:3000` and [add your bridge](../05_Bridge_API/How_to_create_a_new_bridge) to the `rss-bridge/bridges` folder.
-
-If you need to edit any other files, like from the `lib` folder add this to the `volumes` section: `./rss-bridge/lib:/app/lib`.
-
-### Docs with Docker
+## Docs with Docker
If you want to edit the docs add this to your docker-compose.yml:
diff --git a/docs/04_For_Developers/index.md b/docs/04_For_Developers/index.md
index fcce11e3..31e513a5 100644
--- a/docs/04_For_Developers/index.md
+++ b/docs/04_For_Developers/index.md
@@ -1,10 +1,11 @@
-This area is intended for developers who decide to contribute to **RSS-Bridge** which is primarily written in [`PHP`](http://www.php.net/) with some aspects of [`HTML`](https://en.wikipedia.org/wiki/HTML) and [`CSS`](https://en.wikipedia.org/wiki/Cascading_Style_Sheets).
+This area is intended for developers who decide to contribute to **RSS-Bridge**.
+
+It is written in PHP.
If you are new to **RSS-Bridge** you should make yourself familiar with some general aspects:
+
- [Coding style policy](./01_Coding_style_policy.md)
- [Folder structure](./03_Folder_structure.md)
- - [Debug mode](./05_Debug_mode.md)
- [Bridge API](../05_Bridge_API/index.md)
- [Cache API](../07_Cache_API/index.md)
- - [Format API](../08_Format_API/index.md)
- - [Technical recommendations](../09_Technical_recommendations/index.md)
\ No newline at end of file
+ - [Technical recommendations](../09_Technical_recommendations/index.md)
diff --git a/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md b/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md
index 391d179f..e7d401dd 100644
--- a/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md
+++ b/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md
@@ -1,30 +1,36 @@
-Create a new file in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)).
+# How to create a completely new bridge
-The file name must be named according to following specification:
-* It starts with the full name of the site
-* All white-space must be removed
-* The first letter of a word is written in upper-case, unless the site name is specified otherwise (example: Freenews, not FreeNews, because the site is named 'Freenews')
-* The first character must be upper-case
-* The file name must end with 'Bridge'
-* The file type must be PHP, written in **small** letters (seriously!) ".php"
-
-**Examples:**
-
-Site | Filename
------|---------
-Wikipedia | **Wikipedia**Bridge.php
-Facebook | **Facebook**Bridge.php
-GitHub | **GitHub**Bridge.php
-Freenews | **Freenews**Bridge.php
-
-The file must start with the PHP tags and end with an empty line. The closing tag `?>` is [omitted](http://php.net/basic-syntax.instruction-separation).
-
-**Example:**
+New code files MUST have `declare(strict_types=1);` at the top of file:
```php
find('.blog-posts li') as $li) {
+ $a = $li->find('a', 0);
+ $this->items[] = [
+ 'title' => $a->plaintext,
+ 'uri' => 'https://herman.bearblog.dev' . $a->href,
+ ];
+ }
+ }
+}
+```
+
+Learn more in [bridge api](https://rss-bridge.github.io/rss-bridge/Bridge_API/index.html).
diff --git a/docs/05_Bridge_API/02_BridgeAbstract.md b/docs/05_Bridge_API/02_BridgeAbstract.md
index 120361be..9cb16050 100644
--- a/docs/05_Bridge_API/02_BridgeAbstract.md
+++ b/docs/05_Bridge_API/02_BridgeAbstract.md
@@ -1,4 +1,5 @@
-`BridgeAbstract` is a base class for standard bridges. It implements the most common functions to simplify the process of adding new bridges.
+`BridgeAbstract` is a base class for standard bridges.
+It implements the most common functions to simplify the process of adding new bridges.
***
@@ -11,7 +12,9 @@ You need four basic steps in order to create a new bridge:
[**Step 3**](#step-3---add-general-constants-to-the-class) - Add general constants to the class
[**Step 4**](#step-4---implement-a-function-to-collect-feed-data) - Implement a function to collect feed data
-These steps are described in more detail below. At the end of this document you'll find a complete [template](#template) based on these instructions. The pictures below show an example based on these instructions:
+These steps are described in more detail below.
+At the end of this document you'll find a complete [template](#template) based on these instructions.
+The pictures below show an example based on these instructions:
Show pictures
@@ -23,7 +26,12 @@ These steps are described in more detail below. At the end of this document you'
-Make sure to read these instructions carefully. Please don't hesitate to open an [Issue](https://github.com/RSS-Bridge/rss-bridge/issues) if you have further questions (or suggestions). Once your bridge is finished, please open a [Pull Request](https://github.com/RSS-Bridge/rss-bridge/pulls), in order to get your bridge merge into RSS-Bridge.
+Make sure to read these instructions carefully.
+Please don't hesitate to open an
+[Issue](https://github.com/RSS-Bridge/rss-bridge/issues)
+if you have further questions (or suggestions).
+Once your bridge is finished, please open a [Pull Request](https://github.com/RSS-Bridge/rss-bridge/pulls),
+in order to get your bridge merge into RSS-Bridge.
***
@@ -33,7 +41,8 @@ Please read [these instructions](./01_How_to_create_a_new_bridge.md) on how to c
## Step 2 - Add a class, extending `BridgeAbstract`
-Your bridge needs to be a class, which extends `BridgeAbstract`. The class name must **exactly** match the name of the file, without the file extension.
+Your bridge needs to be a class, which extends `BridgeAbstract`.
+The class name must **exactly** match the name of the file, without the file extension.
For example: `MyBridge.php` => `MyBridge`
@@ -41,10 +50,10 @@ For example: `MyBridge.php` => `MyBridge`
```PHP
@@ -65,43 +74,47 @@ const CACHE_TIMEOUT // (optional) Defines the maximum duration for the cache in
Show example
```PHP
-
-**Notice**: `const PARAMETERS` can be used to request information from the user. Refer to [these instructions](#parameters) for more information.
+**Notice**: `const PARAMETERS` can be used to request information from the user.
+Refer to [these instructions](#parameters) for more information.
## Step 4 - Implement a function to collect feed data
-In order for RSS-Bridge to collect data, you must implement the **public** function `collectData`. This function takes no arguments and returns nothing. It generates a list of feed elements, which must be placed into the variable `$this->items`.
+In order for RSS-Bridge to collect data, you must implement the **public** function `collectData`.
+This function takes no arguments and returns nothing.
+It generates a list of feed elements, which must be placed into the variable `$this->items`.
Show example
```PHP
-items[] = $item; // Add item to the list
- }
+ public function collectData()
+ {
+ $item = [];
+ $item['title'] = 'Hello World!';
+ $this->items[] = $item;
+ }
}
-// This line is empty (just imagine it!)
```
@@ -112,32 +125,36 @@ For more details on the `collectData` function refer to [these instructions](#co
# Template
-Use this template to create your own bridge. Please remove any unnecessary comments and parameters.
+Use this template to create your own bridge.
+Please remove any unnecessary comments and parameters.
```php
items[] = $item; // Add item to the list
- }
+class MyBridge extends BridgeAbstract
+{
+ const NAME = 'Unnamed bridge';
+ const URI = '';
+ const DESCRIPTION = 'No description provided';
+ const MAINTAINER = 'No maintainer';
+ const PARAMETERS = []; // Can be omitted!
+ const CACHE_TIMEOUT = 3600; // Can be omitted!
+
+ public function collectData()
+ {
+ $item = []; // Create an empty item
+
+ $item['title'] = 'Hello World!';
+
+ $this->items[] = $item; // Add item to the list
+ }
}
-// This line is empty (just imagine it!)
```
# PARAMETERS
-You can specify additional parameters in order to customize the bridge (i.e. to specify how many items to return). This document explains how to specify those parameters and which options are available to you.
+You can specify additional parameters in order to customize the bridge (i.e. to specify how many items to return).
+This document explains how to specify those parameters and which options are available to you.
For information on how to read parameter values during execution, please refer to the [getInput](../06_Helper_functions/index.md#getinput) function.
@@ -145,15 +162,17 @@ For information on how to read parameter values during execution, please refer t
## Adding parameters to a bridge
-Parameters are specified as part of the bridge class. An empty list of parameters is defined as `const PARAMETERS = array();`
+Parameters are specified as part of the bridge class.
+An empty list of parameters is defined as `const PARAMETERS = [];`
Show example
```PHP
-Show example
```PHP
-const PARAMETERS = array(
- 'My Context 1' => array(),
- 'My Context 2' => array()
-);
+const PARAMETERS = [
+ 'My Context 1' => [],
+ 'My Context 2' => [],
+];
```
**Output**
@@ -189,37 +209,41 @@ _Notice_: The name of a context can be left empty if only one context is needed!
Show example
-You can also define a set of parameters that will be applied to every possible context of your bridge. To do this, specify a context named `global`.
+You can also define a set of parameters that will be applied to every possible context of your bridge.
+To do this, specify a context named `global`.
Show example
```PHP
-const PARAMETERS = array(
- 'global' => array() // Applies to all contexts!
-);
+const PARAMETERS = [
+ 'global' => [] // Applies to all contexts!
+];
```
## Level 2 - Parameter
-Parameters are placed inside a context. They are defined as associative array of parameter specifications. Each parameter is defined by it's internal input name, a definition in the form `'n' => array();`, where `n` is the name with which the bridge can access the parameter during execution.
+Parameters are placed inside a context.
+They are defined as associative array of parameter specifications.
+Each parameter is defined by it's internal input name, a definition in the form `'n' => [];`,
+where `n` is the name with which the bridge can access the parameter during execution.
Show example
@@ -229,17 +253,17 @@ The parameter specification consists of various fields, listed in the table belo
Show example
```PHP
-const PARAMETERS = array(
- 'My Context' => array(
- 'n' => array(
+const PARAMETERS = [
+ 'My Context' => [
+ 'n' => [
'name' => 'Limit',
'type' => 'number',
'required' => false,
'title' => 'Maximum number of items to return',
- 'defaultValue' => 10
- )
- )
-);
+ 'defaultValue' => 10,
+ ]
+ ]
+];
```
**Output**
@@ -268,29 +292,30 @@ List values are defined in an associative array where keys are the string displa
```PHP
...
'type' => 'list',
- 'values' => array(
+ 'values' => [
'Item A' => 'itemA'
'Item B' => 'itemB'
- )
+ ]
...
```
If a more complex organization is required to display the values, the above key/value can be used to set a title as a key and another array as a value:
+
```PHP
...
'type' => 'list',
- 'values' => array(
+ 'values' => [
'Item A' => 'itemA',
- 'List 1' => array(
+ 'List 1' => [
'Item C' => 'itemC',
'Item D' => 'itemD'
- ),
- 'List 2' => array(
+ ],
+ 'List 2' => [
'Item E' => 'itemE',
'Item F' => 'itemF'
- ),
+ ],
'Item B' => 'itemB'
- )
+ ]
...
```
@@ -313,20 +338,18 @@ It provides a way to identify which context the bridge is called with.
Example:
```PHP
-...
- const PARAMETERS = array(
- 'By user name' => array(
- 'u' => array('name' => 'Username')
- ),
- 'By user ID' => array(
- 'id' => array('name' => 'User ID')
- )
- );
-
-...
+const PARAMETERS = [
+ 'By user name' => [
+ 'u' => ['name' => 'Username']
+ ],
+ 'By user ID' => [
+ 'id' => ['name' => 'User ID']
+ ]
+];
```
-In this example `$this->queriedContext` will either return **By user name** or **By user ID**. The queried context might return no value, so the best way to handle it is by using a case-structure:
+In this example `$this->queriedContext` will either return **By user name** or **By user ID**.
+The queried context might return no value, so the best way to handle it is by using a case-structure:
```PHP
switch($this->queriedContext){
@@ -339,32 +362,36 @@ switch($this->queriedContext){
```
# collectData
-The `collectData` function is responsible for collecting data and adding items to generate feeds from. If you are unsure how to solve a specific problem, please don't hesitate to open an [Issue](https://github.com/RSS-Bridge/rss-bridge/issues) on GitHub. Existing bridges are also a good source to learn implementing your own bridge.
+
+The `collectData` function is responsible for collecting data and adding items to generate feeds from.
+If you are unsure how to solve a specific problem, please don't hesitate to open an [Issue](https://github.com/RSS-Bridge/rss-bridge/issues) on GitHub.
+Existing bridges are also a good source to learn implementing your own bridge.
## Implementing the `collectData` function
-Implementation for the `collectData` function is specific to each bridge. However, there are certain reoccurring elements, described below. RSS-Bridge also provides functions to simplify the process of collecting and parsing HTML data (see "Helper Functions" on the sidebar)
+Implementation for the `collectData` function is specific to each bridge.
+However, there are certain reoccurring elements, described below. RSS-Bridge also provides functions to simplify the process of collecting and parsing HTML data (see "Helper Functions" on the sidebar)
-Elements collected by this function must be stored in `$this->items`. The `items` variable is an array of item elements, each of which is an associative array that may contain arbitrary keys. RSS-Bridge specifies common keys which are used to generate most common feed formats.
+Elements collected by this function must be stored in `$this->items`.
+The `items` variable is an array of item elements, each of which is an associative array that may contain arbitrary keys.
+RSS-Bridge specifies common keys which are used to generate most common feed formats.
Show example
```PHP
-
-$item = array(); // Create a new item
-
+$item = [];
$item['title'] = 'Hello World!';
-
-$this->items[] = $item; // Add item to the list
-
+$this->items[] = $item;
```
+
Additional keys may be added for custom APIs (ignored by RSS-Bridge).
## Item parameters
-The item array should provide as much information as possible for RSS-Bridge to generate feature rich feeds. Find below list of keys supported by RSS-Bridge.
+The item array should provide as much information as possible for RSS-Bridge to generate feature rich feeds.
+Find below list of keys supported by RSS-Bridge.
```PHP
$item['uri'] // URI to reach the subject ("https://...")
@@ -376,139 +403,165 @@ $item['enclosures'] // Array of URIs to an attachments (pictures, files, etc...)
$item['categories'] // Array of categories / tags / topics
$item['uid'] // A unique ID to identify the current item
```
+
All formats support these parameters. The formats `Plaintext` and `JSON` also support custom parameters.
# getDescription
The `getDescription` function returns the description for a bridge.
-**Notice:** By default **RSS-Bridge** returns the contents of `const DESCRIPTION`, so you only have to implement this function if you require different behavior!
+**Notice:** By default **RSS-Bridge** returns the contents of `const DESCRIPTION`,
+so you only have to implement this function if you require different behavior!
```PHP
- public function getDescription(){
- return self::DESCRIPTION;
- }
+public function getDescription()
+{
+ return self::DESCRIPTION;
+}
```
# getMaintainer
The `getMaintainer` function returns the name of the maintainer for a bridge.
-**Notice:** By default **RSS-Bridge** returns `const MAINTAINER`, so you only have to implement this function if you require different behavior!
+**Notice:** By default **RSS-Bridge** returns `const MAINTAINER`,
+so you only have to implement this function if you require different behavior!
```PHP
- public function getMaintainer(){
- return self::MAINTAINER;
- }
+public function getMaintainer()
+{
+ return self::MAINTAINER;
+}
```
# getName
+
The `getName` function returns the name of a bridge.
-**Notice:** By default **RSS-Bridge** returns `const NAME`, so you only have to implement this function if you require different behavior!
+**Notice:** By default **RSS-Bridge** returns `const NAME`,
+so you only have to implement this function if you require different behavior!
```PHP
- public function getName(){
- return self::NAME;
- }
+public function getName()
+{
+ return self::NAME;
+}
```
# getURI
+
The `getURI` function returns the base URI for a bridge.
-**Notice:** By default **RSS-Bridge** returns `const URI`, so you only have to implement this function if you require different behavior!
+**Notice:** By default **RSS-Bridge** returns `const URI`,
+so you only have to implement this function if you require different behavior!
```PHP
- public function getURI(){
- return self::URI;
- }
+public function getURI()
+{
+ return self::URI;
+}
```
# getIcon
+
The `getIcon` function returns the URI for an icon, used as favicon in feeds.
-If no icon is specified by the bridge, RSS-Bridge will use a default location: `static::URI . '/favicon.ico'` (i.e. "https://github.com/favicon.ico") which may or may not exist.
+If no icon is specified by the bridge,
+RSS-Bridge will use a default location: `static::URI . '/favicon.ico'` (i.e. "https://github.com/favicon.ico") which may or may not exist.
```PHP
- public function getIcon(){
- return static::URI . '/favicon.ico';
- }
-```
-# detectParameters
-The `detectParameters` function takes a URL and attempts to extract a valid set of parameters for the current bridge.
-
-If the passed URL is valid for this bridge the function should return an array of parameter -> value pairs that can be used by this bridge, or an empty array if the bridge requires no parameters. If the URL is not relevant for this bridge the function should return `null`.
-
-**Notice:** Implementing this function is optional. By default **RSS-Bridge** tries to match the supplied URL to the `URI` constant defined in the bridge which may be enough for bridges without any parameters defined.
-
-```PHP
-public function detectParameters($url){
- $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/';
- if(empty(static::PARAMETERS)
- && preg_match($regex, $url, $urlMatches) > 0
- && preg_match($regex, static::URI, $bridgeUriMatches) > 0
- && $urlMatches[3] === $bridgeUriMatches[3]) {
- return array();
- } else {
- return null;
- }
+public function getIcon()
+{
+ return static::URI . '/favicon.ico';
}
```
+# detectParameters
+
+The `detectParameters` function takes a URL and attempts to extract a valid set of parameters for the current bridge.
+
+If the passed URL is valid for this bridge, the function should return an array of parameter -> value pairs that can be used by this bridge, including context if available, or an empty array if the bridge requires no parameters. If the URL is not relevant for this bridge, the function should return `null`.
+
+**Notice:** Implementing this function is optional. By default, **RSS-Bridge** tries to match the supplied URL to the `URI` constant defined in the bridge, which may be enough for bridges without any parameters defined.
+
+```PHP
+public function detectParameters($url)
+{
+ $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/';
+ if (empty(static::PARAMETERS)
+ && preg_match($regex, $url, $urlMatches) > 0
+ && preg_match($regex, static::URI, $bridgeUriMatches) > 0
+ && $urlMatches[3] === $bridgeUriMatches[3]
+ ) {
+ return [];
+ } else {
+ return null;
+ }
+}
+```
+
+**Notice:** This function is also used by the [findFeed](../04_For_Developers/04_Actions.md#findfeed) action.
+This action allows an user to get a list of all feeds corresponding to an URL.
+
+You can implement automated tests for the `detectParameters` function by adding the `TEST_DETECT_PARAMETERS` constant to your bridge class constant.
+
+`TEST_DETECT_PARAMETERS` is an array, with as key the URL passed to the `detectParameters`function and as value, the array of parameters returned by `detectParameters`
+
+```PHP
+const TEST_DETECT_PARAMETERS = [
+ 'https://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'],
+ 'https://instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'],
+ 'http://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'],
+];
+```
+
+**Notice:** Adding this constant is optional. If the constant is not present, no automated test will be executed.
+
+
***
# Helper Methods
-`BridgeAbstract` implements helper methods to make it easier for bridge maintainers to create bridges. Use these methods whenever possible instead of writing your own.
-- [saveCacheValue](#savecachevalue)
-- [loadCacheValue](#loadcachevalue)
+`BridgeAbstract` implements helper methods to make it easier for bridge maintainers to create bridges.
+Use these methods whenever possible instead of writing your own.
## saveCacheValue
-Within the context of the current bridge, stores a value by key in the cache. The value can later be retrieved with [loadCacheValue](#loadcachevalue).
+
+Within the context of the current bridge, stores a value by key in the cache.
+The value can later be retrieved with [loadCacheValue](#loadcachevalue).
```php
-protected function saveCacheValue($key, $value)
+protected function saveCacheValue($key, $value, $ttl = null)
```
-- `$key` - the name under which the value is stored in the cache.
-- `$value` - the value to store in the cache.
-
-Usage example:
+Example:
```php
-const MY_KEY = 'MyKey';
-
public function collectData()
{
- $value = 'my value';
- $this->saveCacheValue(MY_KEY, $value);
+ $this->saveCacheValue('my_key', 'my_value', 3600); // 1h
}
```
## loadCacheValue
-Within the context of the current bridge, loads a value by key from cache. Optionally specifies the cache duration for the key. Returns `null` if the key doesn't exist or the value is expired.
+
+Within the context of the current bridge, loads a value by key from cache.
+Optionally specifies the cache duration for the key.
+Returns `null` if the key doesn't exist or the value is expired.
```php
-protected function loadCacheValue($key, $duration = null)
+protected function loadCacheValue($key, $default = null)
```
-- `$key` - the name under which the value is stored in the cache.
-- `$duration` - the maximum time in seconds after which the value expires.
-
-Usage example:
+Example:
```php
-const MY_KEY = 'MyKey';
-
public function collectData()
{
- $value = $this->loadCacheValue(MY_KEY, 1800 /* 30 minutes */);
+ $value = $this->loadCacheValue('my_key');
- if (!isset($value)){
- // load value
- $this->saveCacheValue(MY_KEY, $value);
- }
-
- // ...
+ if (! $value) {
+ $this->saveCacheValue('my_key', 'foobar');
+ }
}
```
diff --git a/docs/05_Bridge_API/03_FeedExpander.md b/docs/05_Bridge_API/03_FeedExpander.md
index 7e72670a..62356fc9 100644
--- a/docs/05_Bridge_API/03_FeedExpander.md
+++ b/docs/05_Bridge_API/03_FeedExpander.md
@@ -1,85 +1,35 @@
-`FeedExpander` extends [`BridgeAbstract`](./02_BridgeAbstract.md) and adds functions to collect data from existing feeds.
-
**Usage example**: _You have discovered a site that provides feeds which are hidden and inaccessible by normal means. You want your bridge to directly read the feeds and provide them via **RSS-Bridge**_
-To create a new Bridge extending `FeedExpander` you must implement all required functions of [`BridgeAbstract`](./02_BridgeAbstract.md). `FeedExpander` additionally provides following functions:
-
-* [`parseItem`](#the-parseitem-function)
-* [`getName`](#the-getname-function)
-* [`getURI`](#the-geturi-function)
-* [`getDescription`](#the-getdescription-function)
-
Find a [template](#template) at the end of this file.
**Notice:** For a standard feed only `collectData` need to be implemented. `collectData` should call `$this->collectExpandableDatas('your URI here');` to automatically load feed items and header data (will subsequently call `parseItem` for each item in the feed). You can limit the number of items to fetch by specifying an additional parameter for: `$this->collectExpandableDatas('your URI here', 10)` (limited to 10 items).
-## The `parseItem` function
+## The `parseItem` method
-This function receives one item from the current feed and should return one **RSS-Bridge** item.
+This method receives one item from the current feed and should return one **RSS-Bridge** item.
The default function does all the work to get the item data from the feed, whether it is RSS 1.0,
-RSS 2.0 or Atom 1.0. If you have to redefine this function in your **RSS-Bridge** for whatever reason,
-you should first call the parent function to initialize the item, then apply the changes that you require.
+RSS 2.0 or Atom 1.0.
**Notice:** The following code sample is just an example. Implementation depends on your requirements!
```PHP
-protected function parseItem($feedItem){
- $item = parent::parseItem($feedItem);
- $item['content'] = str_replace('rssbridge','RSS-Bridge',$feedItem->content);
-
+protected function parseItem(array $item)
+{
+ $item['content'] = str_replace('rssbridge','RSS-Bridge',$item['content']);
return $item;
}
```
-### Helper functions
+### Feed parsing
-The `FeedExpander` already provides a set of functions to parse RSS or Atom items based on the specifications. Where possible make use of these functions:
-
-Function | Description
----------|------------
-`parseATOMItem` | Parses an Atom 1.0 feed item
-`parseRSS_0_9_1_Item` | Parses an RSS 0.91 feed item
-`parseRSS_1_0_Item` | Parses an RSS 1.0 feed item
-`parseRSS_2_0_Item` | Parses an RSS 2.0 feed item
-
-In the following list you'll find the feed tags assigned to the the **RSS-Bridge** item keys:
+How rss-bridge processes xml feeds:
Function | uri | title | timestamp | author | content
---------|-----|-------|-----------|--------|--------
-`parseATOMItem` | id | title | updated | author | content
-`parseRSS_0_9_1_Item` | link | title | | | description
-`parseRSS_1_0_Item` | link | title | dc:date | dc:creator | description
-`parseRSS_2_0_Item` | link, guid | title | pubDate, dc:date | author, dc:creator | description
-
-## The `getName` function
-
-Returns the name of the current feed.
-
-```PHP
-return $this->name;
-```
-
-**Notice:** Only implement this function if you require different behavior!
-
-## The `getURI` function
-
-Return the uri for the current feed.
-
-```PHP
-return $this->uri;
-```
-
-**Notice:** Only implement this function if you require different behavior!
-
-## The `getDescription` function
-
-Returns the description for the current bridge.
-
-```PHP
-return $this->description;
-```
-
-**Notice:** Only implement this function if you require different behavior!
+`atom` | id | title | updated | author | content
+`rss 0.91` | link | title | | | description
+`rss 1.0` | link | title | dc:date | dc:creator | description
+`rss 2.0` | link, guid | title | pubDate, dc:date | author, dc:creator | description
# Template
@@ -87,18 +37,19 @@ This is the template for a new bridge:
```PHP
collectExpandableDatas('your feed URI');
}
}
-// Imaginary empty line!
```
\ No newline at end of file
diff --git a/docs/05_Bridge_API/04_WebDriverAbstract.md b/docs/05_Bridge_API/04_WebDriverAbstract.md
new file mode 100644
index 00000000..60b5e99d
--- /dev/null
+++ b/docs/05_Bridge_API/04_WebDriverAbstract.md
@@ -0,0 +1,83 @@
+`WebDriverAbstract` extends [`BridgeAbstract`](./02_BridgeAbstract.md) and adds functionality for generating feeds
+from active websites that use XMLHttpRequest (XHR) to load content and / or JavaScript to
+modify content.
+It highly depends on the php-webdriver library which offers Selenium WebDriver bindings for PHP.
+
+- https://github.com/php-webdriver/php-webdriver (Project Repository)
+- https://php-webdriver.github.io/php-webdriver/latest/ (API)
+
+Please note that this class is intended as a solution for websites _that cannot be covered
+by the other classes_. The WebDriver starts a browser and is therefore very resource-intensive.
+
+# Configuration
+
+You need a running WebDriver to use bridges that depend on `WebDriverAbstract`.
+The easiest way is to start the Selenium server from the project of the same name:
+```
+docker run -d -p 4444:4444 --shm-size="2g" docker.io/selenium/standalone-chrome:latest
+```
+
+- https://github.com/SeleniumHQ/docker-selenium
+
+With these parameters only one browser window can be started at a time.
+On a multi-user site, Selenium Grid should be used
+and the number of sessions should be adjusted to the number of processor cores.
+
+Finally, the `config.ini.php` file must be adjusted so that the WebDriver
+can find the Selenium server:
+```
+[webdriver]
+
+selenium_server_url = "http://localhost:4444"
+```
+
+# Development
+
+While you are programming a new bridge, it is easier to start a local WebDriver because then you can see what is happening and where the errors are. I've also had good experience recording the process with a screen video to find any timing problems.
+
+```
+chromedriver --port=4444
+```
+
+- https://chromedriver.chromium.org/
+
+If you start rss-bridge from a container, then Chrome driver is only accessible
+if you call it with the `--allowed-ips` option so that it binds to all network interfaces.
+
+```
+chromedriver --port=4444 --allowed-ips=192.168.1.42
+```
+
+The **most important rule** is that after an event such as loading the web page
+or pressing a button, you often have to explicitly wait for the desired elements to appear.
+
+A simple example is the bridge `ScalableCapitalBlogBridge.php`.
+A more complex and relatively complete example is the bridge `GULPProjekteBridge.php`.
+
+# Template
+
+Use this template to create your own bridge.
+
+```PHP
+cleanUp();
+ }
+ }
+}
+
+```
\ No newline at end of file
diff --git a/docs/05_Bridge_API/04_XPathAbstract.md b/docs/05_Bridge_API/05_XPathAbstract.md
similarity index 94%
rename from docs/05_Bridge_API/04_XPathAbstract.md
rename to docs/05_Bridge_API/05_XPathAbstract.md
index cf091edc..28a20a81 100644
--- a/docs/05_Bridge_API/04_XPathAbstract.md
+++ b/docs/05_Bridge_API/05_XPathAbstract.md
@@ -68,6 +68,9 @@ Should return the XPath expression for extracting an item title from the item co
### Method `getExpressionItemContent()`
Should return the XPath expression for extracting an item's content from the item context.
+### Method `getSettingUseRawItemContent()`
+Should return the 'Use raw item content' setting value (bool true or false).
+
### Method `getExpressionItemUri()`
Should return the XPath expression for extracting an item link from the item context.
@@ -96,7 +99,7 @@ This method should return the HTML source as a base for the XPath expressions. U
This method should provide the feed title. Usually the XPath expression defined in `XPATH_EXPRESSION_FEED_TITLE` is used for extracting the title directly from the page source.
### Method `provideFeedIcon()`
-This method should provide the feed title. Usually the XPath expression defined in `XPATH_EXPRESSION_FEED_ICON` is used for extracting the title directly from the page source.
+This method should provide the URL of the feed's favicon. Usually the XPath expression defined in `XPATH_EXPRESSION_FEED_ICON` is used for extracting the title directly from the page source.
### Method `provideFeedItems()`
This method should provide the feed items. Usually the XPath expression defined in `XPATH_EXPRESSION_ITEM` is used for extracting the items from the page source. All other XPath expressions are applied on a per-item basis, item by item, and only on the item's contents.
@@ -119,8 +122,8 @@ Accepts the items author as parameter, processes and returns it. Should return a
### Method `formatItemTimestamp()`
Accepts the items creation timestamp as parameter, processes and returns it. Should return a unix timestamp as integer.
-### Method `cleanImageUrl()`
-Method invoked for cleaning feed icon and item image URL's. Extracts the image URL from the passed parameter, stripping any additional content. Furthermore makes sure that relative image URL's get transformed to absolute ones.
+### Method `cleanMediaUrl()`
+Method invoked for cleaning feed icon, item image and media attachment (like .mp3, .webp) URL's. Extracts the media URL from the passed parameter, stripping any additional content. Furthermore, makes sure that relative media URL's get transformed to absolute ones.
### Method `fixEncoding()`
Only invoked when class constant `SETTING_FIX_ENCODING` is set to true. It then passes all extracted string values through PHP's `utf8_decode` function.
diff --git a/docs/05_Bridge_API/index.md b/docs/05_Bridge_API/index.md
index 6115fa01..ea6fd315 100644
--- a/docs/05_Bridge_API/index.md
+++ b/docs/05_Bridge_API/index.md
@@ -1,9 +1,14 @@
-A _Bridge_ is an class that allows **RSS-Bridge** to create an RSS-feed from a website. A _Bridge_ represents one element on the [Welcome screen](../01_General/04_Screenshots.md) and covers one or more sites to return feeds for. It is developed in a PHP file located in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)) and extends one of the base classes of **RSS-Bridge**:
+A _Bridge_ is a class that allows **RSS-Bridge** to create an RSS-feed from a website.
+A _Bridge_ represents one element on the [Welcome screen](../01_General/04_Screenshots.md)
+and covers one or more sites to return feeds for.
+It is developed in a PHP file located in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md))
+and extends one of the base classes of **RSS-Bridge**:
Base class | Description
-----------|------------
[`BridgeAbstract`](./02_BridgeAbstract.md) | This class is intended for standard _Bridges_ that need to filter HTML pages for content.
-[`FeedExpander`](./03_FeedExpander.md) | This class is an extension of `HttpCachingBridgeAbstract`, designed to load existing feeds into **RSS-Bridge**
-[`XPathAbstract`](./04_XPathAbstract.md) | This class is meant as an alternative base class for bridge implementations. It offers preliminary functionality for generating feeds based on _XPath expressions_.
+[`FeedExpander`](./03_FeedExpander.md) | Expand/modify existing feed urls
+[`WebDriverAbstract`](./04_WebDriverAbstract) |
+[`XPathAbstract`](./05_XPathAbstract) | This class is meant as an alternative base class for bridge implementations. It offers preliminary functionality for generating feeds based on _XPath expressions_.
For more information about how to create a new _Bridge_, read [How to create a new Bridge?](./01_How_to_create_a_new_bridge.md)
\ No newline at end of file
diff --git a/docs/06_Helper_functions/index.md b/docs/06_Helper_functions/index.md
index 2f0c513c..4004138e 100644
--- a/docs/06_Helper_functions/index.md
+++ b/docs/06_Helper_functions/index.md
@@ -5,10 +5,14 @@ The `getInput` function is used to receive a value for a parameter, specified in
$this->getInput('your input name here');
```
-`getInput` will either return the value for your parameter or `null` if the parameter is unknown or not specified.
+`getInput` will either return the value for your parameter
+or `null` if the parameter is unknown or not specified.
+
+[Defined in lib/BridgeAbstract.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/BridgeAbstract.php)
# getKey
-The `getKey` function is used to receive the key name to a selected list value given the name of the list, specified in `const PARAMETERS`
+The `getKey` function is used to receive the key name to a selected list
+value given the name of the list, specified in `const PARAMETERS`
Is able to work with multidimensional list arrays.
```PHP
@@ -34,7 +38,10 @@ $this->getKey('country');
// if the selected value was "ve", this function will return "Venezuela"
```
-`getKey` will either return the key name for your parameter or `null` if the parameter is unknown or not specified.
+`getKey` will either return the key name for your parameter or `null` if the parameter
+is unknown or not specified.
+
+[Defined in lib/BridgeAbstract.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/BridgeAbstract.php)
# getContents
The `getContents` function uses [cURL](https://secure.php.net/manual/en/book.curl.php) to acquire data from the specified URI while respecting the various settings defined at a global level by RSS-Bridge (i.e., proxy host, user agent, etc.). This function accepts a few parameters:
@@ -52,52 +59,64 @@ $opts = array(CURLOPT_POST => 1);
$html = getContents($url, $header, $opts);
```
+[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php)
+
# getSimpleHTMLDOM
-The `getSimpleHTMLDOM` function is a wrapper for the [simple_html_dom](https://simplehtmldom.sourceforge.io/) [file_get_html](https://simplehtmldom.sourceforge.io/docs/1.9/api/file_get_html/) function in order to provide context by design.
+The `getSimpleHTMLDOM` function is a wrapper for the
+[simple_html_dom](https://simplehtmldom.sourceforge.io/) [file_get_html](https://simplehtmldom.sourceforge.io/docs/1.9/api/file_get_html/) function in order to provide context by design.
```PHP
$html = getSimpleHTMLDOM('your URI');
```
+
+[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php)
+
# getSimpleHTMLDOMCached
-The `getSimpleHTMLDOMCached` function does the same as the [`getSimpleHTMLDOM`](#getsimplehtmldom) function, except that the content received for the given URI is stored in a cache and loaded from cache on the next request if the specified cache duration was not reached. Use this function for data that is very unlikely to change between consecutive requests to **RSS-Bridge**. This function allows to specify the cache duration with the second parameter (default is 24 hours / 86400 seconds).
+The `getSimpleHTMLDOMCached` function does the same as the
+[`getSimpleHTMLDOM`](#getsimplehtmldom) function,
+except that the content received for the given URI is stored in a cache
+and loaded from cache on the next request if the specified cache duration
+was not reached.
+
+Use this function for data that is very unlikely to change between consecutive requests to **RSS-Bridge**.
+This function allows to specify the cache duration with the second parameter.
```PHP
$html = getSimpleHTMLDOMCached('your URI', 86400); // Duration 24h
```
-**Notice:** Due to the current implementation a value greater than 86400 seconds (24 hours) will not work as the cache is purged every 24 hours automatically.
+[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php)
-# returnError
-**Notice:** Whenever possible make use of [`returnClientError`](#returnclienterror) or [`returnServerError`](#returnservererror)
-
-The `returnError` function aborts execution of the current bridge and returns the given error message with the provided error number:
+# throwClientException($message = '')
+The `throwClientException` function aborts execution of the current bridge.
```PHP
-returnError('Your error message', 404);
-```
-
-Check the [list of error codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) for applicable error numbers.
-
-# returnClientError
-The `returnClientError` function aborts execution of the current bridge and returns the given error message with error code **400**:
-
-```PHP
-returnClientError('Your error message')
+throwClientException('Bad user input')
```
Use this function when the user provided invalid parameter or a required parameter is missing.
-# returnServerError
-The `returnServerError` function aborts execution of the current bridge and returns the given error message with error code **500**:
+[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php)
+
+# throwServerException($message = '')
+The `throwServerException` function aborts execution of the current bridge.
```PHP
-returnServerError('Your error message')
+throwServerException('Received empty reply from thirdparty api')
```
-Use this function when a problem occurs that has nothing to do with the parameters provided by the user. (like: Host service gone missing, empty data received, etc...)
+Use this function when a problem occurs that has nothing to do with the parameters provided by the user.
+(like: Host service gone missing, empty data received, etc...)
+
+[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php)
+
+# throwRateLimitException($message = '')
+
+Throws a `RateLimitException` which produces an HTTP 429 response.
# defaultLinkTo
-Automatically replaces any relative URL in a given string or DOM object (i.e. the one returned by [getSimpleHTMLDOM](#getsimplehtmldom)) with an absolute URL.
+Automatically replaces any relative URL in a given string or DOM object
+(i.e. the one returned by [getSimpleHTMLDOM](#getsimplehtmldom)) with an absolute URL.
```php
defaultLinkTo ( mixed $content, string $server ) : object
@@ -121,6 +140,8 @@ $html = defaultLinkTo($html, $this->getURI()); // Using bridge URL
//
```
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
# backgroundToImg
Replaces tags with styles of `backgroud-image` by `` tags.
@@ -130,6 +151,8 @@ backgroundToImg(mixed $htmlContent) : object
Returns a DOM object (even if provided a string).
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
# extractFromDelimiters
Extract the first part of a string matching the specified start and end delimiters.
```php
@@ -150,6 +173,8 @@ $extracted = extractFromDelimiters($string, $start, $end);
// 'John Doe'
```
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
# stripWithDelimiters
Remove one or more part(s) of a string using a start and end delimiter.
It is the inverse of `extractFromDelimiters`.
@@ -172,6 +197,8 @@ $cleaned = stripWithDelimiters($string, $start, $end);
// 'foobar'
```
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
# stripRecursiveHTMLSection
Remove HTML sections containing one or more sections using the same HTML tag.
@@ -191,6 +218,8 @@ $cleaned = stripRecursiveHTMLSection($string, $tag_name, $tag_start);
// 'foobar'
```
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
# markdownToHtml
Converts markdown input to HTML using [Parsedown](https://parsedown.org/).
@@ -232,3 +261,84 @@ $html = markdownToHtml($input);
//
Translation improvements
//
```
+
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
+# e
+The `e` function is used to convert special characters to HTML entities
+
+```PHP
+e('0 < 1 and 2 > 1');
+```
+
+`e` will return the content of the string escape that can be rendered as is in HTML
+
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
+# truncate
+The `truncate` function is used to shorten a string if exceeds a certain length, and add a string indicating that the string has been shortened.
+
+```PHP
+truncate('Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed a neque nunc. Nam nibh sem.', 20 , '...');
+```
+
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
+# sanitize
+The `sanitize` function is used to remove some tags from a given HTML text.
+
+```PHP
+$html = 'Sample Page
+
Lorem ipsum dolor sit amet, consectetur adipiscing elit...
+
+
+';
+$tags_to_remove = ['script', 'iframe', 'input', 'form'];
+$attributes_to_keep = ['title', 'href', 'src'];
+$text_to_keep = [];
+sanitize($html, $tags_to_remove, $attributes_to_keep, $text_to_keep);
+```
+
+This function returns a simplehtmldom object of the remaining contents.
+
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
+# convertLazyLoading
+The `convertLazyLoading` function is used to convert onvert lazy-loading images and frames (video embeds) into static elements. It accepts the HTML content as HTML objects or string objects. It returns the HTML content with fixed image/frame URLs (same type as input).
+
+```PHP
+$html = '
+
+
Hello world!
+
+
+backgroundToImg($html);
+```
+
+[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php)
+
+# Json::encode
+The `Json::encode` function is used to encode a value as à JSON string.
+
+```PHP
+$array = [
+ "foo" => "bar",
+ "bar" => "foo",
+];
+Json::encode($array, true, true);
+```
+
+[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php)
+
+# Json::decode
+The `Json::decode` function is used to decode a JSON string into à PHP variable.
+
+```PHP
+$json = '{
+ "foo": "bar",
+ "bar": "foo"
+}';
+Json::decode($json);
+```
+
+[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php)
diff --git a/docs/07_Cache_API/02_CacheInterface.md b/docs/07_Cache_API/02_CacheInterface.md
index 61127a0d..3e71237d 100644
--- a/docs/07_Cache_API/02_CacheInterface.md
+++ b/docs/07_Cache_API/02_CacheInterface.md
@@ -3,16 +3,14 @@ See `CacheInterface`.
```php
interface CacheInterface
{
- public function setScope(string $scope): void;
+ public function get(string $key, $default = null);
- public function setKey(array $key): void;
+ public function set(string $key, $value, int $ttl = null): void;
- public function loadData();
+ public function delete(string $key): void;
- public function saveData($data): void;
+ public function clear(): void;
- public function getTime(): ?int;
-
- public function purgeCache(int $seconds): void;
+ public function prune(): void;
}
-```
\ No newline at end of file
+```
diff --git a/docs/07_Cache_API/index.md b/docs/07_Cache_API/index.md
index 57692847..17afbacc 100644
--- a/docs/07_Cache_API/index.md
+++ b/docs/07_Cache_API/index.md
@@ -1,4 +1,6 @@
-A _Cache_ is a class that allows **RSS-Bridge** to store fetched data in a local storage area on the server. Cache imlementations are placed in the `caches/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). A cache must implement the [`CacheInterface`](../07_Cache_API/02_CacheInterface.md) interface.
-
-For more information about how to create a new `Cache`, read [How to create a new cache?](../07_Cache_API/01_How_to_create_a_new_cache.md)
+A _Cache_ is a class that allows **RSS-Bridge** to store fetched data in a local storage area on the server.
+Cache imlementations are placed in the `caches/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)).
+A cache must implement the [`CacheInterface`](../07_Cache_API/02_CacheInterface.md) interface.
+For more information about how to create a new `Cache`, read
+[How to create a new cache?](../07_Cache_API/01_How_to_create_a_new_cache.md)
diff --git a/docs/08_Format_API/01_How_to_create_a_new_format.md b/docs/08_Format_API/01_How_to_create_a_new_format.md
deleted file mode 100644
index f031e65b..00000000
--- a/docs/08_Format_API/01_How_to_create_a_new_format.md
+++ /dev/null
@@ -1,24 +0,0 @@
-Create a new file in the `formats/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)).
-
-The file must be named according to following specification:
-
-* It starts with the type
-* The file name must end with 'Format'
-* The file type must be PHP, written in small letters (seriously!) ".php"
-
-**Examples:**
-
-Type | Filename
------|---------
-Atom | AtomFormat.php
-Html | HtmlFormat.php
-
-The file must start with the PHP tags and end with an empty line. The closing tag `?>` is [omitted](http://php.net/basic-syntax.instruction-separation).
-
-Example:
-
-```PHP
-items = $items;
- return $this;
- }
-
- public function getItems(){
- return $this->items;
- }
-
- public function setCharset($charset){
- $this->charset = $charset;
- return $this;
- }
-
- public function getCharset(){
- return $this->charset;
- }
-
- public function setExtraInfos(array $infos){
- $this->extraInfos = $infos;
- return $this;
- }
-
- public function getExtraInfos(){
- return $this->extraInfos;
- }
-}
-// Imaginary empty line!
-```
diff --git a/docs/08_Format_API/index.md b/docs/08_Format_API/index.md
deleted file mode 100644
index c5b9e6af..00000000
--- a/docs/08_Format_API/index.md
+++ /dev/null
@@ -1,9 +0,0 @@
-A Format is a class that allows RSS-Bridge to turn items from a bridge into an RSS-feed format.
-It is developed in a PHP file located in the `formats/` folder
-[Folder structure](../04_For_Developers/03_Folder_structure.md)
-and either implements the
-[FormatInterface](../08_Format_API/02_FormatInterface.md)
-interface or extends the FormatAbstract class.
-
-For more information about how to create a new _Format_, read
-[How to create a new Format?](./01_How_to_create_a_new_format.md)
\ No newline at end of file
diff --git a/docs/09_Technical_recommendations/index.md b/docs/09_Technical_recommendations/index.md
index f2b15476..92c672ca 100644
--- a/docs/09_Technical_recommendations/index.md
+++ b/docs/09_Technical_recommendations/index.md
@@ -1,28 +1,32 @@
## General recommendations
-* Use [HTTPS](https://en.wikipedia.org/wiki/HTTPS) (`https://...`) over [HTTP](https://en.wikipedia.org/wiki/HTTPS) (`http://...`) whenever possible
-
## Test a site before building a bridge
-Some sites make use of anti-bot mechanisms (e.g.: by using JavaScript) in which case they work fine in regular browsers, but not in the PHP environment. To check if a site works with RSS-Bridge, create a new bridge using the [template](../05_Bridge_API/02_BridgeAbstract.md#template) and load a valid URL (not the base URL!).
+Some sites make use of anti-bot mechanisms (e.g.: by using JavaScript) in which case they work fine in regular browsers,
+but not in the PHP environment. RSS-Bridge Docker container by default resorts to using libcurl-impersonate, which helps mitigating anti-bot mechanisms.
+
+To check if a site works with RSS-Bridge, create a new bridge using the
+[template](../05_Bridge_API/02_BridgeAbstract.md#template)
+and load a valid URL (not the base URL!).
**Example (using github.com)**
```PHP
"
+
+[EconomistBridge]
+cookie = ""
+```
diff --git a/docs/10_Bridge_Specific/FacebookBridge.md b/docs/10_Bridge_Specific/FacebookBridge.md
index c2a1fd0e..665b802f 100644
--- a/docs/10_Bridge_Specific/FacebookBridge.md
+++ b/docs/10_Bridge_Specific/FacebookBridge.md
@@ -1,18 +1,18 @@
FacebookBridge
===============
-Resume of the actual state of this bridge:
+State of this bridge:
+- Facebook Groups (and probably other sections too) do not work at all
- No maintainer
-- Need Cookies consent
-- New design architecture deployed
+- Needs cookie consent support for public pages
+- Needs login support see [this example](https://github.com/RSS-Bridge/rss-bridge/issues/1891) for Instagram) for private groups
-Due [facebook-redesing](https://engineering.fb.com/2020/05/08/web/facebook-redesign/)
+Due to the 2020 [Facebook redesign](https://engineering.fb.com/2020/05/08/web/facebook-redesign/)
and the requirement to [accept cookies](https://www.facebook.com/business/help/348535683460989)
-users start getting [Problems with Facebook on public RSS-Bridge instances](https://github.com/RSS-Bridge/rss-bridge/issues/2047 )
+users are getting [problems with Facebook on public RSS-Bridge instances](https://github.com/RSS-Bridge/rss-bridge/issues/2047).
+Relevant Info
+--------------
-[Facebook Cookies](https://www.facebook.com/policy/cookies/)
-
-"Datr" is a unique identifier for your browser and it has a lifespan of two years.
-
-"c_user" and "xs" cookies to verify the account and have a lifespan of 365 days
-
+- [Facebook Cookies](https://www.facebook.com/policy/cookies/)
+- "Datr" is a unique identifier for your browser and it has a lifespan of two years.
+- "c_user" and "xs" cookies to verify the account and have a lifespan of 365 days
diff --git a/docs/10_Bridge_Specific/FurAffinityBridge.md b/docs/10_Bridge_Specific/FurAffinityBridge.md
new file mode 100644
index 00000000..369f41cd
--- /dev/null
+++ b/docs/10_Bridge_Specific/FurAffinityBridge.md
@@ -0,0 +1,13 @@
+FurAffinityBridge
+===============
+By default this bridge will only return submissions that are rated "General" and are public.
+
+To unlock the ability to load submissions that require an account to view or are rated "Mature" and higher, you must set the following in `config.ini.php` with cookies from an existing FurAffinity account with the desired maturity ratings enabled in [Account Settings](https://www.furaffinity.net/controls/settings/).
+
+```
+[FurAffinityBridge]
+aCookie = "your-a-cookie-value-here" ; from cookie "a"
+bCookie = "your-b-cookie-value-here" ; from cookie "b"
+```
+
+To confirm the bridge is authenticated, the name of the authenticating account will be shown in the bridge's name once the bridge has been used at least once. (Example: `user's FurAffinity Bridge`)
diff --git a/docs/10_Bridge_Specific/Instagram.md b/docs/10_Bridge_Specific/Instagram.md
index 7b9bf6b2..85e7b8a8 100644
--- a/docs/10_Bridge_Specific/Instagram.md
+++ b/docs/10_Bridge_Specific/Instagram.md
@@ -4,6 +4,8 @@ InstagramBridge
To somehow bypass the [rate limiting issue](https://github.com/RSS-Bridge/rss-bridge/issues/1891)
it is suggested to deploy a private RSS-Bridge instance that uses a working Instagram account.
+**NOTE**: There exists alternative bridges (e.g. PicukiBridge and PicnobBridge) for viewing posts without a working account.
+
Configuration
-------------
diff --git a/docs/10_Bridge_Specific/PixivBridge.md b/docs/10_Bridge_Specific/PixivBridge.md
new file mode 100644
index 00000000..ba8da2d8
--- /dev/null
+++ b/docs/10_Bridge_Specific/PixivBridge.md
@@ -0,0 +1,30 @@
+PixivBridge
+===============
+
+# Image proxy
+
+As Pixiv requires images to be loaded with the `Referer "https://www.pixiv.net/"` header set,
+caching or image proxy is required to use this bridge.
+
+To turn off image caching, set the `proxy_url` value in this bridge's configuration section of `config.ini.php`
+to the url of the proxy.
+
+The bridge will then use the proxy in this format (essentially replacing `https://i.pximg.net` with the proxy):
+
+Before: `https://i.pximg.net/img-original/img/0000/00/00/00/00/00/12345678_p0.png`
+
+After: `https://proxy.example.com/img-original/img/0000/00/00/00/00/00/12345678_p0.png`
+
+```
+proxy_url = "https://proxy.example.com"
+```
+
+# Authentication
+
+Authentication is required to view and search R-18+ and non-public images.
+To enable this, set the following in this bridge's configuration in `config.ini.php`.
+
+```ini
+; from cookie "PHPSESSID". Recommend to get in incognito browser.
+cookie = "00000000_hashedsessionidhere"
+```
\ No newline at end of file
diff --git a/docs/10_Bridge_Specific/Substack.md b/docs/10_Bridge_Specific/Substack.md
new file mode 100644
index 00000000..7595bbef
--- /dev/null
+++ b/docs/10_Bridge_Specific/Substack.md
@@ -0,0 +1,18 @@
+# SubstackBridge
+
+[Substack](https://substack.com) provides RSS feeds at `/feed` path, e.g., https://newsletter.pragmaticengineer.com/feed/. However, these feeds have two problems, addressed by this bridge:
+- They use RSS 2.0 with the draft [content extension](https://web.resource.org/rss/1.0/modules/content/), which isn't supported by some readers;
+- They don't have the full content for paywalled posts.
+
+Retrieving the full content is only possible _with an active subscription to the blog_. If you have one, Substack will return the full feed if it's fetched with the right set of cookies. Figuring out whether it's the intended behaviour is left as an exercise for the reader.
+
+To obtain the session cookie, authorize at https://substack.com/, open DevTools, go to Application -> Cookies -> https://substack.com, copy the value of `substack.sid` and paste it to the RSS bridge config:
+
+```
+[SubstackBridge]
+sid = ""
+```
+
+Authorization sometimes requires CAPTCHA, hence this operation is manual. The cookie lives for three months.
+
+After you've done this, the bridge should return full feeds for your subscriptions.
diff --git a/docs/10_Bridge_Specific/Telegram.md b/docs/10_Bridge_Specific/Telegram.md
new file mode 100644
index 00000000..528de788
--- /dev/null
+++ b/docs/10_Bridge_Specific/Telegram.md
@@ -0,0 +1,12 @@
+# TelegramBridge
+
+By default, it fetches a single page with up to 20 messages.
+
+To increase this limit, tweak the `max_pages` config:
+
+```ini
+[TelegramBridge]
+
+; Fetch a maximum of 3 pages (requires 3 http requests)
+max_pages = 3
+```
diff --git a/docs/10_Bridge_Specific/Vk2.md b/docs/10_Bridge_Specific/Vk2.md
new file mode 100644
index 00000000..7c48ad0a
--- /dev/null
+++ b/docs/10_Bridge_Specific/Vk2.md
@@ -0,0 +1,41 @@
+Vk2Bridge
+=========
+
+Работа этого скрипта основана [VK API](https://dev.vk.com/reference).
+По сравнению с VkBridge у этого скрипта есть свои приемущества и недостатки.
+
+Приемущества
+------------
+
+- Стабильность.
+ Скрипт не зависит от HTML-структуры страницы VK групп или пользователей, которые могут поменяться в любой момент.
+
+Недостатки
+----------
+
+- Требуется наличие зарегистированного в ВК пользователя.
+ Данный пользователь должен получить `access_token`, который используется для этого скрипта.
+ Подробнее в разделе "Настройка"
+
+- Количество запросов при выключенном кэше ограничено - [5000 запросов в сутки](https://dev.vk.com/ru/reference/roadmap#%D0%9E%D0%B3%D1%80%D0%B0%D0%BD%D0%B8%D1%87%D0%B5%D0%BD%D0%B8%D1%8F%20API%20%D0%B4%D0%BB%D1%8F%20%D0%BF%D0%BE%D0%B8%D1%81%D0%BA%D0%B0)
+
+Настройка
+---------
+
+1. Перейдите по [ссылке](https://oauth.vk.com/oauth/authorize?client_id=5149410&scope=offline&redirect_uri=https://oauth.vk.com/blank.html&display=page&response_type=token)
+
+2. Авторизуйтесь в приложение `my_personal_app`
+
+3. Получите ссылку вида `https://oauth.vk.com/blank.html#access_token=MNOGO_BUKAV&expires_in=0&user_id=123456`.
+ Из этой ссылки скопируйте `MNOGO_BUKAV`.
+
+4. В `config.ini.php` в раздел Vk2Bridge вставьте `access_token`
+
+```
+[Vk2Bridge]
+access_token = "MNOGO_BUKAV"
+```
+
+Примечание: в данной инструкции используется приложение, администратор которого является [@em92](https://github.com/em92).
+Допускается вместо упомянутого приложения использование своего standalone-приложения.
+Для этого надо в ссылке из п.1. заменить значение `client_id` на свой.
diff --git a/docs/images/screenshot_rss-bridge_welcome.png b/docs/images/screenshot_rss-bridge_welcome.png
index 3ceb432e..9a0dcfc5 100644
Binary files a/docs/images/screenshot_rss-bridge_welcome.png and b/docs/images/screenshot_rss-bridge_welcome.png differ
diff --git a/docs/index.md b/docs/index.md
index 71fa9f37..c370cb1b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,4 +1,8 @@
-**RSS-Bridge** is free and open source software for generating Atom or RSS feeds from websites which don't have one. It is written in PHP and intended to run on a Web server. See the [Screenshots](01_General/04_Screenshots.md) for a quick introduction to **RSS-Bridge**
+RSS-Bridge is a web application.
+
+It generates web feeds for websites that don't have one.
+
+Officially hosted instance: https://rss-bridge.org/bridge01/
- You want to know more about **RSS-Bridge**?
Check out our **[project goals](01_General/01_Project-goals.md)**.
diff --git a/formats/AtomFormat.php b/formats/AtomFormat.php
index 1b9053fa..8df51189 100644
--- a/formats/AtomFormat.php
+++ b/formats/AtomFormat.php
@@ -14,52 +14,67 @@ class AtomFormat extends FormatAbstract
protected const ATOM_NS = 'http://www.w3.org/2005/Atom';
protected const MRSS_NS = 'http://search.yahoo.com/mrss/';
- const LIMIT_TITLE = 140;
-
- public function stringify()
+ public function render(): string
{
+ $document = new \DomDocument('1.0', 'UTF-8');
+ $document->formatOutput = true;
+
$feedUrl = get_current_url();
- $extraInfos = $this->getExtraInfos();
- if (empty($extraInfos['uri'])) {
- $uri = REPOSITORY;
- } else {
- $uri = $extraInfos['uri'];
- }
-
- $document = new \DomDocument('1.0', $this->getCharset());
- $document->formatOutput = true;
$feed = $document->createElementNS(self::ATOM_NS, 'feed');
$document->appendChild($feed);
$feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:media', self::MRSS_NS);
- $title = $document->createElement('title');
- $feed->appendChild($title);
- $title->setAttribute('type', 'text');
- $title->appendChild($document->createTextNode($extraInfos['name']));
+ $feedArray = $this->getFeed();
+ foreach ($feedArray as $feedKey => $feedValue) {
+ if (in_array($feedKey, ['donationUri'])) {
+ continue;
+ }
+ if ($feedKey === 'name') {
+ $title = $document->createElement('title');
+ $feed->appendChild($title);
+ $title->setAttribute('type', 'text');
+ $title->appendChild($document->createTextNode($feedValue));
+ } elseif ($feedKey === 'icon') {
+ if ($feedValue) {
+ $icon = $document->createElement('icon');
+ $feed->appendChild($icon);
+ $icon->appendChild($document->createTextNode($feedValue));
+
+ $logo = $document->createElement('logo');
+ $feed->appendChild($logo);
+ $logo->appendChild($document->createTextNode($feedValue));
+ }
+ } elseif ($feedKey === 'uri') {
+ if ($feedValue) {
+ $linkAlternate = $document->createElement('link');
+ $feed->appendChild($linkAlternate);
+ $linkAlternate->setAttribute('rel', 'alternate');
+ $linkAlternate->setAttribute('type', 'text/html');
+ $linkAlternate->setAttribute('href', $feedValue);
+
+ $linkSelf = $document->createElement('link');
+ $feed->appendChild($linkSelf);
+ $linkSelf->setAttribute('rel', 'self');
+ $linkSelf->setAttribute('type', 'application/atom+xml');
+ $linkSelf->setAttribute('href', $feedUrl);
+ }
+ } elseif ($feedKey === 'itunes') {
+ // todo: skip?
+ } else {
+ $element = $document->createElement($feedKey);
+ $feed->appendChild($element);
+ $element->appendChild($document->createTextNode($feedValue));
+ }
+ }
$id = $document->createElement('id');
$feed->appendChild($id);
$id->appendChild($document->createTextNode($feedUrl));
- $uriparts = parse_url($uri);
- if (empty($extraInfos['icon'])) {
- $iconUrl = $uriparts['scheme'] . '://' . $uriparts['host'] . '/favicon.ico';
- } else {
- $iconUrl = $extraInfos['icon'];
- }
- $icon = $document->createElement('icon');
- $feed->appendChild($icon);
- $icon->appendChild($document->createTextNode($iconUrl));
-
- $logo = $document->createElement('logo');
- $feed->appendChild($logo);
- $logo->appendChild($document->createTextNode($iconUrl));
-
- $feedTimestamp = gmdate(DATE_ATOM, $this->lastModified);
$updated = $document->createElement('updated');
$feed->appendChild($updated);
- $updated->appendChild($document->createTextNode($feedTimestamp));
+ $updated->appendChild($document->createTextNode(gmdate(DATE_ATOM, $this->lastModified)));
// since we can't guarantee that all items have an author,
// a global feed author is mandatory
@@ -70,19 +85,8 @@ class AtomFormat extends FormatAbstract
$author->appendChild($authorName);
$authorName->appendChild($document->createTextNode($feedAuthor));
- $linkAlternate = $document->createElement('link');
- $feed->appendChild($linkAlternate);
- $linkAlternate->setAttribute('rel', 'alternate');
- $linkAlternate->setAttribute('type', 'text/html');
- $linkAlternate->setAttribute('href', $uri);
-
- $linkSelf = $document->createElement('link');
- $feed->appendChild($linkSelf);
- $linkSelf->setAttribute('rel', 'self');
- $linkSelf->setAttribute('type', 'application/atom+xml');
- $linkSelf->setAttribute('href', $feedUrl);
-
foreach ($this->getItems() as $item) {
+ $itemArray = $item->toArray();
$entryTimestamp = $item->getTimestamp();
$entryTitle = $item->getTitle();
$entryContent = $item->getContent();
@@ -103,14 +107,10 @@ class AtomFormat extends FormatAbstract
$entryID = 'urn:sha1:' . hash('sha1', $entryTitle . $entryContent);
}
- if (empty($entryTimestamp)) {
- $entryTimestamp = $this->lastModified;
- }
-
if (empty($entryTitle)) {
$entryTitle = str_replace("\n", ' ', strip_tags($entryContent));
- if (strlen($entryTitle) > self::LIMIT_TITLE) {
- $wrapPos = strpos(wordwrap($entryTitle, self::LIMIT_TITLE), "\n");
+ if (strlen($entryTitle) > 140) {
+ $wrapPos = strpos(wordwrap($entryTitle, 140), "\n");
$entryTitle = substr($entryTitle, 0, $wrapPos) . '...';
}
}
@@ -127,20 +127,37 @@ class AtomFormat extends FormatAbstract
$title->setAttribute('type', 'html');
$title->appendChild($document->createTextNode($entryTitle));
- $entryTimestamp = gmdate(\DATE_ATOM, $entryTimestamp);
- $published = $document->createElement('published');
- $entry->appendChild($published);
- $published->appendChild($document->createTextNode($entryTimestamp));
+ if ($entryTimestamp) {
+ $timestamp = gmdate(\DATE_ATOM, $entryTimestamp);
- $updated = $document->createElement('updated');
- $entry->appendChild($updated);
- $updated->appendChild($document->createTextNode($entryTimestamp));
+ $published = $document->createElement('published');
+ $entry->appendChild($published);
+ $published->appendChild($document->createTextNode($timestamp));
+
+ $updated = $document->createElement('updated');
+ $entry->appendChild($updated);
+ $updated->appendChild($document->createTextNode($timestamp));
+ }
$id = $document->createElement('id');
$entry->appendChild($id);
$id->appendChild($document->createTextNode($entryID));
- if (!empty($entryUri)) {
+ if (isset($itemArray['itunes'])) {
+ $feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:itunes', self::ITUNES_NS);
+ foreach ($itemArray['itunes'] as $itunesKey => $itunesValue) {
+ $itunesProperty = $document->createElementNS(self::ITUNES_NS, $itunesKey);
+ $entry->appendChild($itunesProperty);
+ $itunesProperty->appendChild($document->createTextNode($itunesValue));
+ }
+ if (isset($itemArray['enclosure'])) {
+ $itunesEnclosure = $document->createElement('enclosure');
+ $entry->appendChild($itunesEnclosure);
+ $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']);
+ $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']);
+ $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']);
+ }
+ } elseif (!empty($entryUri)) {
$entryLinkAlternate = $document->createElement('link');
$entry->appendChild($entryLinkAlternate);
$entryLinkAlternate->setAttribute('rel', 'alternate');
@@ -158,7 +175,7 @@ class AtomFormat extends FormatAbstract
$content = $document->createElement('content');
$content->setAttribute('type', 'html');
- $content->appendChild($document->createTextNode(break_annoying_html_tags($entryContent)));
+ $content->appendChild($document->createTextNode($entryContent));
$entry->appendChild($content);
foreach ($item->getEnclosures() as $enclosure) {
@@ -182,11 +199,7 @@ class AtomFormat extends FormatAbstract
}
}
- $toReturn = $document->saveXML();
-
- // Remove invalid characters
- ini_set('mbstring.substitute_character', 'none');
- $toReturn = mb_convert_encoding($toReturn, $this->getCharset(), 'UTF-8');
- return $toReturn;
+ $xml = $document->saveXML();
+ return $xml;
}
}
diff --git a/formats/HtmlFormat.php b/formats/HtmlFormat.php
index 9ee4aeea..04721ae1 100644
--- a/formats/HtmlFormat.php
+++ b/formats/HtmlFormat.php
@@ -4,40 +4,38 @@ class HtmlFormat extends FormatAbstract
{
const MIME_TYPE = 'text/html';
- public function stringify()
+ public function render(): string
{
- $extraInfos = $this->getExtraInfos();
+ // This query string is url encoded
+ $queryString = $_SERVER['QUERY_STRING'];
+
+ // TODO: this should be the proper bridge short name and not user provided string
+ $bridgeName = $_GET['bridge'];
+
+ $feedArray = $this->getFeed();
$formatFactory = new FormatFactory();
- $buttons = [];
- $linkTags = [];
- foreach ($formatFactory->getFormatNames() as $format) {
- // Dynamically build buttons for all formats (except HTML)
- if ($format === 'Html') {
+ $formats = [];
+
+ // Create all formats (except HTML)
+ $formatNames = $formatFactory->getFormatNames();
+ foreach ($formatNames as $formatName) {
+ if ($formatName === 'Html') {
continue;
}
- $formatUrl = '?' . str_ireplace('format=Html', 'format=' . $format, htmlentities($_SERVER['QUERY_STRING']));
- $buttons[] = [
- 'href' => $formatUrl,
- 'value' => $format,
- ];
- $linkTags[] = [
- 'href' => $formatUrl,
- 'title' => $format,
- 'type' => $formatFactory->create($format)->getMimeType(),
- ];
- }
-
- if (Configuration::getConfig('admin', 'donations') && $extraInfos['donationUri'] !== '') {
- $buttons[] = [
- 'href' => e($extraInfos['donationUri']),
- 'value' => 'Donate to maintainer',
+ // The format url is relative, but should be absolute in order to help feed readers.
+ $formatUrl = '?' . str_ireplace('format=Html', 'format=' . $formatName, $queryString);
+ $formatObject = $formatFactory->create($formatName);
+ $formats[] = [
+ 'url' => $formatUrl,
+ 'name' => $formatName,
+ 'type' => $formatObject->getMimeType(),
];
}
$items = [];
foreach ($this->getItems() as $item) {
$items[] = [
- 'url' => $item->getURI() ?: $extraInfos['uri'],
+ 'url' => $item->getURI() ?: $feedArray['uri'],
'title' => $item->getTitle() ?? '(no title)',
'timestamp' => $item->getTimestamp(),
'author' => $item->getAuthor(),
@@ -47,16 +45,19 @@ class HtmlFormat extends FormatAbstract
];
}
+ $donationUri = null;
+ if (Configuration::getConfig('admin', 'donations') && $feedArray['donationUri']) {
+ $donationUri = $feedArray['donationUri'];
+ }
+
$html = render_template(__DIR__ . '/../templates/html-format.html.php', [
- 'charset' => $this->getCharset(),
- 'title' => $extraInfos['name'],
- 'linkTags' => $linkTags,
- 'uri' => $extraInfos['uri'],
- 'buttons' => $buttons,
- 'items' => $items,
+ 'bridge_name' => $bridgeName,
+ 'title' => $feedArray['name'],
+ 'formats' => $formats,
+ 'uri' => $feedArray['uri'],
+ 'items' => $items,
+ 'donation_uri' => $donationUri,
]);
- // Remove invalid characters
- ini_set('mbstring.substitute_character', 'none');
- return mb_convert_encoding($html, $this->getCharset(), 'UTF-8');
+ return $html;
}
}
diff --git a/formats/JsonFormat.php b/formats/JsonFormat.php
index 5bb1a525..3548ef6e 100644
--- a/formats/JsonFormat.php
+++ b/formats/JsonFormat.php
@@ -23,20 +23,20 @@ class JsonFormat extends FormatAbstract
'uid',
];
- public function stringify()
+ public function render(): string
{
- $host = $_SERVER['HTTP_HOST'] ?? '';
- $extraInfos = $this->getExtraInfos();
+ $feedArray = $this->getFeed();
+
$data = [
- 'version' => 'https://jsonfeed.org/version/1',
- 'title' => empty($extraInfos['name']) ? $host : $extraInfos['name'],
- 'home_page_url' => empty($extraInfos['uri']) ? REPOSITORY : $extraInfos['uri'],
- 'feed_url' => get_current_url(),
+ 'version' => 'https://jsonfeed.org/version/1',
+ 'title' => $feedArray['name'],
+ 'home_page_url' => $feedArray['uri'],
+ 'feed_url' => get_current_url(),
];
- if (!empty($extraInfos['icon'])) {
- $data['icon'] = $extraInfos['icon'];
- $data['favicon'] = $extraInfos['icon'];
+ if ($feedArray['icon']) {
+ $data['icon'] = $feedArray['icon'];
+ $data['favicon'] = $feedArray['icon'];
}
$items = [];
@@ -47,7 +47,7 @@ class JsonFormat extends FormatAbstract
$entryTitle = $item->getTitle();
$entryUri = $item->getURI();
$entryTimestamp = $item->getTimestamp();
- $entryContent = $item->getContent() ? break_annoying_html_tags($item->getContent()) : '';
+ $entryContent = $item->getContent() ?? '';
$entryEnclosures = $item->getEnclosures();
$entryCategories = $item->getCategories();
@@ -110,13 +110,8 @@ class JsonFormat extends FormatAbstract
}
$data['items'] = $items;
- /**
- * The intention here is to discard non-utf8 byte sequences.
- * But the JSON_PARTIAL_OUTPUT_ON_ERROR also discards lots of other errors.
- * So consider this a hack.
- * Switch to JSON_INVALID_UTF8_IGNORE when PHP 7.2 is the latest platform requirement.
- */
- $json = json_encode($data, \JSON_PRETTY_PRINT | \JSON_PARTIAL_OUTPUT_ON_ERROR);
+ // Ignoring invalid json
+ $json = json_encode($data, \JSON_PRETTY_PRINT | \JSON_INVALID_UTF8_IGNORE);
return $json;
}
diff --git a/formats/MrssFormat.php b/formats/MrssFormat.php
index 3c3dce5a..4066ec73 100644
--- a/formats/MrssFormat.php
+++ b/formats/MrssFormat.php
@@ -32,24 +32,11 @@ class MrssFormat extends FormatAbstract
protected const ATOM_NS = 'http://www.w3.org/2005/Atom';
protected const MRSS_NS = 'http://search.yahoo.com/mrss/';
- const ALLOWED_IMAGE_EXT = [
- '.gif',
- '.jpg',
- '.png',
- ];
-
- public function stringify()
+ public function render(): string
{
- $feedUrl = get_current_url();
- $extraInfos = $this->getExtraInfos();
- if (empty($extraInfos['uri'])) {
- $uri = REPOSITORY;
- } else {
- $uri = $extraInfos['uri'];
- }
-
- $document = new \DomDocument('1.0', $this->getCharset());
+ $document = new \DomDocument('1.0', 'UTF-8');
$document->formatOutput = true;
+
$feed = $document->createElement('rss');
$document->appendChild($feed);
$feed->setAttribute('version', '2.0');
@@ -59,63 +46,86 @@ class MrssFormat extends FormatAbstract
$channel = $document->createElement('channel');
$feed->appendChild($channel);
- $title = $extraInfos['name'];
- $channelTitle = $document->createElement('title');
- $channel->appendChild($channelTitle);
- $channelTitle->appendChild($document->createTextNode($title));
+ $feedArray = $this->getFeed();
+ $uri = $feedArray['uri'];
+ $title = $feedArray['name'];
- $link = $document->createElement('link');
- $channel->appendChild($link);
- $link->appendChild($document->createTextNode($uri));
+ foreach ($feedArray as $feedKey => $feedValue) {
+ if (in_array($feedKey, ['atom', 'donationUri'])) {
+ continue;
+ }
+ if ($feedKey === 'name') {
+ $channelTitle = $document->createElement('title');
+ $channel->appendChild($channelTitle);
+ $channelTitle->appendChild($document->createTextNode($title));
- $description = $document->createElement('description');
- $channel->appendChild($description);
- $description->appendChild($document->createTextNode($extraInfos['name']));
+ $description = $document->createElement('description');
+ $channel->appendChild($description);
+ $description->appendChild($document->createTextNode($title));
+ } elseif ($feedKey === 'uri') {
+ $link = $document->createElement('link');
+ $channel->appendChild($link);
+ $link->appendChild($document->createTextNode($uri));
- $icon = $extraInfos['icon'];
- if (!empty($icon) && in_array(substr($icon, -4), self::ALLOWED_IMAGE_EXT)) {
- $feedImage = $document->createElement('image');
- $channel->appendChild($feedImage);
- $iconUrl = $document->createElement('url');
- $iconUrl->appendChild($document->createTextNode($icon));
- $feedImage->appendChild($iconUrl);
- $iconTitle = $document->createElement('title');
- $iconTitle->appendChild($document->createTextNode($title));
- $feedImage->appendChild($iconTitle);
- $iconLink = $document->createElement('link');
- $iconLink->appendChild($document->createTextNode($uri));
- $feedImage->appendChild($iconLink);
+ $linkAlternate = $document->createElementNS(self::ATOM_NS, 'link');
+ $channel->appendChild($linkAlternate);
+ $linkAlternate->setAttribute('rel', 'alternate');
+ $linkAlternate->setAttribute('type', 'text/html');
+ $linkAlternate->setAttribute('href', $uri);
+
+ $linkSelf = $document->createElementNS(self::ATOM_NS, 'link');
+ $channel->appendChild($linkSelf);
+ $linkSelf->setAttribute('rel', 'self');
+ $linkSelf->setAttribute('type', 'application/atom+xml');
+ $feedUrl = get_current_url();
+ $linkSelf->setAttribute('href', $feedUrl);
+ } elseif ($feedKey === 'icon') {
+ $icon = $feedValue;
+ if ($icon) {
+ $feedImage = $document->createElement('image');
+ $channel->appendChild($feedImage);
+ $iconUrl = $document->createElement('url');
+ $iconUrl->appendChild($document->createTextNode($icon));
+ $feedImage->appendChild($iconUrl);
+ $iconTitle = $document->createElement('title');
+ $iconTitle->appendChild($document->createTextNode($title));
+ $feedImage->appendChild($iconTitle);
+ $iconLink = $document->createElement('link');
+ $iconLink->appendChild($document->createTextNode($uri));
+ $feedImage->appendChild($iconLink);
+ }
+ } elseif ($feedKey === 'itunes') {
+ $feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:itunes', self::ITUNES_NS);
+ foreach ($feedValue as $itunesKey => $itunesValue) {
+ $itunesProperty = $document->createElementNS(self::ITUNES_NS, $itunesKey);
+ $channel->appendChild($itunesProperty);
+ $itunesProperty->appendChild($document->createTextNode($itunesValue));
+ }
+ } else {
+ $element = $document->createElement($feedKey);
+ $channel->appendChild($element);
+ $element->appendChild($document->createTextNode($feedValue));
+ }
}
- $linkAlternate = $document->createElementNS(self::ATOM_NS, 'link');
- $channel->appendChild($linkAlternate);
- $linkAlternate->setAttribute('rel', 'alternate');
- $linkAlternate->setAttribute('type', 'text/html');
- $linkAlternate->setAttribute('href', $uri);
-
- $linkSelf = $document->createElementNS(self::ATOM_NS, 'link');
- $channel->appendChild($linkSelf);
- $linkSelf->setAttribute('rel', 'self');
- $linkSelf->setAttribute('type', 'application/atom+xml');
- $linkSelf->setAttribute('href', $feedUrl);
-
foreach ($this->getItems() as $item) {
+ $itemArray = $item->toArray();
$itemTimestamp = $item->getTimestamp();
$itemTitle = $item->getTitle();
$itemUri = $item->getURI();
- $itemContent = $item->getContent() ? break_annoying_html_tags($item->getContent()) : '';
- $entryID = $item->getUid();
+ $itemContent = $item->getContent() ?? '';
+ $itemUid = $item->getUid();
$isPermaLink = 'false';
- if (empty($entryID) && !empty($itemUri)) {
+ if (empty($itemUid) && !empty($itemUri)) {
// Fallback to provided URI
- $entryID = $itemUri;
+ $itemUid = $itemUri;
$isPermaLink = 'true';
}
- if (empty($entryID)) {
+ if (empty($itemUid)) {
// Fallback to title and content
- $entryID = hash('sha1', $itemTitle . $itemContent);
+ $itemUid = hash('sha1', $itemTitle . $itemContent);
}
$entry = $document->createElement('item');
@@ -127,6 +137,23 @@ class MrssFormat extends FormatAbstract
$entryTitle->appendChild($document->createTextNode($itemTitle));
}
+ if (isset($itemArray['itunes'])) {
+ $feed->setAttributeNS('http://www.w3.org/2000/xmlns/', 'xmlns:itunes', self::ITUNES_NS);
+ foreach ($itemArray['itunes'] as $itunesKey => $itunesValue) {
+ $itunesProperty = $document->createElementNS(self::ITUNES_NS, $itunesKey);
+ $entry->appendChild($itunesProperty);
+ $itunesProperty->appendChild($document->createTextNode($itunesValue));
+ }
+
+ if (isset($itemArray['enclosure'])) {
+ $itunesEnclosure = $document->createElement('enclosure');
+ $entry->appendChild($itunesEnclosure);
+ $itunesEnclosure->setAttribute('url', $itemArray['enclosure']['url']);
+ $itunesEnclosure->setAttribute('length', $itemArray['enclosure']['length']);
+ $itunesEnclosure->setAttribute('type', $itemArray['enclosure']['type']);
+ }
+ }
+
if (!empty($itemUri)) {
$entryLink = $document->createElement('link');
$entry->appendChild($entryLink);
@@ -136,7 +163,7 @@ class MrssFormat extends FormatAbstract
$entryGuid = $document->createElement('guid');
$entryGuid->setAttribute('isPermaLink', $isPermaLink);
$entry->appendChild($entryGuid);
- $entryGuid->appendChild($document->createTextNode($entryID));
+ $entryGuid->appendChild($document->createTextNode($itemUid));
if (!empty($itemTimestamp)) {
$entryPublished = $document->createElement('pubDate');
@@ -164,11 +191,7 @@ class MrssFormat extends FormatAbstract
}
}
- $toReturn = $document->saveXML();
-
- // Remove invalid non-UTF8 characters
- ini_set('mbstring.substitute_character', 'none');
- $toReturn = mb_convert_encoding($toReturn, $this->getCharset(), 'UTF-8');
- return $toReturn;
+ $xml = $document->saveXML();
+ return $xml;
}
}
diff --git a/formats/PlaintextFormat.php b/formats/PlaintextFormat.php
index c8c4e9d6..e93c94b5 100644
--- a/formats/PlaintextFormat.php
+++ b/formats/PlaintextFormat.php
@@ -4,20 +4,13 @@ class PlaintextFormat extends FormatAbstract
{
const MIME_TYPE = 'text/plain';
- public function stringify()
+ public function render(): string
{
- $items = $this->getItems();
- $data = [];
-
- foreach ($items as $item) {
- $data[] = $item->toArray();
+ $feed = $this->getFeed();
+ foreach ($this->getItems() as $item) {
+ $feed['items'][] = $item->toArray();
}
-
- $toReturn = print_r($data, true);
-
- // Remove invalid non-UTF8 characters
- ini_set('mbstring.substitute_character', 'none');
- $toReturn = mb_convert_encoding($toReturn, $this->getCharset(), 'UTF-8');
- return $toReturn;
+ $text = print_r($feed, true);
+ return $text;
}
}
diff --git a/formats/SfeedFormat.php b/formats/SfeedFormat.php
index 85d5f608..063e4543 100644
--- a/formats/SfeedFormat.php
+++ b/formats/SfeedFormat.php
@@ -4,6 +4,31 @@ class SfeedFormat extends FormatAbstract
{
const MIME_TYPE = 'text/plain';
+ public function render(): string
+ {
+ $text = '';
+ foreach ($this->getItems() as $item) {
+ $text .= sprintf(
+ "%s\t%s\t%s\t%s\thtml\t\t%s\t%s\t%s\n",
+ $item->toArray()['timestamp'],
+ preg_replace('/\s/', ' ', $item->toArray()['title']),
+ $item->toArray()['uri'],
+ $this->escape($item->toArray()['content']),
+ $item->toArray()['author'],
+ $this->getFirstEnclosure(
+ $item->toArray()['enclosures']
+ ),
+ $this->escape(
+ $this->getCategories(
+ $item->toArray()['categories']
+ )
+ )
+ );
+ }
+
+ return $text;
+ }
+
private function escape(string $str)
{
$str = str_replace('\\', '\\\\', $str);
@@ -31,39 +56,4 @@ class SfeedFormat extends FormatAbstract
}
return $toReturn;
}
-
- public function stringify()
- {
- $items = $this->getItems();
-
- $toReturn = '';
- foreach ($items as $item) {
- $toReturn .= sprintf(
- "%s\t%s\t%s\t%s\thtml\t\t%s\t%s\t%s\n",
- $item->toArray()['timestamp'],
- preg_replace('/\s/', ' ', $item->toArray()['title']),
- $item->toArray()['uri'],
- $this->escape($item->toArray()['content']),
- $item->toArray()['author'],
- $this->getFirstEnclosure(
- $item->toArray()['enclosures']
- ),
- $this->escape(
- $this->getCategories(
- $item->toArray()['categories']
- )
- )
- );
- }
-
- // Remove invalid non-UTF8 characters
- ini_set('mbstring.substitute_character', 'none');
- $toReturn = mb_convert_encoding(
- $toReturn,
- $this->getCharset(),
- 'UTF-8'
- );
- return $toReturn;
- }
}
-// vi: expandtab
diff --git a/index.php b/index.php
index 538f1c6e..79403e11 100644
--- a/index.php
+++ b/index.php
@@ -1,7 +1,75 @@
main($argv ?? []);
+require __DIR__ . '/lib/bootstrap.php';
+require __DIR__ . '/lib/config.php';
+
+$container = require __DIR__ . '/lib/dependencies.php';
+
+$logger = $container['logger'];
+
+set_exception_handler(function (\Throwable $e) use ($logger) {
+ $response = new Response(render(__DIR__ . '/templates/exception.html.php', ['e' => $e]), 500);
+ $response->send();
+ $logger->error('Uncaught Exception', ['e' => $e]);
+});
+
+set_error_handler(function ($code, $message, $file, $line) use ($logger) {
+ // Consider: ini_set('error_reporting', E_ALL & ~E_DEPRECATED);
+ if ((error_reporting() & $code) === 0) {
+ // Deprecation messages and other masked errors are typically ignored here
+ return false;
+ }
+ if (Configuration::getConfig('system', 'env') === 'dev') {
+ // This might be annoying, but it's for the greater good
+ throw new \ErrorException($message, 0, $code, $file, $line);
+ }
+ $text = sprintf(
+ '%s at %s line %s',
+ sanitize_root($message),
+ sanitize_root($file),
+ $line
+ );
+ $logger->warning($text);
+ // todo: return false to prevent default error handler from running?
+});
+
+// There might be some fatal errors which are not caught by set_error_handler() or \Throwable.
+register_shutdown_function(function () use ($logger) {
+ $error = error_get_last();
+ if ($error) {
+ $message = sprintf(
+ '(shutdown) %s: %s in %s line %s',
+ $error['type'],
+ sanitize_root($error['message']),
+ sanitize_root($error['file']),
+ $error['line']
+ );
+ $logger->error($message);
+ }
+});
+
+date_default_timezone_set(Configuration::getConfig('system', 'timezone'));
+
+$argv = $argv ?? null;
+if ($argv) {
+ parse_str(implode('&', array_slice($argv, 1)), $cliArgs);
+ $request = Request::fromCli($cliArgs);
+} else {
+ $request = Request::fromGlobals();
+}
+
+$rssBridge = new RssBridge($container);
+
+$response = $rssBridge->main($request);
+
+$response->send();
\ No newline at end of file
diff --git a/lib/ActionInterface.php b/lib/ActionInterface.php
index 4eb9cc65..d2e1c709 100644
--- a/lib/ActionInterface.php
+++ b/lib/ActionInterface.php
@@ -1,28 +1,6 @@
exit('Access token is not set in this instance', 403);
- }
-
- if (isset($request['access_token'])) {
- $accessTokenGiven = $request['access_token'];
- } else {
- $header = trim($_SERVER['HTTP_AUTHORIZATION'] ?? '');
- $position = strrpos($header, 'Bearer ');
-
- if ($position !== false) {
- $accessTokenGiven = substr($header, $position + 7);
- } else {
- $accessTokenGiven = '';
- }
- }
-
- if (!$accessTokenGiven) {
- $this->exit('No access token given', 403);
- }
-
- if ($accessTokenGiven != $accessTokenInConfig) {
- $this->exit('Incorrect access token', 403);
- }
- }
-
- private function exit($message, $code)
- {
- http_response_code($code);
- header('content-type: text/plain');
- die($message);
- }
-}
diff --git a/lib/AuthenticationMiddleware.php b/lib/AuthenticationMiddleware.php
deleted file mode 100644
index c77e1b91..00000000
--- a/lib/AuthenticationMiddleware.php
+++ /dev/null
@@ -1,49 +0,0 @@
-renderAuthenticationDialog();
- exit;
- }
- if (
- Configuration::getConfig('authentication', 'username') === $user
- && Configuration::getConfig('authentication', 'password') === $password
- ) {
- return;
- }
- print $this->renderAuthenticationDialog();
- exit;
- }
-
- private function renderAuthenticationDialog(): string
- {
- http_response_code(401);
- header('WWW-Authenticate: Basic realm="RSS-Bridge"');
- return render('access-denied.html.php');
- }
-}
diff --git a/lib/BridgeAbstract.php b/lib/BridgeAbstract.php
index f024393d..f08d201f 100644
--- a/lib/BridgeAbstract.php
+++ b/lib/BridgeAbstract.php
@@ -1,78 +1,24 @@
'Maximum number of items to return',
];
- /**
- * Holds the list of items collected by the bridge
- *
- * Items must be collected by {@see BridgeInterface::collectData()}
- *
- * Use {@see BridgeAbstract::getItems()} to access items.
- *
- * @var array
- */
protected array $items = [];
-
- /**
- * Holds the list of input parameters used by the bridge
- *
- * Do not access this parameter directly!
- * Use {@see BridgeAbstract::setInputs()} and {@see BridgeAbstract::getInput()} instead!
- *
- * @var array
- */
protected array $inputs = [];
+ protected ?string $queriedContext = '';
+ private array $configuration = [];
+
+ protected CacheInterface $cache;
+ protected Logger $logger;
+
+ public function __construct(
+ CacheInterface $cache,
+ Logger $logger
+ ) {
+ $this->cache = $cache;
+ $this->logger = $logger;
+ }
+
+ abstract public function collectData();
+
+ public function getFeed(): array
+ {
+ return [
+ 'name' => $this->getName(),
+ 'uri' => $this->getURI(),
+ 'donationUri' => $this->getDonationURI(),
+ 'icon' => $this->getIcon(),
+ ];
+ }
+
+ public function getName()
+ {
+ return static::NAME ?? $this->getShortName();
+ }
+
+ public function getURI()
+ {
+ return static::URI ?? 'https://github.com/RSS-Bridge/rss-bridge/';
+ }
+
+ public function getDonationURI(): string
+ {
+ return static::DONATION_URI;
+ }
+
+ public function getIcon()
+ {
+ if (static::URI) {
+ // This favicon may or may not exist
+ return rtrim(static::URI, '/') . '/favicon.ico';
+ }
+ return '';
+ }
+
+ public function getOption(string $name)
+ {
+ return $this->configuration[$name] ?? null;
+ }
/**
- * Holds the name of the queried context
- *
- * @var string
+ * The description is only used in bridge card rendering on frontpage
*/
- protected $queriedContext = '';
+ public function getDescription()
+ {
+ return static::DESCRIPTION;
+ }
+
+ public function getMaintainer(): string
+ {
+ return static::MAINTAINER;
+ }
+
+ /**
+ * A more correct method name would have been "getContexts"
+ */
+ public function getParameters(): array
+ {
+ return static::PARAMETERS;
+ }
- /** {@inheritdoc} */
public function getItems()
{
return $this->items;
}
- /**
- * Sets the input values for a given context.
- *
- * @param array $inputs Associative array of inputs
- * @param string $queriedContext The context name
- * @return void
- */
- protected function setInputs(array $inputs, $queriedContext)
+ public function getCacheTimeout()
+ {
+ return static::CACHE_TIMEOUT;
+ }
+
+ public function loadConfiguration()
+ {
+ foreach (static::CONFIGURATION as $optionName => $optionValue) {
+ $section = $this->getShortName();
+ $configurationOption = Configuration::getConfig($section, $optionName);
+
+ if ($configurationOption !== null) {
+ $this->configuration[$optionName] = $configurationOption;
+ continue;
+ }
+
+ if (isset($optionValue['required']) && $optionValue['required'] === true) {
+ throw new \Exception(sprintf('Missing configuration option: %s', $optionName));
+ } elseif (isset($optionValue['defaultValue'])) {
+ $this->configuration[$optionName] = $optionValue['defaultValue'];
+ }
+ }
+ }
+
+ public function setInput(array $input)
+ {
+ // This is the submitted context
+ $contextName = $input['context'] ?? null;
+ if ($contextName) {
+ // Context hinting (optional)
+ $this->queriedContext = $contextName;
+ unset($input['context']);
+ }
+
+ $contexts = $this->getParameters();
+
+ if (!$contexts) {
+ if ($input) {
+ throw new \Exception('Invalid parameters value(s)');
+ }
+ return;
+ }
+
+ $validator = new ParameterValidator();
+
+ // $input IS PASSED BY REFERENCE!
+ $errors = $validator->validateInput($input, $contexts);
+ if ($errors !== []) {
+ $invalidParameterKeys = array_column($errors, 'name');
+ throw new \Exception(sprintf('Invalid parameters value(s): %s', implode(', ', $invalidParameterKeys)));
+ }
+
+ // Guess the context from input data
+ if (empty($this->queriedContext)) {
+ $queriedContext = $validator->getQueriedContext($input, $contexts);
+ $this->queriedContext = $queriedContext;
+ }
+
+ if (is_null($this->queriedContext)) {
+ throw new \Exception('Required parameter(s) missing');
+ } elseif ($this->queriedContext === false) {
+ throw new \Exception('Mixed context parameters');
+ }
+
+ $this->setInputWithContext($input, $this->queriedContext);
+ }
+
+ private function setInputWithContext(array $input, $queriedContext)
{
// Import and assign all inputs to their context
- foreach ($inputs as $name => $value) {
- foreach (static::PARAMETERS as $context => $set) {
- if (array_key_exists($name, static::PARAMETERS[$context])) {
+ foreach ($input as $name => $value) {
+ foreach ($this->getParameters() as $context => $set) {
+ if (array_key_exists($name, $this->getParameters()[$context])) {
$this->inputs[$context][$name]['value'] = $value;
}
}
}
// Apply default values to missing data
- $contexts = [$queriedContext];
- if (array_key_exists('global', static::PARAMETERS)) {
- $contexts[] = 'global';
+ $contextNames = [$queriedContext];
+ if (array_key_exists('global', $this->getParameters())) {
+ $contextNames[] = 'global';
}
- foreach ($contexts as $context) {
- foreach (static::PARAMETERS[$context] as $name => $properties) {
+ foreach ($contextNames as $context) {
+ if (!isset($this->getParameters()[$context])) {
+ // unknown context provided by client, throw exception here? or continue?
+ }
+
+ foreach ($this->getParameters()[$context] as $name => $properties) {
if (isset($this->inputs[$context][$name]['value'])) {
continue;
}
@@ -153,11 +212,7 @@ abstract class BridgeAbstract implements BridgeInterface
switch ($type) {
case 'checkbox':
- if (!isset($properties['defaultValue'])) {
- $this->inputs[$context][$name]['value'] = false;
- } else {
- $this->inputs[$context][$name]['value'] = $properties['defaultValue'];
- }
+ $this->inputs[$context][$name]['value'] = $input[$context][$name]['value'] ?? false;
break;
case 'list':
if (!isset($properties['defaultValue'])) {
@@ -180,14 +235,18 @@ abstract class BridgeAbstract implements BridgeInterface
}
// Copy global parameter values to the guessed context
- if (array_key_exists('global', static::PARAMETERS)) {
- foreach (static::PARAMETERS['global'] as $name => $properties) {
- if (isset($inputs[$name])) {
- $value = $inputs[$name];
- } elseif (isset($properties['defaultValue'])) {
- $value = $properties['defaultValue'];
+ if (array_key_exists('global', $this->getParameters())) {
+ foreach ($this->getParameters()['global'] as $name => $properties) {
+ if (isset($input[$name])) {
+ $value = $input[$name];
} else {
- continue;
+ if (($properties['type'] ?? null) === 'checkbox') {
+ $value = false;
+ } elseif (isset($properties['defaultValue'])) {
+ $value = $properties['defaultValue'];
+ } else {
+ continue;
+ }
}
$this->inputs[$queriedContext][$name]['value'] = $value;
}
@@ -195,103 +254,17 @@ abstract class BridgeAbstract implements BridgeInterface
// Only keep guessed context parameters values
if (isset($this->inputs[$queriedContext])) {
- $this->inputs = [$queriedContext => $this->inputs[$queriedContext]];
+ $this->inputs = [
+ $queriedContext => $this->inputs[$queriedContext],
+ ];
} else {
$this->inputs = [];
}
}
- /**
- * Set inputs for the bridge
- *
- * Returns errors and aborts execution if the provided input parameters are
- * invalid.
- *
- * @param array List of input parameters. Each element in this list must
- * relate to an item in {@see BridgeAbstract::PARAMETERS}
- * @return void
- */
- public function setDatas(array $inputs)
- {
- if (isset($inputs['context'])) { // Context hinting (optional)
- $this->queriedContext = $inputs['context'];
- unset($inputs['context']);
- }
-
- if (empty(static::PARAMETERS)) {
- if (!empty($inputs)) {
- throw new \Exception('Invalid parameters value(s)');
- }
-
- return;
- }
-
- $validator = new ParameterValidator();
-
- if (!$validator->validateData($inputs, static::PARAMETERS)) {
- $parameters = array_map(
- function ($i) {
- return $i['name'];
- }, // Just display parameter names
- $validator->getInvalidParameters()
- );
-
- throw new \Exception(sprintf('Invalid parameters value(s): %s', implode(', ', $parameters)));
- }
-
- // Guess the context from input data
- if (empty($this->queriedContext)) {
- $this->queriedContext = $validator->getQueriedContext($inputs, static::PARAMETERS);
- }
-
- if (is_null($this->queriedContext)) {
- throw new \Exception('Required parameter(s) missing');
- } elseif ($this->queriedContext === false) {
- throw new \Exception('Mixed context parameters');
- }
-
- $this->setInputs($inputs, $this->queriedContext);
- }
-
- /**
- * Loads configuration for the bridge
- *
- * Returns errors and aborts execution if the provided configuration is
- * invalid.
- *
- * @return void
- */
- public function loadConfiguration()
- {
- foreach (static::CONFIGURATION as $optionName => $optionValue) {
- $section = $this->getShortName();
- $configurationOption = Configuration::getConfig($section, $optionName);
-
- if ($configurationOption !== null) {
- $this->configuration[$optionName] = $configurationOption;
- continue;
- }
-
- if (isset($optionValue['required']) && $optionValue['required'] === true) {
- throw new \Exception(sprintf('Missing configuration option: %s', $optionName));
- } elseif (isset($optionValue['defaultValue'])) {
- $this->configuration[$optionName] = $optionValue['defaultValue'];
- }
- }
- }
-
- /**
- * Returns the value for the provided input
- *
- * @param string $input The input name
- * @return mixed|null The input value or null if the input is not defined
- */
protected function getInput($input)
{
- if (!isset($this->inputs[$this->queriedContext][$input]['value'])) {
- return null;
- }
- return $this->inputs[$this->queriedContext][$input]['value'];
+ return $this->inputs[$this->queriedContext][$input]['value'] ?? null;
}
/**
@@ -306,17 +279,20 @@ abstract class BridgeAbstract implements BridgeInterface
if (!isset($this->inputs[$this->queriedContext][$input]['value'])) {
return null;
}
- if (array_key_exists('global', static::PARAMETERS)) {
- if (array_key_exists($input, static::PARAMETERS['global'])) {
- $context = 'global';
+
+ $contexts = $this->getParameters();
+
+ if (array_key_exists('global', $contexts)) {
+ if (array_key_exists($input, $contexts['global'])) {
+ $contextName = 'global';
}
}
- if (!isset($context)) {
- $context = $this->queriedContext;
+ if (!isset($contextName)) {
+ $contextName = $this->queriedContext;
}
$needle = $this->inputs[$this->queriedContext][$input]['value'];
- foreach (static::PARAMETERS[$context][$input]['values'] as $first_level_key => $first_level_value) {
+ foreach ($contexts[$contextName][$input]['values'] as $first_level_key => $first_level_value) {
if (!is_array($first_level_value) && $needle === (string)$first_level_value) {
return $first_level_key;
} elseif (is_array($first_level_value)) {
@@ -329,74 +305,14 @@ abstract class BridgeAbstract implements BridgeInterface
}
}
- /**
- * Get bridge configuration value
- */
- public function getOption($name)
- {
- return $this->configuration[$name] ?? null;
- }
-
- /** {@inheritdoc} */
- public function getDescription()
- {
- return static::DESCRIPTION;
- }
-
- /** {@inheritdoc} */
- public function getMaintainer()
- {
- return static::MAINTAINER;
- }
-
- /** {@inheritdoc} */
- public function getName()
- {
- return static::NAME;
- }
-
- /** {@inheritdoc} */
- public function getIcon()
- {
- return static::URI . '/favicon.ico';
- }
-
- /** {@inheritdoc} */
- public function getConfiguration()
- {
- return static::CONFIGURATION;
- }
-
- /** {@inheritdoc} */
- public function getParameters()
- {
- return static::PARAMETERS;
- }
-
- /** {@inheritdoc} */
- public function getURI()
- {
- return static::URI;
- }
-
- /** {@inheritdoc} */
- public function getDonationURI()
- {
- return static::DONATION_URI;
- }
-
- /** {@inheritdoc} */
- public function getCacheTimeout()
- {
- return static::CACHE_TIMEOUT;
- }
-
- /** {@inheritdoc} */
public function detectParameters($url)
{
$regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/';
+
+ $contexts = $this->getParameters();
+
if (
- empty(static::PARAMETERS)
+ empty($contexts)
&& preg_match($regex, $url, $urlMatches) > 0
&& preg_match($regex, static::URI, $bridgeUriMatches) > 0
&& $urlMatches[3] === $bridgeUriMatches[3]
@@ -406,43 +322,14 @@ abstract class BridgeAbstract implements BridgeInterface
return null;
}
- /**
- * Loads a cached value for the specified key
- *
- * @param int $duration Cache duration (optional)
- * @return mixed Cached value or null if the key doesn't exist or has expired
- */
- protected function loadCacheValue(string $key, $duration = null)
+ protected function loadCacheValue(string $key, $default = null)
{
- $cache = RssBridge::getCache();
- // Create class name without the namespace part
- $scope = $this->getShortName();
- $cache->setScope($scope);
- $cache->setKey([$key]);
- $timestamp = $cache->getTime();
-
- if (
- $duration
- && $timestamp
- && $timestamp < time() - $duration
- ) {
- return null;
- }
- return $cache->loadData();
+ return $this->cache->get($this->getShortName() . '_' . $key, $default);
}
- /**
- * Stores a value to cache with the specified key
- *
- * @param mixed $value Value to cache
- */
- protected function saveCacheValue(string $key, $value)
+ protected function saveCacheValue(string $key, $value, int $ttl = 86400)
{
- $cache = RssBridge::getCache();
- $scope = $this->getShortName();
- $cache->setScope($scope);
- $cache->setKey([$key]);
- $cache->saveData($value);
+ $this->cache->set($this->getShortName() . '_' . $key, $value, $ttl);
}
public function getShortName(): string
diff --git a/lib/BridgeCard.php b/lib/BridgeCard.php
index 6eef3879..2f171002 100644
--- a/lib/BridgeCard.php
+++ b/lib/BridgeCard.php
@@ -1,57 +1,34 @@
$bridgeClassName The bridge name
- * @param array $formats A list of formats
- * @param bool $isActive Indicates if the bridge is active or not
- * @return string The bridge card
- */
- public static function displayBridgeCard($bridgeClassName, $formats, $isActive = true)
- {
- $bridgeFactory = new BridgeFactory();
-
+ public static function render(
+ BridgeFactory $bridgeFactory,
+ string $bridgeClassName,
+ ?string $token
+ ): string {
$bridge = $bridgeFactory->create($bridgeClassName);
- $isHttps = strpos($bridge->getURI(), 'https') === 0;
-
$uri = $bridge->getURI();
$name = $bridge->getName();
$icon = $bridge->getIcon();
$description = $bridge->getDescription();
- $parameters = $bridge->getParameters();
- if (Configuration::getConfig('proxy', 'url') && Configuration::getConfig('proxy', 'by_bridge')) {
- $parameters['global']['_noproxy'] = [
- 'name' => 'Disable proxy (' . (Configuration::getConfig('proxy', 'name') ?: Configuration::getConfig('proxy', 'url')) . ')',
- 'type' => 'checkbox'
+ $contexts = $bridge->getParameters();
+
+ // Checkbox for disabling of proxy (if enabled)
+ if (
+ Configuration::getConfig('proxy', 'url')
+ && Configuration::getConfig('proxy', 'by_bridge')
+ ) {
+ $proxyName = Configuration::getConfig('proxy', 'name') ?: Configuration::getConfig('proxy', 'url');
+ $contexts['global']['_noproxy'] = [
+ 'name' => sprintf('Disable proxy (%s)', $proxyName),
+ 'type' => 'checkbox',
];
}
if (Configuration::getConfig('cache', 'custom_timeout')) {
- $parameters['global']['_cache_timeout'] = [
+ $contexts['global']['_cache_timeout'] = [
'name' => 'Cache timeout in seconds',
'type' => 'number',
'defaultValue' => $bridge->getCacheTimeout()
@@ -60,46 +37,56 @@ final class BridgeCard
$shortName = $bridge->getShortName();
$card = <<
+
-
',
$bridge->getMaintainer(),
@@ -113,58 +100,30 @@ CARD;
return $card;
}
- /**
- * Get the form header for a bridge card
- *
- * @param class-string $bridgeClassName The bridge name
- * @param bool $isHttps If disabled, adds a warning to the form
- * @return string The form header
- */
- private static function getFormHeader($bridgeClassName, $isHttps = false, $parameterName = '')
- {
- $form = <<
-
-
-EOD;
-
- if (!empty($parameterName)) {
- $form .= sprintf('', $parameterName);
- }
-
- if (!$isHttps) {
- $form .= '
Warning :
-This bridge is not fetching its content through a secure connection
';
- }
-
- return $form;
- }
-
- /**
- * Get the form body for a bridge
- *
- * @param class-string $bridgeClassName The bridge name
- * @param array $formats A list of supported formats
- * @param bool $isActive Indicates if a bridge is enabled or not
- * @param bool $isHttps Indicates if a bridge uses HTTPS or not
- * @param string $parameterName Sets the bridge context for the current form
- * @param array $parameters The bridge parameters
- * @return string The form body
- */
- private static function getForm(
- $bridgeClassName,
- $formats,
- $isActive = false,
- $isHttps = false,
- $parameterName = '',
- $parameters = []
+ private static function renderForm(
+ string $bridgeClassName,
+ string $contextName,
+ array $contextParameters,
+ ?string $token
) {
- $form = self::getFormHeader($bridgeClassName, $isHttps, $parameterName);
+ $form = <<
+
+
+ EOD;
- if (count($parameters) > 0) {
+ if (Configuration::getConfig('authentication', 'token') && $token) {
+ $form .= sprintf('', e($token));
+ }
+
+ if (!empty($contextName)) {
+ $form .= sprintf('', $contextName);
+ }
+
+ if (count($contextParameters) > 0) {
$form .= '
';
- foreach ($parameters as $id => $inputEntry) {
+ foreach ($contextParameters as $id => $inputEntry) {
if (!isset($inputEntry['exampleValue'])) {
$inputEntry['exampleValue'] = '';
}
@@ -173,33 +132,39 @@ This bridge is not fetching its content through a secure connection
This feed may be only one of the possible feeds. You may find more feeds using one of the bridges with different parameters, for example.
';
+ findfeedresults.innerHTML = content;
+ }
+
+ // Display an error if no feed were found
+ function rss_bridge_feed_display_feed_search_fail() {
+ const findfeedresults = document.getElementById('findfeedresults');
+ findfeedresults.innerHTML = 'No Feed found !
Not every bridge supports feed detection. You can check below within the bridge parameters to create a feed.
- RSS-Bridge tried to fetch a website.
- The fetching was blocked by CloudFlare.
- CloudFlare is anti-bot software.
- Its purpose is to block non-humans.
-
-
-
- getCode() === 404): ?>
-
The website was not found
-
- RSS-Bridge tried to fetch a page on a website.
- But it doesn't exists.
-
-
-
- getCode() === 429): ?>
-
Try again later
-
- RSS-Bridge tried to fetch a website.
- They told us to try again later.
-
-
-
-
- getCode() === 10): ?>
-
The rss feed is completely empty
-
- RSS-Bridge tried parse the empty string as xml.
- The fetched url is not pointing to real xml.
-
-
-
- getCode() === 11): ?>
-
There is something wrong with the rss feed
-
- RSS-Bridge tried parse xml. It failed. The xml is probably broken.
-
diff --git a/templates/exception.html.php b/templates/exception.html.php
new file mode 100644
index 00000000..62ac90b4
--- /dev/null
+++ b/templates/exception.html.php
@@ -0,0 +1,148 @@
+
+
+
+
+
+
The website is protected by CloudFlare
+
+ RSS-Bridge tried to fetch a website.
+ The fetching was blocked by CloudFlare.
+ CloudFlare is anti-bot software.
+ Its purpose is to block non-humans.
+
+
+
+ getCode() === 400): ?>
+
400 Bad Request
+
+ This is usually caused by an incorrectly constructed http request.
+
+
+
+ getCode() === 403): ?>
+
403 Forbidden
+
+ The HTTP 403 Forbidden response status code indicates that the
+ server understands the request but refuses to authorize it.
+
+
+
+ getCode() === 404): ?>
+
404 Page Not Found
+
+ RSS-Bridge tried to fetch a page on a website.
+ But it doesn't exists.
+
+
+
+ getCode() === 429): ?>
+
429 Too Many Requests
+
+ RSS-Bridge tried to fetch a website.
+ They told us to try again later.
+
+
+
+ getCode() === 503): ?>
+
503 Service Unavailable
+
+ Common causes are a server that is down for maintenance
+ or that is overloaded.
+
') : '';
+ $externalDescription = strlen($externalDescription) > 0 ? " . '){kind=avatar}
' : '';
+ $feedViewName = e($record['displayName'] ?? $record['name']);
+ $feedViewDescription = e($record['description'] ?? '');
+ $authorDisplayName = e($record['creator']['displayName']);
+ $authorHandle = e($record['creator']['handle']);
+ $likeCount = isset($record['likeCount']) ? '
![](\"https://cdn.thewirecutter.com/{$img_link}?width=314&quality=75&crop=3:2&auto=webp\")
"; + + $content .= "\${$deal->price}
"; + + foreach ($deal->buyButtons as $buy) { + $content .= "\${$deal->streetPrice}Buy from url}\">$buy->merchant"; + if ($buy->promo->effect) { + $content .= " {$buy->promo->effect}"; + } + if ($buy->promo->code) { + $content .= " (Use promo code {$buy->promo->code})"; + } + $content .= '
'; + } + + $content .= '
'; + $structuredContent = json_decode($deal->structuredContent, true); + foreach ($structuredContent as $node) { + $content .= $this->jsonToHtml($node); + } + + if ($deal->relatedArticle) { + $review = $deal->relatedArticle; + $content .= '
'; + $content .= "Read the review: link}\">{$review->title}
"; + } + + return $content; + } +} diff --git a/bridges/WiredBridge.php b/bridges/WiredBridge.php index d4c7cbbb..b77e3973 100644 --- a/bridges/WiredBridge.php +++ b/bridges/WiredBridge.php @@ -32,7 +32,7 @@ class WiredBridge extends FeedExpander { $feed = $this->getInput('feed'); if (empty($feed) || !ctype_alpha(str_replace('-', '', $feed))) { - returnClientError('Invalid feed, please check the "feed" parameter.'); + throwClientException('Invalid feed, please check the "feed" parameter.'); } $feed_url = $this->getURI() . 'feed/'; @@ -50,13 +50,14 @@ class WiredBridge extends FeedExpander $this->collectExpandableDatas($feed_url, $limit); } - protected function parseItem($newsItem) + protected function parseItem(array $item) { - $item = parent::parseItem($newsItem); + $originalContent = $item['content']; + $article = getSimpleHTMLDOMCached($item['uri']); $item['content'] = $this->extractArticleContent($article); - $headline = strval($newsItem->description); + $headline = $originalContent; if (!empty($headline)) { $item['content'] = '' . $headline . '
' . $item['content']; } diff --git a/bridges/WordPressBridge.php b/bridges/WordPressBridge.php index 1d46958d..723a16c4 100644 --- a/bridges/WordPressBridge.php +++ b/bridges/WordPressBridge.php @@ -20,50 +20,54 @@ class WordPressBridge extends FeedExpander ], ]]; - private function cleanContent($content) + public function collectData() { - $content = stripWithDelimiters($content, 'Poll ({$attachments->pollRenderer->totalVotes->simpleText})
{$pollChoices}
EOD; + } elseif (isset($attachments->postMultiImageRenderer->images)) { + // Multiple images + $images = $attachments->postMultiImageRenderer->images; + + if (is_array($images)) { + if (empty($this->itemTitle)) { + $this->itemTitle = $this->feedName . ' posted ' . count($images) . ' images'; + } + + foreach ($images as $image) { + $lastThumb = end($image->backstageImageRenderer->image->thumbnails); + + $content .= <<
+EOD; + } + } } } @@ -267,6 +300,7 @@ EOD; { $length = 100; + $text = strip_tags($text); if (strlen($text) > $length) { $text = explode('', wordwrap($text, $length, '
')); return $text[0] . '...'; @@ -275,12 +309,26 @@ EOD; return $text; } - private function formatUrls($content) + private function formatUrls($content, $url) { - return preg_replace( - '/(http[s]{0,1}\:\/\/[a-zA-Z0-9.\/\?\&=\-_]{4,})/ims', - '$1 ', - $content - ); + if (substr(strval($url), 0, 1) == '/') { + // fix relative URL + $url = 'https://www.youtube.com' . $url; + } elseif (substr(strval($url), 0, 33) == 'https://www.youtube.com/redirect?') { + // extract actual URL from YouTube redirect + parse_str(substr($url, 33), $params); + if (strpos(($params['q'] ?? ''), rtrim($content, '.')) === 0) { + $url = $params['q']; + } + } + + // ensure all URLs are made clickable + $url = $url ?? $content; + + if (filter_var($url, FILTER_VALIDATE_URL)) { + return '' . $content . ''; + } + + return $content; } } diff --git a/bridges/YouTubeFeedExpanderBridge.php b/bridges/YouTubeFeedExpanderBridge.php new file mode 100644 index 00000000..3811d960 --- /dev/null +++ b/bridges/YouTubeFeedExpanderBridge.php @@ -0,0 +1,81 @@ + [ + 'name' => 'Channel ID', + 'required' => true, + // Example: vinesauce + 'exampleValue' => 'UCzORJV8l3FWY4cFO8ot-F2w', + ], + 'embed' => [ + 'name' => 'Add embed to entry', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Add embed to entry', + 'defaultValue' => 'checked', + ], + 'embedurl' => [ + 'name' => 'Use embed page as entry url', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Use embed page as entry url', + ], + 'nocookie' => [ + 'name' => 'Use nocookie embed page', + 'type' => 'checkbox', + 'required' => false, + 'title' => 'Use nocookie embed page' + ], + ]]; + + public function getIcon() + { + if ($this->getInput('channel') != null) { + $html = getSimpleHTMLDOMCached($this->getURI()); + return $html->find('[itemprop="thumbnailUrl"]', 0)->href; + } + return parent::getIcon(); + } + + public function collectData() + { + $url = 'https://www.youtube.com/feeds/videos.xml?channel_id=' . $this->getInput('channel'); + $this->collectExpandableDatas($url); + } + + protected function parseItem(array $item) + { + $id = $item['yt']['videoId']; + $item['comments'] = $item['uri'] . '#comments'; + $item['uid'] = $item['id']; + + $thumbnail = sprintf('https://img.youtube.com/vi/%s/maxresdefault.jpg', $id); + $item['enclosures'] = [$thumbnail]; + + $item['content'] = $item['media']['group']['description']; + $item['content'] = str_replace("\n", '
', $item['content']); + unset($item['media']); + + $embedURI = self::URI; + if ($this->getInput('nocookie')) { + $embedURI = 'https://www.youtube-nocookie.com/'; + } + $embed = $embedURI . 'embed/' . $id; + if ($this->getInput('embed')) { + $iframe_fmt = ''; //phpcs:ignore + $iframe = sprintf($iframe_fmt, $embed, $item['title']) . '
'; + $item['content'] = $iframe . $item['content']; + } + if ($this->getInput('embedurl')) { + $item['uri'] = $embed; + } + + return $item; + } +} diff --git a/bridges/YoutubeBridge.php b/bridges/YoutubeBridge.php index 54a38d98..0c37be0c 100644 --- a/bridges/YoutubeBridge.php +++ b/bridges/YoutubeBridge.php @@ -1,17 +1,10 @@ ytGetSimpleHTMLDOM(self::URI . "watch?v=$vid", true); + $cacheKey = 'youtube_rate_limit'; + if ($this->cache->get($cacheKey)) { + throwRateLimitException(); + } + try { + $this->collectDataInternal(); + } catch (HttpException $e) { + if ($e->getCode() === 429) { + $this->cache->set($cacheKey, true, 60 * 16); + throwRateLimitException(); + } + throw $e; + } + } + + private function collectDataInternal() + { + $html = ''; + $url_feed = ''; + $url_listing = ''; + + $username = $this->getInput('u'); + $channel = $this->getInput('c'); + $custom = $this->getInput('custom'); + $playlist = $this->getInput('p'); + $search = $this->getInput('s'); + + $durationMin = $this->getInput('duration_min'); + $durationMax = $this->getInput('duration_max'); + + // Whether to discriminate videos by duration + $filterByDuration = $durationMin || $durationMax; + + if ($username) { + // user and channel + $url_feed = self::URI . '/feeds/videos.xml?user=' . urlencode($username); + $url_listing = self::URI . '/user/' . urlencode($username) . '/videos'; + } elseif ($channel) { + $url_feed = self::URI . '/feeds/videos.xml?channel_id=' . urlencode($channel); + $url_listing = self::URI . '/channel/' . urlencode($channel) . '/videos'; + } elseif ($custom) { + $url_listing = self::URI . '/' . urlencode($custom) . '/videos'; + } + + if ($url_feed || $url_listing) { + // user, channel or custom + $this->feeduri = $url_listing; + if ($custom) { + // Extract the feed url for the custom name + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + // Pluck out the rss feed url + $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl; + $this->feedIconUrl = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; + } + if ($filterByDuration) { + if (!$custom) { + // Fetch the html page + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + } + $channel_id = ''; + if (isset($jsonData->contents)) { + $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId; + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1]; + $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents; + // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; + $this->fetchItemsFromFromJsonData($jsonData); + } else { + throwServerException('Unable to get data from YouTube'); + } + } else { + // Fetch the xml feed + $html = $this->fetch($url_feed); + $this->extractItemsFromXmlFeed($html); + } + $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); + } elseif ($playlist) { + // playlist + $url_feed = self::URI . '/feeds/videos.xml?playlist_id=' . urlencode($playlist); + $url_listing = self::URI . '/playlist?list=' . urlencode($playlist); + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + // TODO: this method returns only first 100 video items + // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element + $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0] ?? null; + if (!$jsonData) { + // playlist probably doesnt exists + throw new \Exception('Unable to find playlist: ' . $url_listing); + } + $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; + $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; + $item_count = count($jsonData); + + if ($item_count > 15 || $filterByDuration) { + $this->fetchItemsFromFromJsonData($jsonData); + } else { + $xml = $this->fetch($url_feed); + $this->extractItemsFromXmlFeed($xml); + } + $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); + usort($this->items, function ($item1, $item2) { + if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) { + $item1['timestamp'] = strtotime($item1['timestamp']); + $item2['timestamp'] = strtotime($item2['timestamp']); + } + return $item2['timestamp'] - $item1['timestamp']; + }); + } elseif ($search) { + // search + $url_listing = self::URI . '/results?search_query=' . urlencode($search) . '&sp=CAI%253D'; + $html = $this->fetch($url_listing); + $jsonData = $this->extractJsonFromHtml($html); + $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; + $jsonData = $jsonData->sectionListRenderer->contents[0]->itemSectionRenderer->contents; + $this->fetchItemsFromFromJsonData($jsonData); + $this->feeduri = $url_listing; + $this->feedName = 'Search: ' . $search; + } else { + throwClientException("You must either specify either:\n - YouTube username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); + } + } + + private function fetchVideoDetails($videoId, &$author, &$description, &$timestamp) + { + $url = self::URI . "/watch?v=$videoId"; + $html = $this->fetch($url, true); // Skip unavailable videos if (strpos($html->innertext, 'IS_UNAVAILABLE_PAGE') !== false) { @@ -94,15 +213,18 @@ class YoutubeBridge extends BridgeAbstract $elDatePublished = $html->find('meta[itemprop=datePublished]', 0); if (!is_null($elDatePublished)) { - $time = strtotime($elDatePublished->getAttribute('content')); + $timestamp = strtotime($elDatePublished->getAttribute('content')); } - $jsonData = $this->getJSONData($html); - if (! isset($jsonData->contents)) { + $jsonData = $this->extractJsonFromHtml($html); + if (!isset($jsonData->contents)) { return; } - $jsonData = $jsonData->contents->twoColumnWatchNextResults->results->results->contents; + $jsonData = $jsonData->contents->twoColumnWatchNextResults->results->results->contents ?? null; + if (!$jsonData) { + throw new \Exception('Unable to find json data'); + } $videoSecondaryInfo = null; foreach ($jsonData as $item) { if (isset($item->videoSecondaryInfoRenderer)) { @@ -111,149 +233,199 @@ class YoutubeBridge extends BridgeAbstract } } if (!$videoSecondaryInfo) { - returnServerError('Could not find videoSecondaryInfoRenderer. Error at: ' . $vid); + throwServerException('Could not find videoSecondaryInfoRenderer. Error at: ' . $videoId); } - if (isset($videoSecondaryInfo->description)) { - foreach ($videoSecondaryInfo->description->runs as $description) { - if (isset($description->navigationEndpoint)) { - $metadata = $description->navigationEndpoint->commandMetadata->webCommandMetadata; - $web_type = $metadata->webPageType; - $url = $metadata->url; - $text = ''; - switch ($web_type) { - case 'WEB_PAGE_TYPE_UNKNOWN': - $url_components = parse_url($url); - if (isset($url_components['query']) && strpos($url_components['query'], '&q=') !== false) { - parse_str($url_components['query'], $params); - $url = urldecode($params['q']); - } - $text = $url; - break; - case 'WEB_PAGE_TYPE_WATCH': - case 'WEB_PAGE_TYPE_BROWSE': - $url = 'https://www.youtube.com' . $url; - $text = $description->text; - break; - } - $desc .= "$text"; - } else { - $desc .= nl2br($description->text); - } + $description = $videoSecondaryInfo->attributedDescription->content ?? ''; + + // Default whitespace chars used by trim + non-breaking spaces (https://en.wikipedia.org/wiki/Non-breaking_space) + $whitespaceChars = " \t\n\r\0\x0B\u{A0}\u{2060}\u{202F}\u{2007}"; + $descEnhancements = $this->ytBridgeGetVideoDescriptionEnhancements($videoSecondaryInfo, $description, self::URI, $whitespaceChars); + foreach ($descEnhancements as $descEnhancement) { + if (isset($descEnhancement['url'])) { + $descBefore = mb_substr($description, 0, $descEnhancement['pos']); + $descValue = mb_substr($description, $descEnhancement['pos'], $descEnhancement['len']); + $descAfter = mb_substr($description, $descEnhancement['pos'] + $descEnhancement['len'], null); + + // Extended trim for the display value of internal links, e.g.: + // FAVICON • Video Name + // FAVICON / @ChannelName + $descValue = trim($descValue, $whitespaceChars . '•/'); + + $description = sprintf('%s%s%s', $descBefore, $descEnhancement['url'], $descValue, $descAfter); } } } - private function ytBridgeAddItem($vid, $title, $author, $desc, $time, $thumbnail = '') - { - $item = []; - $item['id'] = $vid; - $item['title'] = $title; - $item['author'] = $author; - $item['timestamp'] = $time; - $item['uri'] = self::URI . 'watch?v=' . $vid; - if (!$thumbnail) { - $thumbnail = '0'; // Fallback to default thumbnail if there aren't any provided. + private function ytBridgeGetVideoDescriptionEnhancements( + object $videoSecondaryInfo, + string $descriptionContent, + string $baseUrl, + string $whitespaceChars + ): array { + $commandRuns = $videoSecondaryInfo->attributedDescription->commandRuns ?? []; + if (count($commandRuns) <= 0) { + return []; } - $thumbnailUri = str_replace('/www.', '/img.', self::URI) . 'vi/' . $vid . '/' . $thumbnail . '.jpg'; - $item['content'] = '
' . $desc; - $this->items[] = $item; + + $enhancements = []; + + $boundaryWhitespaceChars = mb_str_split($whitespaceChars); + $boundaryStartChars = array_merge($boundaryWhitespaceChars, [':', '-', '(']); + $boundaryEndChars = array_merge($boundaryWhitespaceChars, [',', '.', "'", ')']); + $hashtagBoundaryEndChars = array_merge($boundaryEndChars, ['#', '-']); + + $descriptionContentLength = mb_strlen($descriptionContent); + + $minPositionOffset = 0; + + $prevStartPosition = 0; + $totalLength = 0; + $maxPositionByStartIndex = []; + foreach (array_reverse($commandRuns) as $commandRun) { + $endPosition = $commandRun->startIndex + $commandRun->length; + if ($endPosition < $prevStartPosition) { + $totalLength += 1; + } + $totalLength += $commandRun->length; + $maxPositionByStartIndex[$commandRun->startIndex] = $totalLength; + $prevStartPosition = $commandRun->startIndex; + } + + foreach ($commandRuns as $commandRun) { + $commandMetadata = $commandRun->onTap->innertubeCommand->commandMetadata->webCommandMetadata ?? null; + if (!isset($commandMetadata)) { + continue; + } + + $enhancement = null; + + /* + $commandRun->startIndex can be offset by few positions in the positive direction + when some multibyte characters (e.g. emojis, but maybe also others) are used in the plain text video description. + (probably some difference between php and javascript in handling multibyte characters) + This loop should correct the position in most cases. It searches for the next word (determined by a set of boundary chars) with the expected length. + Several safeguards ensure that the correct word is chosen. When a link can not be matched, + everything will be discarded to prevent corrupting the description. + Hashtags require a different set of boundary chars. + */ + $isHashtag = $commandMetadata->webPageType === 'WEB_PAGE_TYPE_BROWSE'; + $prevEnhancement = end($enhancements); + $minPosition = $prevEnhancement === false ? 0 : $prevEnhancement['pos'] + $prevEnhancement['len']; + $maxPosition = $descriptionContentLength - $maxPositionByStartIndex[$commandRun->startIndex]; + $position = min($commandRun->startIndex - $minPositionOffset, $maxPosition); + while ($position >= $minPosition) { + // The link display value can only ever include a new line at the end (which will be removed further below), never in between. + $newLinePosition = mb_strpos($descriptionContent, "\n", $position); + if ($newLinePosition !== false && $newLinePosition < $position + ($commandRun->length - 1)) { + $position = $newLinePosition - ($commandRun->length - 1); + continue; + } + + $firstChar = mb_substr($descriptionContent, $position, 1); + $boundaryStart = mb_substr($descriptionContent, $position - 1, 1); + $boundaryEndIndex = $position + $commandRun->length; + $boundaryEnd = mb_substr($descriptionContent, $boundaryEndIndex, 1); + + $boundaryStartIsValid = $position === 0 || + in_array($boundaryStart, $boundaryStartChars) || + ($isHashtag && $firstChar === '#'); + $boundaryEndIsValid = $boundaryEndIndex === $descriptionContentLength || + in_array($boundaryEnd, $isHashtag ? $hashtagBoundaryEndChars : $boundaryEndChars); + + if ($boundaryStartIsValid && $boundaryEndIsValid) { + $minPositionOffset = $commandRun->startIndex - $position; + $enhancement = [ + 'pos' => $position, + 'len' => $commandRun->length, + ]; + break; + } + + $position--; + } + + if (!isset($enhancement)) { + $this->logger->debug(sprintf('Position %d cannot be corrected in "%s"', $commandRun->startIndex, substr($descriptionContent, 0, 50) . '...')); + // Skip to prevent the description from becoming corrupted + continue; + } + + // $commandRun->length sometimes incorrectly includes the newline as last char + $lastChar = mb_substr($descriptionContent, $enhancement['pos'] + $enhancement['len'] - 1, 1); + if ($lastChar === "\n") { + $enhancement['len'] -= 1; + } + + $commandUrl = parse_url($commandMetadata->url); + if ($commandUrl['path'] === '/redirect') { + parse_str($commandUrl['query'], $commandUrlQuery); + $enhancement['url'] = urldecode($commandUrlQuery['q']); + } elseif (isset($commandUrl['host'])) { + $enhancement['url'] = $commandMetadata->url; + } else { + $enhancement['url'] = $baseUrl . $commandMetadata->url; + } + + $enhancements[] = $enhancement; + } + + if (count($enhancements) !== count($commandRuns)) { + // At least one link can not be matched. Discard everything to prevent corrupting the description. + return []; + } + + // Sort by position in descending order to be able to safely replace values + return array_reverse($enhancements); } - private function ytBridgeParseXmlFeed($xml) + private function extractItemsFromXmlFeed($xml) { + $this->feedName = $this->decodeTitle($xml->find('feed > title', 0)->plaintext); + foreach ($xml->find('entry') as $element) { - $title = $this->ytBridgeFixTitle($element->find('title', 0)->plaintext); + $videoId = str_replace('yt:video:', '', $element->find('id', 0)->plaintext); + if (strpos($videoId, 'googleads') !== false) { + continue; + } + $title = $this->decodeTitle($element->find('title', 0)->plaintext); $author = $element->find('name', 0)->plaintext; $desc = $element->find('media:description', 0)->innertext; - - // Make sure the description is easy on the eye :) $desc = htmlspecialchars($desc); $desc = nl2br($desc); - $desc = preg_replace( - self::URI_REGEX, - '$1 ', - $desc - ); - - $vid = str_replace('yt:video:', '', $element->find('id', 0)->plaintext); + $desc = preg_replace(self::URI_REGEX, '$1 ', $desc); $time = strtotime($element->find('published', 0)->plaintext); - if (strpos($vid, 'googleads') === false) { - $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); - } + $this->addItem($videoId, $title, $author, $desc, $time); } - $this->feedName = $this->ytBridgeFixTitle($xml->find('feed > title', 0)->plaintext); // feedName will be used by getName() } - private function ytBridgeFixTitle($title) + private function fetch($url, bool $cache = false) { - // convert both Ӓ and " to UTF-8 - return html_entity_decode($title, ENT_QUOTES, 'UTF-8'); - } - - private function ytGetSimpleHTMLDOM($url, $cached = false) - { - $header = [ - 'Accept-Language: en-US' - ]; - $opts = []; - $lowercase = true; - $forceTagsClosed = true; - $target_charset = DEFAULT_TARGET_CHARSET; - $stripRN = false; - $defaultBRText = DEFAULT_BR_TEXT; - $defaultSpanText = DEFAULT_SPAN_TEXT; - if ($cached) { - return getSimpleHTMLDOMCached( - $url, - 86400, - $header, - $opts, - $lowercase, - $forceTagsClosed, - $target_charset, - $stripRN, - $defaultBRText, - $defaultSpanText - ); + $header = ['Accept-Language: en-US']; + $ttl = 86400 * 3; // 3d + $stripNewlines = false; + if ($cache) { + return getSimpleHTMLDOMCached($url, $ttl, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines); } - return getSimpleHTMLDOM( - $url, - $header, - $opts, - $lowercase, - $forceTagsClosed, - $target_charset, - $stripRN, - $defaultBRText, - $defaultSpanText - ); + return getSimpleHTMLDOM($url, $header, [], true, true, DEFAULT_TARGET_CHARSET, $stripNewlines); } - private function getJSONData($html) + private function extractJsonFromHtml($html) { $scriptRegex = '/var ytInitialData = (.*?);<\/script>/'; $result = preg_match($scriptRegex, $html, $matches); if (! $result) { - Logger::debug('Could not find ytInitialData'); + $this->logger->debug('Could not find ytInitialData'); return null; } - return json_decode($matches[1]); + $data = json_decode($matches[1]); + return $data; } - private function parseJSONListing($jsonData) + private function fetchItemsFromFromJsonData($jsonData) { - $duration_min = $this->getInput('duration_min') ?: -1; - $duration_min = $duration_min * 60; - - $duration_max = $this->getInput('duration_max') ?: INF; - $duration_max = $duration_max * 60; - - if ($duration_max < $duration_min) { - returnClientError('Max duration must be greater than min duration!'); - } - - // $vid_list = ''; + $minimumDurationSeconds = ($this->getInput('duration_min') ?: -1) * 60; + $maximumDurationSeconds = ($this->getInput('duration_max') ?: INF) * 60; foreach ($jsonData as $item) { $wrapper = null; @@ -269,24 +441,32 @@ class YoutubeBridge extends BridgeAbstract continue; } - $vid = $wrapper->videoId; - $title = $wrapper->title->runs[0]->text; - if (isset($wrapper->ownerText)) { - $this->channel_name = $wrapper->ownerText->runs[0]->text; - } elseif (isset($wrapper->shortBylineText)) { - $this->channel_name = $wrapper->shortBylineText->runs[0]->text; + // 01:03:30 | 15:06 | 1:24 + $lengthText = $wrapper->lengthText->simpleText ?? null; + // 6,875 views + $viewCount = $wrapper->viewCountText->simpleText ?? null; + // Dc645M8Het8 + $videoId = $wrapper->videoId; + // Jumbo frames - transfer more data faster! + $title = $wrapper->title->runs[0]->text ?? $wrapper->title->accessibility->accessibilityData->label ?? null; + $author = null; + $description = $wrapper->descriptionSnippet->runs[0]->text ?? null; + // 5 days ago | 1 month ago + $publishedTimeText = $wrapper->publishedTimeText->simpleText ?? $wrapper->videoInfo->runs[2]->text ?? null; + $timestamp = null; + if ($publishedTimeText) { + try { + $publicationDate = new \DateTimeImmutable($publishedTimeText); + // Hard-code hour, minute and second + $publicationDate = $publicationDate->setTime(0, 0, 0); + $timestamp = $publicationDate->getTimestamp(); + } catch (\Exception $e) { + } } - $author = ''; - $desc = ''; - $time = ''; - - // The duration comes in one of the formats: - // hh:mm:ss / mm:ss / m:ss - // 01:03:30 / 15:06 / 1:24 $durationText = 0; - if (isset($wrapper->lengthText)) { - $durationText = $wrapper->lengthText->simpleText; + if ($lengthText) { + $durationText = $lengthText; } else { foreach ($wrapper->thumbnailOverlays as $overlay) { if (isset($overlay->thumbnailOverlayTimeStatusRenderer)) { @@ -295,7 +475,6 @@ class YoutubeBridge extends BridgeAbstract } } } - if (is_string($durationText)) { if (preg_match('/([\d]{1,2})\:([\d]{1,2})\:([\d]{2})/', $durationText)) { $durationText = preg_replace('/([\d]{1,2})\:([\d]{1,2})\:([\d]{2})/', '$1:$2:$3', $durationText); @@ -304,131 +483,50 @@ class YoutubeBridge extends BridgeAbstract } sscanf($durationText, '%d:%d:%d', $hours, $minutes, $seconds); $duration = $hours * 3600 + $minutes * 60 + $seconds; - if ($duration < $duration_min || $duration > $duration_max) { + if ($duration < $minimumDurationSeconds || $duration > $maximumDurationSeconds) { continue; } } - - // $vid_list .= $vid . ','; - $this->ytBridgeQueryVideoInfo($vid, $author, $desc, $time); - $this->ytBridgeAddItem($vid, $title, $author, $desc, $time); + if (!$description || !$timestamp) { + $this->fetchVideoDetails($videoId, $author, $description, $timestamp); + } + $this->addItem($videoId, $title, $author, $description, $timestamp); + if (count($this->items) >= 99) { + break; + } } } - public function collectData() + private function addItem($videoId, $title, $author, $description, $timestamp, $thumbnail = '') { - $xml = ''; - $html = ''; - $url_feed = ''; - $url_listing = ''; + $description = nl2br($description); - if ($this->getInput('u')) { /* User and Channel modes */ - $this->request = $this->getInput('u'); - $url_feed = self::URI . 'feeds/videos.xml?user=' . urlencode($this->request); - $url_listing = self::URI . 'user/' . urlencode($this->request) . '/videos'; - } elseif ($this->getInput('c')) { - $this->request = $this->getInput('c'); - $url_feed = self::URI . 'feeds/videos.xml?channel_id=' . urlencode($this->request); - $url_listing = self::URI . 'channel/' . urlencode($this->request) . '/videos'; - } elseif ($this->getInput('custom')) { - $this->request = $this->getInput('custom'); - $url_listing = self::URI . urlencode($this->request) . '/videos'; - } - - if (!empty($url_feed) || !empty($url_listing)) { - $this->feeduri = $url_listing; - if (!empty($this->getInput('custom'))) { - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - $url_feed = $jsonData->metadata->channelMetadataRenderer->rssUrl; - $this->iconURL = $jsonData->metadata->channelMetadataRenderer->avatar->thumbnails[0]->url; - } - if (!$this->skipFeeds()) { - $html = $this->ytGetSimpleHTMLDOM($url_feed); - $this->ytBridgeParseXmlFeed($html); - } else { - if (empty($this->getInput('custom'))) { - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - } - $channel_id = ''; - if (isset($jsonData->contents)) { - $channel_id = $jsonData->metadata->channelMetadataRenderer->externalId; - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[1]; - $jsonData = $jsonData->tabRenderer->content->richGridRenderer->contents; - // $jsonData = $jsonData->itemSectionRenderer->contents[0]->gridRenderer->items; - $this->parseJSONListing($jsonData); - } else { - returnServerError('Unable to get data from YouTube. Username/Channel: ' . $this->request); - } - } - $this->feedName = str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); - } elseif ($this->getInput('p')) { /* playlist mode */ - // TODO: this mode makes a lot of excess video query requests. - // To make less requests, we need to cache following dictionary "videoId -> datePublished, duration" - // This cache will be used to find out, which videos to fetch - // to make feed of 15 items or more, if there a lot of videos published on that date. - $this->request = $this->getInput('p'); - $url_feed = self::URI . 'feeds/videos.xml?playlist_id=' . urlencode($this->request); - $url_listing = self::URI . 'playlist?list=' . urlencode($this->request); - $html = $this->ytGetSimpleHTMLDOM($url_listing); - $jsonData = $this->getJSONData($html); - // TODO: this method returns only first 100 video items - // if it has more videos, playlistVideoListRenderer will have continuationItemRenderer as last element - $jsonData = $jsonData->contents->twoColumnBrowseResultsRenderer->tabs[0]; - $jsonData = $jsonData->tabRenderer->content->sectionListRenderer->contents[0]->itemSectionRenderer; - $jsonData = $jsonData->contents[0]->playlistVideoListRenderer->contents; - $item_count = count($jsonData); - - if ($item_count <= 15 && !$this->skipFeeds() && ($xml = $this->ytGetSimpleHTMLDOM($url_feed))) { - $this->ytBridgeParseXmlFeed($xml); - } else { - $this->parseJSONListing($jsonData); - } - $this->feedName = 'Playlist: ' . str_replace(' - YouTube', '', $html->find('title', 0)->plaintext); // feedName will be used by getName() - usort($this->items, function ($item1, $item2) { - if (!is_int($item1['timestamp']) && !is_int($item2['timestamp'])) { - $item1['timestamp'] = strtotime($item1['timestamp']); - $item2['timestamp'] = strtotime($item2['timestamp']); - } - return $item2['timestamp'] - $item1['timestamp']; - }); - } elseif ($this->getInput('s')) { /* search mode */ - $this->request = $this->getInput('s'); - $url_listing = self::URI - . 'results?search_query=' - . urlencode($this->request) - . '&sp=CAI%253D'; - - $html = $this->ytGetSimpleHTMLDOM($url_listing); - - $jsonData = $this->getJSONData($html); - $jsonData = $jsonData->contents->twoColumnSearchResultsRenderer->primaryContents; - $jsonData = $jsonData->sectionListRenderer->contents; - foreach ($jsonData as $data) { // Search result includes some ads, have to filter them - if (isset($data->itemSectionRenderer->contents[0]->videoRenderer)) { - $jsonData = $data->itemSectionRenderer->contents; - break; - } - } - $this->parseJSONListing($jsonData); - $this->feeduri = $url_listing; - $this->feedName = 'Search: ' . $this->request; // feedName will be used by getName() - } else { /* no valid mode */ - returnClientError("You must either specify either:\n - YouTube - username (?u=...)\n - Channel id (?c=...)\n - Playlist id (?p=...)\n - Search (?s=...)"); + $item = []; + // This should probably be uid? + $item['id'] = $videoId; + $item['title'] = $title; + $item['author'] = $author ?? ''; + $item['timestamp'] = $timestamp; + $item['uri'] = self::URI . '/watch?v=' . $videoId; + if (!$thumbnail) { + // Fallback to default thumbnail if there aren't any provided. + $thumbnail = '0'; } + $thumbnailUri = str_replace('/www.', '/img.', self::URI) . '/vi/' . $videoId . '/' . $thumbnail . '.jpg'; + $item['content'] = sprintf('
%s', $item['uri'], $thumbnailUri, $description); + $this->items[] = $item; } - private function skipFeeds() + private function decodeTitle($title) { - return ($this->getInput('duration_min') || $this->getInput('duration_max')); + // convert both Ӓ and " to UTF-8 + return html_entity_decode($title, ENT_QUOTES, 'UTF-8'); } public function getURI() { if (!is_null($this->getInput('p'))) { - return static::URI . 'playlist?list=' . $this->getInput('p'); + return static::URI . '/playlist?list=' . $this->getInput('p'); } elseif ($this->feeduri) { return $this->feeduri; } @@ -438,14 +536,13 @@ class YoutubeBridge extends BridgeAbstract public function getName() { - // Name depends on queriedContext: switch ($this->queriedContext) { case 'By username': case 'By channel id': case 'By custom name': case 'By playlist Id': case 'Search result': - return htmlspecialchars_decode($this->feedName) . ' - YouTube'; // We already know it's a bridge, right? + return htmlspecialchars_decode($this->feedName) . ' - YouTube'; default: return parent::getName(); } @@ -453,10 +550,10 @@ class YoutubeBridge extends BridgeAbstract public function getIcon() { - if (empty($this->iconURL)) { + if (empty($this->feedIconUrl)) { return parent::getIcon(); } else { - return $this->iconURL; + return $this->feedIconUrl; } } } diff --git a/bridges/ZDNetBridge.php b/bridges/ZDNetBridge.php index 693f542c..e3b659a8 100644 --- a/bridges/ZDNetBridge.php +++ b/bridges/ZDNetBridge.php @@ -174,19 +174,17 @@ class ZDNetBridge extends FeedExpander $this->collectExpandableDatas($url, $limit); } - protected function parseItem($item) + protected function parseItem(array $item) { - $item = parent::parseItem($item); - $article = getSimpleHTMLDOMCached($item['uri']); if (!$article) { - Logger::info('Unable to parse the dom from ' . $item['uri']); + $this->logger->info('Unable to parse the dom from ' . $item['uri']); return $item; } $articleTag = $article->find('article', 0) ?? $article->find('.c-articleContent', 0); if (!$articleTag) { - Logger::info('Unable to parse
Warning!
+Debug mode has been removed. -Enabling debug mode on a public server may result in malicious clients retrieving sensitive data about your server and possibly gaining access to it. Do not enable debug mode on a public server, unless you understand the implications of your doing! +If you want to disable caching you can set cache type to array (in-memory cache): -*** +```ini +[cache] -Debug mode enables error reporting and prevents loading data from the cache (data is still written to the cache). -To enable debug mode, set in `config.ini.php`: +; Cache type: file, sqlite, memcached, array, null +type = "array" +``` - enable_debug_mode = true +Alternatively, you can comment out the cache middleware in `lib/RssBridge.php`: -Allow only explicit ip addresses: +```diff +diff --git a/lib/RssBridge.php b/lib/RssBridge.php +index d16f1d89..da3df8be 100644 +--- a/lib/RssBridge.php ++++ b/lib/RssBridge.php +@@ -24,7 +24,7 @@ final class RssBridge - debug_mode_whitelist[] = 127.0.0.1 - debug_mode_whitelist[] = 192.168.1.10 - -_Notice_: - -* An empty file enables debug mode for anyone! -* The bridge whitelist still applies! (debug mode does **not** enable all bridges) - -RSS-Bridge will give you a visual feedback when debug mode is enabled. - -While debug mode is active, RSS-Bridge will write additional data to your servers `error.log`. - -Debug mode is controlled by the static class `Debug`. It provides three core functions: - -* `Debug::isEnabled()`: Returns `true` if debug mode is enabled. -* `Debug::log($message)`: Adds a message to `error.log`. It takes one parameter, which can be anything. - -Example: `Debug::log('Hello World!');` - -**Notice**: `Debug::log($message)` calls `Debug::isEnabled()` internally. You don't have to do that manually. \ No newline at end of file + $middlewares = [ + new BasicAuthMiddleware(), +- new CacheMiddleware($this->container['cache']), ++ //new CacheMiddleware($this->container['cache']), + new ExceptionMiddleware($this->container['logger']), + new SecurityMiddleware(), + new MaintenanceMiddleware(), +``` \ No newline at end of file diff --git a/docs/04_For_Developers/07_Development_Environment_Setup.md b/docs/04_For_Developers/07_Development_Environment_Setup.md index 23a4b101..d3a5ee8d 100644 --- a/docs/04_For_Developers/07_Development_Environment_Setup.md +++ b/docs/04_For_Developers/07_Development_Environment_Setup.md @@ -1,39 +1,5 @@ -These are examples of how to setup a local development environment to add bridges, improve the docs, etc. -## Docker - -The following can serve as an example for using docker: - -``` -# create a new directory -mkdir rss-bridge-contribution -cd rss-bridge-contribution - -# clone the project into a subfolder -git clone https://github.com/RSS-Bridge/rss-bridge -``` - -Then add a `docker-compose.yml` file: - -```yml -version: '3' - -services: - rss-bridge: - build: - context: ./rss-bridge - ports: - - 3000:80 - volumes: - - ./config:/config - - ./rss-bridge/bridges:/app/bridges -``` - -You can then access RSS-Bridge at `localhost:3000` and [add your bridge](../05_Bridge_API/How_to_create_a_new_bridge) to the `rss-bridge/bridges` folder. - -If you need to edit any other files, like from the `lib` folder add this to the `volumes` section: `./rss-bridge/lib:/app/lib`. - -### Docs with Docker +## Docs with Docker If you want to edit the docs add this to your docker-compose.yml: diff --git a/docs/04_For_Developers/index.md b/docs/04_For_Developers/index.md index fcce11e3..31e513a5 100644 --- a/docs/04_For_Developers/index.md +++ b/docs/04_For_Developers/index.md @@ -1,10 +1,11 @@ -This area is intended for developers who decide to contribute to **RSS-Bridge** which is primarily written in [`PHP`](http://www.php.net/) with some aspects of [`HTML`](https://en.wikipedia.org/wiki/HTML) and [`CSS`](https://en.wikipedia.org/wiki/Cascading_Style_Sheets). +This area is intended for developers who decide to contribute to **RSS-Bridge**. + +It is written in PHP. If you are new to **RSS-Bridge** you should make yourself familiar with some general aspects: + - [Coding style policy](./01_Coding_style_policy.md) - [Folder structure](./03_Folder_structure.md) - - [Debug mode](./05_Debug_mode.md) - [Bridge API](../05_Bridge_API/index.md) - [Cache API](../07_Cache_API/index.md) - - [Format API](../08_Format_API/index.md) - - [Technical recommendations](../09_Technical_recommendations/index.md) \ No newline at end of file + - [Technical recommendations](../09_Technical_recommendations/index.md) diff --git a/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md b/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md index 391d179f..e7d401dd 100644 --- a/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md +++ b/docs/05_Bridge_API/01_How_to_create_a_new_bridge.md @@ -1,30 +1,36 @@ -Create a new file in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). +# How to create a completely new bridge -The file name must be named according to following specification: -* It starts with the full name of the site -* All white-space must be removed -* The first letter of a word is written in upper-case, unless the site name is specified otherwise (example: Freenews, not FreeNews, because the site is named 'Freenews') -* The first character must be upper-case -* The file name must end with 'Bridge' -* The file type must be PHP, written in **small** letters (seriously!) ".php" - -**Examples:** - -Site | Filename ------|--------- -Wikipedia | **Wikipedia**Bridge.php -Facebook | **Facebook**Bridge.php -GitHub | **GitHub**Bridge.php -Freenews | **Freenews**Bridge.php - -The file must start with the PHP tags and end with an empty line. The closing tag `?>` is [omitted](http://php.net/basic-syntax.instruction-separation). - -**Example:** +New code files MUST have `declare(strict_types=1);` at the top of file: ```php find('.blog-posts li') as $li) { + $a = $li->find('a', 0); + $this->items[] = [ + 'title' => $a->plaintext, + 'uri' => 'https://herman.bearblog.dev' . $a->href, + ]; + } + } +} +``` + +Learn more in [bridge api](https://rss-bridge.github.io/rss-bridge/Bridge_API/index.html). diff --git a/docs/05_Bridge_API/02_BridgeAbstract.md b/docs/05_Bridge_API/02_BridgeAbstract.md index 120361be..9cb16050 100644 --- a/docs/05_Bridge_API/02_BridgeAbstract.md +++ b/docs/05_Bridge_API/02_BridgeAbstract.md @@ -1,4 +1,5 @@ -`BridgeAbstract` is a base class for standard bridges. It implements the most common functions to simplify the process of adding new bridges. +`BridgeAbstract` is a base class for standard bridges. +It implements the most common functions to simplify the process of adding new bridges. *** @@ -11,7 +12,9 @@ You need four basic steps in order to create a new bridge: [**Step 3**](#step-3---add-general-constants-to-the-class) - Add general constants to the class [**Step 4**](#step-4---implement-a-function-to-collect-feed-data) - Implement a function to collect feed data -These steps are described in more detail below. At the end of this document you'll find a complete [template](#template) based on these instructions. The pictures below show an example based on these instructions: +These steps are described in more detail below. +At the end of this document you'll find a complete [template](#template) based on these instructions. +The pictures below show an example based on these instructions:Show pictures
-Make sure to read these instructions carefully. Please don't hesitate to open an [Issue](https://github.com/RSS-Bridge/rss-bridge/issues) if you have further questions (or suggestions). Once your bridge is finished, please open a [Pull Request](https://github.com/RSS-Bridge/rss-bridge/pulls), in order to get your bridge merge into RSS-Bridge. +Make sure to read these instructions carefully. +Please don't hesitate to open an +[Issue](https://github.com/RSS-Bridge/rss-bridge/issues) +if you have further questions (or suggestions). +Once your bridge is finished, please open a [Pull Request](https://github.com/RSS-Bridge/rss-bridge/pulls), +in order to get your bridge merge into RSS-Bridge. *** @@ -33,7 +41,8 @@ Please read [these instructions](./01_How_to_create_a_new_bridge.md) on how to c ## Step 2 - Add a class, extending `BridgeAbstract` -Your bridge needs to be a class, which extends `BridgeAbstract`. The class name must **exactly** match the name of the file, without the file extension. +Your bridge needs to be a class, which extends `BridgeAbstract`. +The class name must **exactly** match the name of the file, without the file extension. For example: `MyBridge.php` => `MyBridge` @@ -41,10 +50,10 @@ For example: `MyBridge.php` => `MyBridge` ```PHP @@ -65,43 +74,47 @@ const CACHE_TIMEOUT // (optional) Defines the maximum duration for the cache in
Show example
-**Notice**: `const PARAMETERS` can be used to request information from the user. Refer to [these instructions](#parameters) for more information. +**Notice**: `const PARAMETERS` can be used to request information from the user. +Refer to [these instructions](#parameters) for more information. ## Step 4 - Implement a function to collect feed data -In order for RSS-Bridge to collect data, you must implement the **public** function `collectData`. This function takes no arguments and returns nothing. It generates a list of feed elements, which must be placed into the variable `$this->items`. +In order for RSS-Bridge to collect data, you must implement the **public** function `collectData`. +This function takes no arguments and returns nothing. +It generates a list of feed elements, which must be placed into the variable `$this->items`.
Show example
@@ -112,32 +125,36 @@ For more details on the `collectData` function refer to [these instructions](#co # Template -Use this template to create your own bridge. Please remove any unnecessary comments and parameters. +Use this template to create your own bridge. +Please remove any unnecessary comments and parameters. ```php items[] = $item; // Add item to the list - } +class MyBridge extends BridgeAbstract +{ + const NAME = 'Unnamed bridge'; + const URI = ''; + const DESCRIPTION = 'No description provided'; + const MAINTAINER = 'No maintainer'; + const PARAMETERS = []; // Can be omitted! + const CACHE_TIMEOUT = 3600; // Can be omitted! + + public function collectData() + { + $item = []; // Create an empty item + + $item['title'] = 'Hello World!'; + + $this->items[] = $item; // Add item to the list + } } -// This line is empty (just imagine it!) ``` # PARAMETERS -You can specify additional parameters in order to customize the bridge (i.e. to specify how many items to return). This document explains how to specify those parameters and which options are available to you. +You can specify additional parameters in order to customize the bridge (i.e. to specify how many items to return). +This document explains how to specify those parameters and which options are available to you. For information on how to read parameter values during execution, please refer to the [getInput](../06_Helper_functions/index.md#getinput) function. @@ -145,15 +162,17 @@ For information on how to read parameter values during execution, please refer t ## Adding parameters to a bridge -Parameters are specified as part of the bridge class. An empty list of parameters is defined as `const PARAMETERS = array();` +Parameters are specified as part of the bridge class. +An empty list of parameters is defined as `const PARAMETERS = [];`
Show example
Show example
Show example
-You can also define a set of parameters that will be applied to every possible context of your bridge. To do this, specify a context named `global`. +You can also define a set of parameters that will be applied to every possible context of your bridge. +To do this, specify a context named `global`.
Show example
Show example
@@ -229,17 +253,17 @@ The parameter specification consists of various fields, listed in the table belo
Show example
Show example
+ Additional keys may be added for custom APIs (ignored by RSS-Bridge). ## Item parameters -The item array should provide as much information as possible for RSS-Bridge to generate feature rich feeds. Find below list of keys supported by RSS-Bridge. +The item array should provide as much information as possible for RSS-Bridge to generate feature rich feeds. +Find below list of keys supported by RSS-Bridge. ```PHP $item['uri'] // URI to reach the subject ("https://...") @@ -376,139 +403,165 @@ $item['enclosures'] // Array of URIs to an attachments (pictures, files, etc...) $item['categories'] // Array of categories / tags / topics $item['uid'] // A unique ID to identify the current item ``` + All formats support these parameters. The formats `Plaintext` and `JSON` also support custom parameters. # getDescription The `getDescription` function returns the description for a bridge. -**Notice:** By default **RSS-Bridge** returns the contents of `const DESCRIPTION`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns the contents of `const DESCRIPTION`, +so you only have to implement this function if you require different behavior! ```PHP - public function getDescription(){ - return self::DESCRIPTION; - } +public function getDescription() +{ + return self::DESCRIPTION; +} ``` # getMaintainer The `getMaintainer` function returns the name of the maintainer for a bridge. -**Notice:** By default **RSS-Bridge** returns `const MAINTAINER`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns `const MAINTAINER`, +so you only have to implement this function if you require different behavior! ```PHP - public function getMaintainer(){ - return self::MAINTAINER; - } +public function getMaintainer() +{ + return self::MAINTAINER; +} ``` # getName + The `getName` function returns the name of a bridge. -**Notice:** By default **RSS-Bridge** returns `const NAME`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns `const NAME`, +so you only have to implement this function if you require different behavior! ```PHP - public function getName(){ - return self::NAME; - } +public function getName() +{ + return self::NAME; +} ``` # getURI + The `getURI` function returns the base URI for a bridge. -**Notice:** By default **RSS-Bridge** returns `const URI`, so you only have to implement this function if you require different behavior! +**Notice:** By default **RSS-Bridge** returns `const URI`, +so you only have to implement this function if you require different behavior! ```PHP - public function getURI(){ - return self::URI; - } +public function getURI() +{ + return self::URI; +} ``` # getIcon + The `getIcon` function returns the URI for an icon, used as favicon in feeds. -If no icon is specified by the bridge, RSS-Bridge will use a default location: `static::URI . '/favicon.ico'` (i.e. "https://github.com/favicon.ico") which may or may not exist. +If no icon is specified by the bridge, +RSS-Bridge will use a default location: `static::URI . '/favicon.ico'` (i.e. "https://github.com/favicon.ico") which may or may not exist. ```PHP - public function getIcon(){ - return static::URI . '/favicon.ico'; - } -``` -# detectParameters -The `detectParameters` function takes a URL and attempts to extract a valid set of parameters for the current bridge. - -If the passed URL is valid for this bridge the function should return an array of parameter -> value pairs that can be used by this bridge, or an empty array if the bridge requires no parameters. If the URL is not relevant for this bridge the function should return `null`. - -**Notice:** Implementing this function is optional. By default **RSS-Bridge** tries to match the supplied URL to the `URI` constant defined in the bridge which may be enough for bridges without any parameters defined. - -```PHP -public function detectParameters($url){ - $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/'; - if(empty(static::PARAMETERS) - && preg_match($regex, $url, $urlMatches) > 0 - && preg_match($regex, static::URI, $bridgeUriMatches) > 0 - && $urlMatches[3] === $bridgeUriMatches[3]) { - return array(); - } else { - return null; - } +public function getIcon() +{ + return static::URI . '/favicon.ico'; } ``` +# detectParameters + +The `detectParameters` function takes a URL and attempts to extract a valid set of parameters for the current bridge. + +If the passed URL is valid for this bridge, the function should return an array of parameter -> value pairs that can be used by this bridge, including context if available, or an empty array if the bridge requires no parameters. If the URL is not relevant for this bridge, the function should return `null`. + +**Notice:** Implementing this function is optional. By default, **RSS-Bridge** tries to match the supplied URL to the `URI` constant defined in the bridge, which may be enough for bridges without any parameters defined. + +```PHP +public function detectParameters($url) +{ + $regex = '/^(https?:\/\/)?(www\.)?(.+?)(\/)?$/'; + if (empty(static::PARAMETERS) + && preg_match($regex, $url, $urlMatches) > 0 + && preg_match($regex, static::URI, $bridgeUriMatches) > 0 + && $urlMatches[3] === $bridgeUriMatches[3] + ) { + return []; + } else { + return null; + } +} +``` + +**Notice:** This function is also used by the [findFeed](../04_For_Developers/04_Actions.md#findfeed) action. +This action allows an user to get a list of all feeds corresponding to an URL. + +You can implement automated tests for the `detectParameters` function by adding the `TEST_DETECT_PARAMETERS` constant to your bridge class constant. + +`TEST_DETECT_PARAMETERS` is an array, with as key the URL passed to the `detectParameters`function and as value, the array of parameters returned by `detectParameters` + +```PHP +const TEST_DETECT_PARAMETERS = [ + 'https://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'https://instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], + 'http://www.instagram.com/metaverse' => ['context' => 'Username', 'u' => 'metaverse'], +]; +``` + +**Notice:** Adding this constant is optional. If the constant is not present, no automated test will be executed. + + *** # Helper Methods -`BridgeAbstract` implements helper methods to make it easier for bridge maintainers to create bridges. Use these methods whenever possible instead of writing your own. -- [saveCacheValue](#savecachevalue) -- [loadCacheValue](#loadcachevalue) +`BridgeAbstract` implements helper methods to make it easier for bridge maintainers to create bridges. +Use these methods whenever possible instead of writing your own. ## saveCacheValue -Within the context of the current bridge, stores a value by key in the cache. The value can later be retrieved with [loadCacheValue](#loadcachevalue). + +Within the context of the current bridge, stores a value by key in the cache. +The value can later be retrieved with [loadCacheValue](#loadcachevalue). ```php -protected function saveCacheValue($key, $value) +protected function saveCacheValue($key, $value, $ttl = null) ``` -- `$key` - the name under which the value is stored in the cache. -- `$value` - the value to store in the cache. - -Usage example: +Example: ```php -const MY_KEY = 'MyKey'; - public function collectData() { - $value = 'my value'; - $this->saveCacheValue(MY_KEY, $value); + $this->saveCacheValue('my_key', 'my_value', 3600); // 1h } ``` ## loadCacheValue -Within the context of the current bridge, loads a value by key from cache. Optionally specifies the cache duration for the key. Returns `null` if the key doesn't exist or the value is expired. + +Within the context of the current bridge, loads a value by key from cache. +Optionally specifies the cache duration for the key. +Returns `null` if the key doesn't exist or the value is expired. ```php -protected function loadCacheValue($key, $duration = null) +protected function loadCacheValue($key, $default = null) ``` -- `$key` - the name under which the value is stored in the cache. -- `$duration` - the maximum time in seconds after which the value expires. - -Usage example: +Example: ```php -const MY_KEY = 'MyKey'; - public function collectData() { - $value = $this->loadCacheValue(MY_KEY, 1800 /* 30 minutes */); + $value = $this->loadCacheValue('my_key'); - if (!isset($value)){ - // load value - $this->saveCacheValue(MY_KEY, $value); - } - - // ... + if (! $value) { + $this->saveCacheValue('my_key', 'foobar'); + } } ``` diff --git a/docs/05_Bridge_API/03_FeedExpander.md b/docs/05_Bridge_API/03_FeedExpander.md index 7e72670a..62356fc9 100644 --- a/docs/05_Bridge_API/03_FeedExpander.md +++ b/docs/05_Bridge_API/03_FeedExpander.md @@ -1,85 +1,35 @@ -`FeedExpander` extends [`BridgeAbstract`](./02_BridgeAbstract.md) and adds functions to collect data from existing feeds. - **Usage example**: _You have discovered a site that provides feeds which are hidden and inaccessible by normal means. You want your bridge to directly read the feeds and provide them via **RSS-Bridge**_ -To create a new Bridge extending `FeedExpander` you must implement all required functions of [`BridgeAbstract`](./02_BridgeAbstract.md). `FeedExpander` additionally provides following functions: - -* [`parseItem`](#the-parseitem-function) -* [`getName`](#the-getname-function) -* [`getURI`](#the-geturi-function) -* [`getDescription`](#the-getdescription-function) - Find a [template](#template) at the end of this file. **Notice:** For a standard feed only `collectData` need to be implemented. `collectData` should call `$this->collectExpandableDatas('your URI here');` to automatically load feed items and header data (will subsequently call `parseItem` for each item in the feed). You can limit the number of items to fetch by specifying an additional parameter for: `$this->collectExpandableDatas('your URI here', 10)` (limited to 10 items). -## The `parseItem` function +## The `parseItem` method -This function receives one item from the current feed and should return one **RSS-Bridge** item. +This method receives one item from the current feed and should return one **RSS-Bridge** item. The default function does all the work to get the item data from the feed, whether it is RSS 1.0, -RSS 2.0 or Atom 1.0. If you have to redefine this function in your **RSS-Bridge** for whatever reason, -you should first call the parent function to initialize the item, then apply the changes that you require. +RSS 2.0 or Atom 1.0. **Notice:** The following code sample is just an example. Implementation depends on your requirements! ```PHP -protected function parseItem($feedItem){ - $item = parent::parseItem($feedItem); - $item['content'] = str_replace('rssbridge','RSS-Bridge',$feedItem->content); - +protected function parseItem(array $item) +{ + $item['content'] = str_replace('rssbridge','RSS-Bridge',$item['content']); return $item; } ``` -### Helper functions +### Feed parsing -The `FeedExpander` already provides a set of functions to parse RSS or Atom items based on the specifications. Where possible make use of these functions: - -Function | Description ----------|------------ -`parseATOMItem` | Parses an Atom 1.0 feed item -`parseRSS_0_9_1_Item` | Parses an RSS 0.91 feed item -`parseRSS_1_0_Item` | Parses an RSS 1.0 feed item -`parseRSS_2_0_Item` | Parses an RSS 2.0 feed item - -In the following list you'll find the feed tags assigned to the the **RSS-Bridge** item keys: +How rss-bridge processes xml feeds: Function | uri | title | timestamp | author | content ---------|-----|-------|-----------|--------|-------- -`parseATOMItem` | id | title | updated | author | content -`parseRSS_0_9_1_Item` | link | title | | | description -`parseRSS_1_0_Item` | link | title | dc:date | dc:creator | description -`parseRSS_2_0_Item` | link, guid | title | pubDate, dc:date | author, dc:creator | description - -## The `getName` function - -Returns the name of the current feed. - -```PHP -return $this->name; -``` - -**Notice:** Only implement this function if you require different behavior! - -## The `getURI` function - -Return the uri for the current feed. - -```PHP -return $this->uri; -``` - -**Notice:** Only implement this function if you require different behavior! - -## The `getDescription` function - -Returns the description for the current bridge. - -```PHP -return $this->description; -``` - -**Notice:** Only implement this function if you require different behavior! +`atom` | id | title | updated | author | content +`rss 0.91` | link | title | | | description +`rss 1.0` | link | title | dc:date | dc:creator | description +`rss 2.0` | link, guid | title | pubDate, dc:date | author, dc:creator | description # Template @@ -87,18 +37,19 @@ This is the template for a new bridge: ```PHP collectExpandableDatas('your feed URI'); } } -// Imaginary empty line! ``` \ No newline at end of file diff --git a/docs/05_Bridge_API/04_WebDriverAbstract.md b/docs/05_Bridge_API/04_WebDriverAbstract.md new file mode 100644 index 00000000..60b5e99d --- /dev/null +++ b/docs/05_Bridge_API/04_WebDriverAbstract.md @@ -0,0 +1,83 @@ +`WebDriverAbstract` extends [`BridgeAbstract`](./02_BridgeAbstract.md) and adds functionality for generating feeds +from active websites that use XMLHttpRequest (XHR) to load content and / or JavaScript to +modify content. +It highly depends on the php-webdriver library which offers Selenium WebDriver bindings for PHP. + +- https://github.com/php-webdriver/php-webdriver (Project Repository) +- https://php-webdriver.github.io/php-webdriver/latest/ (API) + +Please note that this class is intended as a solution for websites _that cannot be covered +by the other classes_. The WebDriver starts a browser and is therefore very resource-intensive. + +# Configuration + +You need a running WebDriver to use bridges that depend on `WebDriverAbstract`. +The easiest way is to start the Selenium server from the project of the same name: +``` +docker run -d -p 4444:4444 --shm-size="2g" docker.io/selenium/standalone-chrome:latest +``` + +- https://github.com/SeleniumHQ/docker-selenium + +With these parameters only one browser window can be started at a time. +On a multi-user site, Selenium Grid should be used +and the number of sessions should be adjusted to the number of processor cores. + +Finally, the `config.ini.php` file must be adjusted so that the WebDriver +can find the Selenium server: +``` +[webdriver] + +selenium_server_url = "http://localhost:4444" +``` + +# Development + +While you are programming a new bridge, it is easier to start a local WebDriver because then you can see what is happening and where the errors are. I've also had good experience recording the process with a screen video to find any timing problems. + +``` +chromedriver --port=4444 +``` + +- https://chromedriver.chromium.org/ + +If you start rss-bridge from a container, then Chrome driver is only accessible +if you call it with the `--allowed-ips` option so that it binds to all network interfaces. + +``` +chromedriver --port=4444 --allowed-ips=192.168.1.42 +``` + +The **most important rule** is that after an event such as loading the web page +or pressing a button, you often have to explicitly wait for the desired elements to appear. + +A simple example is the bridge `ScalableCapitalBlogBridge.php`. +A more complex and relatively complete example is the bridge `GULPProjekteBridge.php`. + +# Template + +Use this template to create your own bridge. + +```PHP +cleanUp(); + } + } +} + +``` \ No newline at end of file diff --git a/docs/05_Bridge_API/04_XPathAbstract.md b/docs/05_Bridge_API/05_XPathAbstract.md similarity index 94% rename from docs/05_Bridge_API/04_XPathAbstract.md rename to docs/05_Bridge_API/05_XPathAbstract.md index cf091edc..28a20a81 100644 --- a/docs/05_Bridge_API/04_XPathAbstract.md +++ b/docs/05_Bridge_API/05_XPathAbstract.md @@ -68,6 +68,9 @@ Should return the XPath expression for extracting an item title from the item co ### Method `getExpressionItemContent()` Should return the XPath expression for extracting an item's content from the item context. +### Method `getSettingUseRawItemContent()` +Should return the 'Use raw item content' setting value (bool true or false). + ### Method `getExpressionItemUri()` Should return the XPath expression for extracting an item link from the item context. @@ -96,7 +99,7 @@ This method should return the HTML source as a base for the XPath expressions. U This method should provide the feed title. Usually the XPath expression defined in `XPATH_EXPRESSION_FEED_TITLE` is used for extracting the title directly from the page source. ### Method `provideFeedIcon()` -This method should provide the feed title. Usually the XPath expression defined in `XPATH_EXPRESSION_FEED_ICON` is used for extracting the title directly from the page source. +This method should provide the URL of the feed's favicon. Usually the XPath expression defined in `XPATH_EXPRESSION_FEED_ICON` is used for extracting the title directly from the page source. ### Method `provideFeedItems()` This method should provide the feed items. Usually the XPath expression defined in `XPATH_EXPRESSION_ITEM` is used for extracting the items from the page source. All other XPath expressions are applied on a per-item basis, item by item, and only on the item's contents. @@ -119,8 +122,8 @@ Accepts the items author as parameter, processes and returns it. Should return a ### Method `formatItemTimestamp()` Accepts the items creation timestamp as parameter, processes and returns it. Should return a unix timestamp as integer. -### Method `cleanImageUrl()` -Method invoked for cleaning feed icon and item image URL's. Extracts the image URL from the passed parameter, stripping any additional content. Furthermore makes sure that relative image URL's get transformed to absolute ones. +### Method `cleanMediaUrl()` +Method invoked for cleaning feed icon, item image and media attachment (like .mp3, .webp) URL's. Extracts the media URL from the passed parameter, stripping any additional content. Furthermore, makes sure that relative media URL's get transformed to absolute ones. ### Method `fixEncoding()` Only invoked when class constant `SETTING_FIX_ENCODING` is set to true. It then passes all extracted string values through PHP's `utf8_decode` function. diff --git a/docs/05_Bridge_API/index.md b/docs/05_Bridge_API/index.md index 6115fa01..ea6fd315 100644 --- a/docs/05_Bridge_API/index.md +++ b/docs/05_Bridge_API/index.md @@ -1,9 +1,14 @@ -A _Bridge_ is an class that allows **RSS-Bridge** to create an RSS-feed from a website. A _Bridge_ represents one element on the [Welcome screen](../01_General/04_Screenshots.md) and covers one or more sites to return feeds for. It is developed in a PHP file located in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)) and extends one of the base classes of **RSS-Bridge**: +A _Bridge_ is a class that allows **RSS-Bridge** to create an RSS-feed from a website. +A _Bridge_ represents one element on the [Welcome screen](../01_General/04_Screenshots.md) +and covers one or more sites to return feeds for. +It is developed in a PHP file located in the `bridges/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)) +and extends one of the base classes of **RSS-Bridge**: Base class | Description -----------|------------ [`BridgeAbstract`](./02_BridgeAbstract.md) | This class is intended for standard _Bridges_ that need to filter HTML pages for content. -[`FeedExpander`](./03_FeedExpander.md) | This class is an extension of `HttpCachingBridgeAbstract`, designed to load existing feeds into **RSS-Bridge** -[`XPathAbstract`](./04_XPathAbstract.md) | This class is meant as an alternative base class for bridge implementations. It offers preliminary functionality for generating feeds based on _XPath expressions_. +[`FeedExpander`](./03_FeedExpander.md) | Expand/modify existing feed urls +[`WebDriverAbstract`](./04_WebDriverAbstract) | +[`XPathAbstract`](./05_XPathAbstract) | This class is meant as an alternative base class for bridge implementations. It offers preliminary functionality for generating feeds based on _XPath expressions_. For more information about how to create a new _Bridge_, read [How to create a new Bridge?](./01_How_to_create_a_new_bridge.md) \ No newline at end of file diff --git a/docs/06_Helper_functions/index.md b/docs/06_Helper_functions/index.md index 2f0c513c..4004138e 100644 --- a/docs/06_Helper_functions/index.md +++ b/docs/06_Helper_functions/index.md @@ -5,10 +5,14 @@ The `getInput` function is used to receive a value for a parameter, specified in $this->getInput('your input name here'); ``` -`getInput` will either return the value for your parameter or `null` if the parameter is unknown or not specified. +`getInput` will either return the value for your parameter +or `null` if the parameter is unknown or not specified. + +[Defined in lib/BridgeAbstract.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/BridgeAbstract.php) # getKey -The `getKey` function is used to receive the key name to a selected list value given the name of the list, specified in `const PARAMETERS` +The `getKey` function is used to receive the key name to a selected list +value given the name of the list, specified in `const PARAMETERS` Is able to work with multidimensional list arrays. ```PHP @@ -34,7 +38,10 @@ $this->getKey('country'); // if the selected value was "ve", this function will return "Venezuela" ``` -`getKey` will either return the key name for your parameter or `null` if the parameter is unknown or not specified. +`getKey` will either return the key name for your parameter or `null` if the parameter +is unknown or not specified. + +[Defined in lib/BridgeAbstract.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/BridgeAbstract.php) # getContents The `getContents` function uses [cURL](https://secure.php.net/manual/en/book.curl.php) to acquire data from the specified URI while respecting the various settings defined at a global level by RSS-Bridge (i.e., proxy host, user agent, etc.). This function accepts a few parameters: @@ -52,52 +59,64 @@ $opts = array(CURLOPT_POST => 1); $html = getContents($url, $header, $opts); ``` +[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php) + # getSimpleHTMLDOM -The `getSimpleHTMLDOM` function is a wrapper for the [simple_html_dom](https://simplehtmldom.sourceforge.io/) [file_get_html](https://simplehtmldom.sourceforge.io/docs/1.9/api/file_get_html/) function in order to provide context by design. +The `getSimpleHTMLDOM` function is a wrapper for the +[simple_html_dom](https://simplehtmldom.sourceforge.io/) [file_get_html](https://simplehtmldom.sourceforge.io/docs/1.9/api/file_get_html/) function in order to provide context by design. ```PHP $html = getSimpleHTMLDOM('your URI'); ``` + +[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php) + # getSimpleHTMLDOMCached -The `getSimpleHTMLDOMCached` function does the same as the [`getSimpleHTMLDOM`](#getsimplehtmldom) function, except that the content received for the given URI is stored in a cache and loaded from cache on the next request if the specified cache duration was not reached. Use this function for data that is very unlikely to change between consecutive requests to **RSS-Bridge**. This function allows to specify the cache duration with the second parameter (default is 24 hours / 86400 seconds). +The `getSimpleHTMLDOMCached` function does the same as the +[`getSimpleHTMLDOM`](#getsimplehtmldom) function, +except that the content received for the given URI is stored in a cache +and loaded from cache on the next request if the specified cache duration +was not reached. + +Use this function for data that is very unlikely to change between consecutive requests to **RSS-Bridge**. +This function allows to specify the cache duration with the second parameter. ```PHP $html = getSimpleHTMLDOMCached('your URI', 86400); // Duration 24h ``` -**Notice:** Due to the current implementation a value greater than 86400 seconds (24 hours) will not work as the cache is purged every 24 hours automatically. +[Defined in lib/contents.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/contents.php) -# returnError -**Notice:** Whenever possible make use of [`returnClientError`](#returnclienterror) or [`returnServerError`](#returnservererror) - -The `returnError` function aborts execution of the current bridge and returns the given error message with the provided error number: +# throwClientException($message = '') +The `throwClientException` function aborts execution of the current bridge. ```PHP -returnError('Your error message', 404); -``` - -Check the [list of error codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) for applicable error numbers. - -# returnClientError -The `returnClientError` function aborts execution of the current bridge and returns the given error message with error code **400**: - -```PHP -returnClientError('Your error message') +throwClientException('Bad user input') ``` Use this function when the user provided invalid parameter or a required parameter is missing. -# returnServerError -The `returnServerError` function aborts execution of the current bridge and returns the given error message with error code **500**: +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) + +# throwServerException($message = '') +The `throwServerException` function aborts execution of the current bridge. ```PHP -returnServerError('Your error message') +throwServerException('Received empty reply from thirdparty api') ``` -Use this function when a problem occurs that has nothing to do with the parameters provided by the user. (like: Host service gone missing, empty data received, etc...) +Use this function when a problem occurs that has nothing to do with the parameters provided by the user. +(like: Host service gone missing, empty data received, etc...) + +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) + +# throwRateLimitException($message = '') + +Throws a `RateLimitException` which produces an HTTP 429 response. # defaultLinkTo -Automatically replaces any relative URL in a given string or DOM object (i.e. the one returned by [getSimpleHTMLDOM](#getsimplehtmldom)) with an absolute URL. +Automatically replaces any relative URL in a given string or DOM object +(i.e. the one returned by [getSimpleHTMLDOM](#getsimplehtmldom)) with an absolute URL. ```php defaultLinkTo ( mixed $content, string $server ) : object @@ -121,6 +140,8 @@ $html = defaultLinkTo($html, $this->getURI()); // Using bridge URL //
Lorem ipsum dolor sit amet, consectetur adipiscing elit...
+ + +'; +$tags_to_remove = ['script', 'iframe', 'input', 'form']; +$attributes_to_keep = ['title', 'href', 'src']; +$text_to_keep = []; +sanitize($html, $tags_to_remove, $attributes_to_keep, $text_to_keep); +``` + +This function returns a simplehtmldom object of the remaining contents. + +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) + +# convertLazyLoading +The `convertLazyLoading` function is used to convert onvert lazy-loading images and frames (video embeds) into static elements. It accepts the HTML content as HTML objects or string objects. It returns the HTML content with fixed image/frame URLs (same type as input). + +```PHP +$html = ' + +Hello world!
+ + +backgroundToImg($html); +``` + +[Defined in lib/html.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/html.php) + +# Json::encode +The `Json::encode` function is used to encode a value as à JSON string. + +```PHP +$array = [ + "foo" => "bar", + "bar" => "foo", +]; +Json::encode($array, true, true); +``` + +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) + +# Json::decode +The `Json::decode` function is used to decode a JSON string into à PHP variable. + +```PHP +$json = '{ + "foo": "bar", + "bar": "foo" +}'; +Json::decode($json); +``` + +[Defined in lib/utils.php](https://github.com/RSS-Bridge/rss-bridge/blob/master/lib/utils.php) diff --git a/docs/07_Cache_API/02_CacheInterface.md b/docs/07_Cache_API/02_CacheInterface.md index 61127a0d..3e71237d 100644 --- a/docs/07_Cache_API/02_CacheInterface.md +++ b/docs/07_Cache_API/02_CacheInterface.md @@ -3,16 +3,14 @@ See `CacheInterface`. ```php interface CacheInterface { - public function setScope(string $scope): void; + public function get(string $key, $default = null); - public function setKey(array $key): void; + public function set(string $key, $value, int $ttl = null): void; - public function loadData(); + public function delete(string $key): void; - public function saveData($data): void; + public function clear(): void; - public function getTime(): ?int; - - public function purgeCache(int $seconds): void; + public function prune(): void; } -``` \ No newline at end of file +``` diff --git a/docs/07_Cache_API/index.md b/docs/07_Cache_API/index.md index 57692847..17afbacc 100644 --- a/docs/07_Cache_API/index.md +++ b/docs/07_Cache_API/index.md @@ -1,4 +1,6 @@ -A _Cache_ is a class that allows **RSS-Bridge** to store fetched data in a local storage area on the server. Cache imlementations are placed in the `caches/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). A cache must implement the [`CacheInterface`](../07_Cache_API/02_CacheInterface.md) interface. - -For more information about how to create a new `Cache`, read [How to create a new cache?](../07_Cache_API/01_How_to_create_a_new_cache.md) +A _Cache_ is a class that allows **RSS-Bridge** to store fetched data in a local storage area on the server. +Cache imlementations are placed in the `caches/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). +A cache must implement the [`CacheInterface`](../07_Cache_API/02_CacheInterface.md) interface. +For more information about how to create a new `Cache`, read +[How to create a new cache?](../07_Cache_API/01_How_to_create_a_new_cache.md) diff --git a/docs/08_Format_API/01_How_to_create_a_new_format.md b/docs/08_Format_API/01_How_to_create_a_new_format.md deleted file mode 100644 index f031e65b..00000000 --- a/docs/08_Format_API/01_How_to_create_a_new_format.md +++ /dev/null @@ -1,24 +0,0 @@ -Create a new file in the `formats/` folder (see [Folder structure](../04_For_Developers/03_Folder_structure.md)). - -The file must be named according to following specification: - -* It starts with the type -* The file name must end with 'Format' -* The file type must be PHP, written in small letters (seriously!) ".php" - -**Examples:** - -Type | Filename ------|--------- -Atom | AtomFormat.php -Html | HtmlFormat.php - -The file must start with the PHP tags and end with an empty line. The closing tag `?>` is [omitted](http://php.net/basic-syntax.instruction-separation). - -Example: - -```PHP -items = $items; - return $this; - } - - public function getItems(){ - return $this->items; - } - - public function setCharset($charset){ - $this->charset = $charset; - return $this; - } - - public function getCharset(){ - return $this->charset; - } - - public function setExtraInfos(array $infos){ - $this->extraInfos = $infos; - return $this; - } - - public function getExtraInfos(){ - return $this->extraInfos; - } -} -// Imaginary empty line! -``` diff --git a/docs/08_Format_API/index.md b/docs/08_Format_API/index.md deleted file mode 100644 index c5b9e6af..00000000 --- a/docs/08_Format_API/index.md +++ /dev/null @@ -1,9 +0,0 @@ -A Format is a class that allows RSS-Bridge to turn items from a bridge into an RSS-feed format. -It is developed in a PHP file located in the `formats/` folder -[Folder structure](../04_For_Developers/03_Folder_structure.md) -and either implements the -[FormatInterface](../08_Format_API/02_FormatInterface.md) -interface or extends the FormatAbstract class. - -For more information about how to create a new _Format_, read -[How to create a new Format?](./01_How_to_create_a_new_format.md) \ No newline at end of file diff --git a/docs/09_Technical_recommendations/index.md b/docs/09_Technical_recommendations/index.md index f2b15476..92c672ca 100644 --- a/docs/09_Technical_recommendations/index.md +++ b/docs/09_Technical_recommendations/index.md @@ -1,28 +1,32 @@ ## General recommendations -* Use [HTTPS](https://en.wikipedia.org/wiki/HTTPS) (`https://...`) over [HTTP](https://en.wikipedia.org/wiki/HTTPS) (`http://...`) whenever possible - ## Test a site before building a bridge -Some sites make use of anti-bot mechanisms (e.g.: by using JavaScript) in which case they work fine in regular browsers, but not in the PHP environment. To check if a site works with RSS-Bridge, create a new bridge using the [template](../05_Bridge_API/02_BridgeAbstract.md#template) and load a valid URL (not the base URL!). +Some sites make use of anti-bot mechanisms (e.g.: by using JavaScript) in which case they work fine in regular browsers, +but not in the PHP environment. RSS-Bridge Docker container by default resorts to using libcurl-impersonate, which helps mitigating anti-bot mechanisms. + +To check if a site works with RSS-Bridge, create a new bridge using the +[template](../05_Bridge_API/02_BridgeAbstract.md#template) +and load a valid URL (not the base URL!). **Example (using github.com)** ```PHP " + +[EconomistBridge] +cookie = "{$name}
-{$description}
- - -CARD; + +#
+ - // If we don't have any parameter for the bridge, we print a generic form to load it. - if (count($parameters) === 0) { - $card .= self::getForm($bridgeClassName, $formats, $isActive, $isHttps); +{$name}
+{$description}
- // Display form with cache timeout and/or noproxy options (if enabled) when bridge has no parameters - } elseif (count($parameters) === 1 && array_key_exists('global', $parameters)) { - $card .= self::getForm($bridgeClassName, $formats, $isActive, $isHttps, '', $parameters['global']); + + + + + CARD; + + if (count($contexts) === 0) { + // The bridge has zero parameters + $card .= self::renderForm($bridgeClassName, '', [], $token); + } elseif (count($contexts) === 1 && array_key_exists('global', $contexts)) { + // The bridge has a single context with key 'global' + $card .= self::renderForm($bridgeClassName, '', $contexts['global'], $token); } else { - foreach ($parameters as $parameterName => $parameter) { - if (!is_numeric($parameterName) && $parameterName === 'global') { + // The bridge has one or more contexts (named or unnamed) + foreach ($contexts as $contextName => $contextParameters) { + if ($contextName === 'global') { continue; } - if (array_key_exists('global', $parameters)) { - $parameter = array_merge($parameter, $parameters['global']); + if (array_key_exists('global', $contexts)) { + // Merge the global parameters into current context + $contextParameters = array_merge($contextParameters, $contexts['global']); } - if (!is_numeric($parameterName)) { - $card .= '' . $parameterName . '
' . PHP_EOL; + if (!is_numeric($contextName)) { + // This is a named context + $card .= '' . $contextName . '
' . PHP_EOL; } - $card .= self::getForm($bridgeClassName, $formats, $isActive, $isHttps, $parameterName, $parameter); + $card .= self::renderForm($bridgeClassName, $contextName, $contextParameters, $token); } } $card .= sprintf('', $bridgeClassName); - if ($bridge->getDonationURI() !== '' && Configuration::getConfig('admin', 'donations')) { + + if (Configuration::getConfig('admin', 'donations') && $bridge->getDonationURI()) { $card .= sprintf( '%s ~ Donate
', $bridge->getMaintainer(), @@ -113,58 +100,30 @@ CARD; return $card; } - /** - * Get the form header for a bridge card - * - * @param class-string${element.bridgeMeta.name}
++ ${element.bridgeMeta.description} +
+`; + + // Now display every Feed parameter + for (const param in element.bridgeData) { + content += `- ${element.bridgeData[param].name} : ${element.bridgeData[param].value}
`;
+ }
+ content += `
- Please authenticate in order to access this instance -
diff --git a/templates/base.html.php b/templates/base.html.php index ca31823d..a8ff7660 100644 --- a/templates/base.html.php +++ b/templates/base.html.php @@ -4,9 +4,11 @@ -The website is protected by CloudFlare
-- RSS-Bridge tried to fetch a website. - The fetching was blocked by CloudFlare. - CloudFlare is anti-bot software. - Its purpose is to block non-humans. -
- - - getCode() === 404): ?> -The website was not found
-- RSS-Bridge tried to fetch a page on a website. - But it doesn't exists. -
- - - getCode() === 429): ?> -Try again later
-- RSS-Bridge tried to fetch a website. - They told us to try again later. -
- - - - getCode() === 10): ?> -The rss feed is completely empty
-- RSS-Bridge tried parse the empty string as xml. - The fetched url is not pointing to real xml. -
- - - getCode() === 11): ?> -There is something wrong with the rss feed
-- RSS-Bridge tried parse xml. It failed. The xml is probably broken. -
- - - -Details
- -Trace
- - $frame) : ?> -- # --- - -
- -
Context
- -- - Go back -
+ +
+ ++ +
diff --git a/templates/exception.html.php b/templates/exception.html.php new file mode 100644 index 00000000..62ac90b4 --- /dev/null +++ b/templates/exception.html.php @@ -0,0 +1,148 @@ + +The website is protected by CloudFlare
++ RSS-Bridge tried to fetch a website. + The fetching was blocked by CloudFlare. + CloudFlare is anti-bot software. + Its purpose is to block non-humans. +
+ + + getCode() === 400): ?> +400 Bad Request
++ This is usually caused by an incorrectly constructed http request. +
+ + + getCode() === 403): ?> +403 Forbidden
++ The HTTP 403 Forbidden response status code indicates that the + server understands the request but refuses to authorize it. +
+ + + getCode() === 404): ?> +404 Page Not Found
++ RSS-Bridge tried to fetch a page on a website. + But it doesn't exists. +
+ + + getCode() === 429): ?> +429 Too Many Requests
++ RSS-Bridge tried to fetch a website. + They told us to try again later. +
+ + + getCode() === 503): ?> +503 Service Unavailable
++ Common causes are a server that is down for maintenance + or that is overloaded. +
+ + + getCode() === 0): ?> ++ See + + https://curl.haxx.se/libcurl/c/libcurl-errors.html + + for description of the curl error code. +
+ ++ + https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/getCode()) ?> + +
+ + + + getCode() === 10): ?> +The rss feed is completely empty
++ RSS-Bridge tried parse the empty string as xml. + The fetched url is not pointing to real xml. +
+ + + getCode() === 11): ?> +There is something wrong with the rss feed
++ RSS-Bridge tried parse xml. It failed. The xml is probably broken. +
+ + + +Details
+ +Trace
+ + $frame) : ?> ++ # +++ + +
+ +
Context
+ ++ + Go back +
- + + https://github.com/RSS-Bridge/rss-bridge + + +
+
+ +
+ +
/ active bridges.- - - - - -
- - - - -
- - -
@@ -49,7 +52,7 @@
@@ -29,15 +29,25 @@
+ Authentication with token required +
+ ++ +
+ + diff --git a/tests/Actions/ActionImplementationTest.php b/tests/Actions/ActionImplementationTest.php deleted file mode 100644 index e70dd7e2..00000000 --- a/tests/Actions/ActionImplementationTest.php +++ /dev/null @@ -1,69 +0,0 @@ -setAction($path); - $this->assertTrue($this->class === ucfirst($this->class), 'class name must start with uppercase character'); - $this->assertEquals(0, substr_count($this->class, ' '), 'class name must not contain spaces'); - $this->assertStringEndsWith('Action', $this->class, 'class name must end with "Action"'); - } - - /** - * @dataProvider dataActionsProvider - */ - public function testClassType($path) - { - $this->setAction($path); - $this->assertInstanceOf(ActionInterface::class, $this->obj); - } - - /** - * @dataProvider dataActionsProvider - */ - public function testVisibleMethods($path) - { - $allowedMethods = get_class_methods(ActionInterface::class); - sort($allowedMethods); - - $this->setAction($path); - - $methods = array_diff(get_class_methods($this->obj), ['__construct']); - sort($methods); - - $this->assertEquals($allowedMethods, $methods); - } - - public function dataActionsProvider() - { - $actions = []; - foreach (glob(PATH_LIB_ACTIONS . '*.php') as $path) { - $actions[basename($path, '.php')] = [$path]; - } - return $actions; - } - - private function setAction($path) - { - $this->class = '\\' . basename($path, '.php'); - $this->assertTrue(class_exists($this->class), 'class ' . $this->class . ' doesn\'t exist'); - $this->obj = new $this->class(); - } -} diff --git a/tests/Actions/ListActionTest.php b/tests/Actions/ListActionTest.php deleted file mode 100644 index e0625fb3..00000000 --- a/tests/Actions/ListActionTest.php +++ /dev/null @@ -1,75 +0,0 @@ -execute([]); - $headers = $response->getHeaders(); - $this->assertSame($headers['Content-Type'], 'application/json'); - } - - public function testOutput() - { - $action = new \ListAction(); - $response = $action->execute([]); - $data = $response->getBody(); - - $items = json_decode($data, true); - - $this->assertNotNull($items, 'invalid JSON output: ' . json_last_error_msg()); - - $this->assertArrayHasKey('total', $items, 'Missing "total" parameter'); - $this->assertIsInt($items['total'], 'Invalid type'); - - $this->assertArrayHasKey('bridges', $items, 'Missing "bridges" array'); - - $this->assertEquals( - $items['total'], - count($items['bridges']), - 'Item count doesn\'t match' - ); - - $bridgeFactory = new BridgeFactory(); - - $this->assertEquals( - count($bridgeFactory->getBridgeClassNames()), - count($items['bridges']), - 'Number of bridges doesn\'t match' - ); - - $expectedKeys = [ - 'status', - 'uri', - 'name', - 'icon', - 'parameters', - 'maintainer', - 'description' - ]; - - $allowedStatus = [ - 'active', - 'inactive' - ]; - - foreach ($items['bridges'] as $bridge) { - foreach ($expectedKeys as $key) { - $this->assertArrayHasKey($key, $bridge, 'Missing key "' . $key . '"'); - } - - $this->assertContains($bridge['status'], $allowedStatus, 'Invalid status value'); - } - } -} diff --git a/tests/BridgeCardTest.php b/tests/BridgeCardTest.php new file mode 100644 index 00000000..519ac7f1 --- /dev/null +++ b/tests/BridgeCardTest.php @@ -0,0 +1,60 @@ +assertSame('', BridgeCard::getInputAttributes([])); + $this->assertSame(' required pattern="\d+"', BridgeCard::getInputAttributes(['required' => true, 'pattern' => '\d+'])); + + $entry = [ + 'defaultValue' => 'checked', + ]; + $this->assertSame('' . "\n", BridgeCard::getCheckboxInput($entry, 'id', 'name')); + + $entry = [ + 'defaultValue' => 42, + 'exampleValue' => 43, + ]; + $this->assertSame('' . "\n", BridgeCard::getNumberInput($entry, 'id', 'name')); + + $entry = [ + 'defaultValue' => 'yo1', + 'exampleValue' => 'yo2', + ]; + $this->assertSame('', BridgeCard::getTextInput($entry, 'id', 'name')); + + $entry = [ + 'values' => [], + ]; + $this->assertSame('', BridgeCard::getListInput($entry, 'id', 'name')); + + $entry = [ + 'defaultValue' => 2, + 'values' => [ + 'foo' => 'bar', + ], + ]; + $this->assertSame('', BridgeCard::getListInput($entry, 'id', 'name')); + + // optgroup + $entry = [ + 'defaultValue' => 2, + 'values' => ['kek' => [ + 'f' => 'b', + ]], + ]; + $this->assertSame( + '', + BridgeCard::getListInput($entry, 'id', 'name') + ); + } +} diff --git a/tests/BridgeFactoryTest.php b/tests/BridgeFactoryTest.php index a97711ef..a12faf48 100644 --- a/tests/BridgeFactoryTest.php +++ b/tests/BridgeFactoryTest.php @@ -6,25 +6,14 @@ use PHPUnit\Framework\TestCase; class BridgeFactoryTest extends TestCase { - public function setUp(): void - { - \Configuration::loadConfiguration(); - } - public function testNormalizeBridgeName() { $this->assertSame('TwitterBridge', \BridgeFactory::normalizeBridgeName('TwitterBridge')); $this->assertSame('TwitterBridge', \BridgeFactory::normalizeBridgeName('TwitterBridge.php')); $this->assertSame('TwitterBridge', \BridgeFactory::normalizeBridgeName('Twitter')); - } - - public function testSanitizeBridgeName() - { - $sut = new \BridgeFactory(); - - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitterbridge')); - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitter')); - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('tWitTer')); - $this->assertSame('TwitterBridge', $sut->createBridgeClassName('TWITTERBRIDGE')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitterbridge')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('twitter')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('tWitTer')); +// $this->assertSame('TwitterBridge', $sut->createBridgeClassName('TWITTERBRIDGE')); } } diff --git a/tests/Bridges/BridgeImplementationTest.php b/tests/BridgeImplementationTest.php similarity index 64% rename from tests/Bridges/BridgeImplementationTest.php rename to tests/BridgeImplementationTest.php index cf03f356..dd68934e 100644 --- a/tests/Bridges/BridgeImplementationTest.php +++ b/tests/BridgeImplementationTest.php @@ -1,16 +1,15 @@ setBridge($path); - $this->assertTrue($this->class === ucfirst($this->class), 'class name must start with uppercase character'); - $this->assertEquals(0, substr_count($this->class, ' '), 'class name must not contain spaces'); - $this->assertStringEndsWith('Bridge', $this->class, 'class name must end with "Bridge"'); + $this->assertTrue($this->className === ucfirst($this->className), 'class name must start with uppercase character'); + $this->assertEquals(0, substr_count($this->className, ' '), 'class name must not contain spaces'); + $this->assertStringEndsWith('Bridge', $this->className, 'class name must end with "Bridge"'); } /** @@ -29,7 +28,7 @@ class BridgeImplementationTest extends TestCase public function testClassType($path) { $this->setBridge($path); - $this->assertInstanceOf(BridgeInterface::class, $this->obj); + $this->assertInstanceOf(BridgeAbstract::class, $this->bridge); } /** @@ -39,18 +38,18 @@ class BridgeImplementationTest extends TestCase { $this->setBridge($path); - $this->assertIsString($this->obj::NAME, 'class::NAME'); - $this->assertNotEmpty($this->obj::NAME, 'class::NAME'); - $this->assertIsString($this->obj::URI, 'class::URI'); - $this->assertNotEmpty($this->obj::URI, 'class::URI'); - $this->assertIsString($this->obj::DESCRIPTION, 'class::DESCRIPTION'); - $this->assertNotEmpty($this->obj::DESCRIPTION, 'class::DESCRIPTION'); - $this->assertIsString($this->obj::MAINTAINER, 'class::MAINTAINER'); - $this->assertNotEmpty($this->obj::MAINTAINER, 'class::MAINTAINER'); + $this->assertIsString($this->bridge::NAME, 'class::NAME'); + $this->assertNotEmpty($this->bridge::NAME, 'class::NAME'); + $this->assertIsString($this->bridge::URI, 'class::URI'); + $this->assertNotEmpty($this->bridge::URI, 'class::URI'); + $this->assertIsString($this->bridge::DESCRIPTION, 'class::DESCRIPTION'); + $this->assertNotEmpty($this->bridge::DESCRIPTION, 'class::DESCRIPTION'); + $this->assertIsString($this->bridge::MAINTAINER, 'class::MAINTAINER'); + $this->assertNotEmpty($this->bridge::MAINTAINER, 'class::MAINTAINER'); - $this->assertIsArray($this->obj::PARAMETERS, 'class::PARAMETERS'); - $this->assertIsInt($this->obj::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); - $this->assertGreaterThanOrEqual(0, $this->obj::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); + $this->assertIsArray($this->bridge::PARAMETERS, 'class::PARAMETERS'); + $this->assertIsInt($this->bridge::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); + $this->assertGreaterThanOrEqual(0, $this->bridge::CACHE_TIMEOUT, 'class::CACHE_TIMEOUT'); } /** @@ -61,23 +60,22 @@ class BridgeImplementationTest extends TestCase $this->setBridge($path); $multiMinimum = 2; - if (isset($this->obj::PARAMETERS['global'])) { + if (isset($this->bridge::PARAMETERS['global'])) { ++$multiMinimum; } - $multiContexts = (count($this->obj::PARAMETERS) >= $multiMinimum); + $multiContexts = (count($this->bridge::PARAMETERS) >= $multiMinimum); $paramsSeen = []; $allowedTypes = [ 'text', 'number', 'list', - 'checkbox' + 'checkbox', ]; - foreach ($this->obj::PARAMETERS as $context => $params) { + foreach ($this->bridge::PARAMETERS as $context => $params) { if ($multiContexts) { $this->assertIsString($context, 'invalid context name'); - $this->assertNotEmpty($context, 'The context name cannot be empty'); } @@ -153,31 +151,9 @@ class BridgeImplementationTest extends TestCase } } - foreach ($this->obj::TEST_DETECT_PARAMETERS as $url => $params) { - $this->assertEquals($this->obj->detectParameters($url), $params); - } - - $this->assertTrue(true); - } - - /** - * @dataProvider dataBridgesProvider - */ - public function testVisibleMethods($path) - { - $allowedBridgeAbstract = get_class_methods(BridgeAbstract::class); - sort($allowedBridgeAbstract); - $allowedFeedExpander = get_class_methods(FeedExpander::class); - sort($allowedFeedExpander); - - $this->setBridge($path); - - $methods = get_class_methods($this->obj); - sort($methods); - if ($this->obj instanceof FeedExpander) { - $this->assertEquals($allowedFeedExpander, $methods); - } else { - $this->assertEquals($allowedBridgeAbstract, $methods); + foreach ($this->bridge::TEST_DETECT_PARAMETERS as $url => $params) { + $detectedParameters = $this->bridge->detectParameters($url); + $this->assertEquals($detectedParameters, $params); } } @@ -188,23 +164,23 @@ class BridgeImplementationTest extends TestCase { $this->setBridge($path); - $value = $this->obj->getDescription(); + $value = $this->bridge->getDescription(); $this->assertIsString($value, '$class->getDescription()'); $this->assertNotEmpty($value, '$class->getDescription()'); - $value = $this->obj->getMaintainer(); + $value = $this->bridge->getMaintainer(); $this->assertIsString($value, '$class->getMaintainer()'); $this->assertNotEmpty($value, '$class->getMaintainer()'); - $value = $this->obj->getName(); + $value = $this->bridge->getName(); $this->assertIsString($value, '$class->getName()'); $this->assertNotEmpty($value, '$class->getName()'); - $value = $this->obj->getURI(); + $value = $this->bridge->getURI(); $this->assertIsString($value, '$class->getURI()'); $this->assertNotEmpty($value, '$class->getURI()'); - $value = $this->obj->getIcon(); + $value = $this->bridge->getIcon(); $this->assertIsString($value, '$class->getIcon()'); } @@ -215,14 +191,14 @@ class BridgeImplementationTest extends TestCase { $this->setBridge($path); - $this->checkUrl($this->obj::URI); - $this->checkUrl($this->obj->getURI()); + $this->assertNotFalse(filter_var($this->bridge::URI, FILTER_VALIDATE_URL)); + $this->assertNotFalse(filter_var($this->bridge->getURI(), FILTER_VALIDATE_URL)); } public function dataBridgesProvider() { $bridges = []; - foreach (glob(__DIR__ . '/../../bridges/*Bridge.php') as $path) { + foreach (glob(__DIR__ . '/../bridges/*Bridge.php') as $path) { $bridges[basename($path, '.php')] = [$path]; } return $bridges; @@ -230,13 +206,11 @@ class BridgeImplementationTest extends TestCase private function setBridge($path) { - $this->class = '\\' . basename($path, '.php'); - $this->assertTrue(class_exists($this->class), 'class ' . $this->class . ' doesn\'t exist'); - $this->obj = new $this->class(); - } - - private function checkUrl($url) - { - $this->assertNotFalse(filter_var($url, FILTER_VALIDATE_URL), 'no valid URL: ' . $url); + $this->className = '\\' . basename($path, '.php'); + $this->assertTrue(class_exists($this->className), 'class ' . $this->className . ' doesn\'t exist'); + $this->bridge = new $this->className( + new \NullCache(), + new \NullLogger(), + ); } } diff --git a/tests/CacheImplementationTest.php b/tests/CacheImplementationTest.php new file mode 100644 index 00000000..e6ed352b --- /dev/null +++ b/tests/CacheImplementationTest.php @@ -0,0 +1,36 @@ +assertTrue($path === ucfirst($path), 'class name must start with uppercase character'); + $this->assertEquals(0, substr_count($path, ' '), 'class name must not contain spaces'); + $this->assertStringEndsWith('Cache', $path, 'class name must end with "Cache"'); + } + + /** + * @dataProvider getCacheClassNames + */ + public function testClassType($path) + { + $this->assertTrue(is_subclass_of($path, CacheInterface::class), 'class must be subclass of CacheInterface'); + } +} diff --git a/tests/CacheTest.php b/tests/CacheTest.php index 9a8ada14..491db75a 100644 --- a/tests/CacheTest.php +++ b/tests/CacheTest.php @@ -8,13 +8,13 @@ class CacheTest extends TestCase { public function testConfig() { - $sut = new \FileCache(['path' => '/tmp/']); + $sut = new \FileCache(new \NullLogger(), ['path' => '/tmp/']); $this->assertSame(['path' => '/tmp/', 'enable_purge' => true], $sut->getConfig()); - $sut = new \FileCache(['path' => '/', 'enable_purge' => false]); + $sut = new \FileCache(new \NullLogger(), ['path' => '/', 'enable_purge' => false]); $this->assertSame(['path' => '/', 'enable_purge' => false], $sut->getConfig()); - $sut = new \FileCache(['path' => '/tmp', 'enable_purge' => true]); + $sut = new \FileCache(new \NullLogger(), ['path' => '/tmp', 'enable_purge' => true]); $this->assertSame(['path' => '/tmp/', 'enable_purge' => true], $sut->getConfig()); } @@ -23,21 +23,17 @@ class CacheTest extends TestCase $temporaryFolder = sprintf('%s/rss_bridge_%s/', sys_get_temp_dir(), create_random_string()); mkdir($temporaryFolder); - $sut = new \FileCache([ + $sut = new \FileCache(new \NullLogger(), [ 'path' => $temporaryFolder, 'enable_purge' => true, ]); - $sut->setScope('scope'); - $sut->purgeCache(-1); - $sut->setKey(['key']); + $sut->clear(); - $this->assertNull($sut->getTime()); - $this->assertNull($sut->loadData()); + $this->assertNull($sut->get('key')); - $sut->saveData('data'); - $this->assertSame('data', $sut->loadData()); - $this->assertIsNumeric($sut->getTime()); - $sut->purgeCache(-1); + $sut->set('key', 'data', 5); + $this->assertSame('data', $sut->get('key')); + $sut->clear(); // Intentionally not deleting the temp folder } diff --git a/tests/Caches/CacheImplementationTest.php b/tests/Caches/CacheImplementationTest.php deleted file mode 100644 index a3ad5f79..00000000 --- a/tests/Caches/CacheImplementationTest.php +++ /dev/null @@ -1,48 +0,0 @@ -setCache($path); - $this->assertTrue($this->class === ucfirst($this->class), 'class name must start with uppercase character'); - $this->assertEquals(0, substr_count($this->class, ' '), 'class name must not contain spaces'); - $this->assertStringEndsWith('Cache', $this->class, 'class name must end with "Cache"'); - } - - /** - * @dataProvider dataCachesProvider - */ - public function testClassType($path) - { - $this->setCache($path); - $this->assertTrue(is_subclass_of($this->class, CacheInterface::class), 'class must be subclass of CacheInterface'); - } - - //////////////////////////////////////////////////////////////////////////// - - public function dataCachesProvider() - { - $caches = []; - foreach (glob(PATH_LIB_CACHES . '*.php') as $path) { - $caches[basename($path, '.php')] = [$path]; - } - return $caches; - } - - private function setCache($path) - { - $this->class = '\\' . basename($path, '.php'); - $this->assertTrue(class_exists($this->class), 'class ' . $this->class . ' doesn\'t exist'); - } -} diff --git a/tests/FeedItemTest.php b/tests/FeedItemTest.php new file mode 100644 index 00000000..fbc7a4ea --- /dev/null +++ b/tests/FeedItemTest.php @@ -0,0 +1,19 @@ + 'kek', + ]; + $feedItem = \FeedItem::fromArray($item); + $this->assertSame('kek', $feedItem->getTitle()); + } +} diff --git a/tests/FeedParserTest.php b/tests/FeedParserTest.php new file mode 100644 index 00000000..458bdb53 --- /dev/null +++ b/tests/FeedParserTest.php @@ -0,0 +1,265 @@ +sut = new \FeedParser(); + } + + public function testRss1() + { + $xml = <<