From 9462f99fc9dcc17f8b73ef6b8de58a621394ce15 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 27 Jun 2021 11:27:15 -0700 Subject: [PATCH] Overhaul md list create script and instance.yaml See https://github.com/iv-org/documentation/pull/76#issuecomment-864416290 --- instances.yaml | 226 +++++++++++--------------------------- src/create-instance-md.py | 210 +++++++++++++++++++++++------------ src/requirements.txt | 1 + 3 files changed, 204 insertions(+), 233 deletions(-) diff --git a/instances.yaml b/instances.yaml index cb6f4c62..ac613175 100644 --- a/instances.yaml +++ b/instances.yaml @@ -19,9 +19,7 @@ instances: https: # Template instance # - url: https://example.com - # country: - # flag: 🇫🇮 - # name: Finland + # country: FI # status: # url: https://status.example.com # display_content: https://status-image.example.com @@ -30,16 +28,13 @@ instances: # privacy_policy: example.com/privacy # ddos_mitm_protection: null # owner: https://github.com/example - # modified: - # is_modified: False - # source_url: null + # is_modified: false + # source: null - url: https://invidious.snopyta.org - country: - flag: 🇫🇮 - name: Finland - status: + country: FI + status: url: https://status.unixfox.eu/783898765 display_content: https://img.shields.io/uptimerobot/status/m783898765-2a4efa67aa8d1c7be6b1dd9d display_content_is_image: True @@ -47,26 +42,20 @@ instances: privacy_policy: https://snopyta.org/privacy_policy ddos_mitm_protection: null owner: https://github.com/Perflyst - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://yewtu.be - country: - flag: 🇫🇷 - name: France + country: FR status: null privacy_policy: null ddos_mitm_protection: null owner: https://github.com/unixfox - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://invidious.kavin.rocks - country: - flag: 🇮🇳 - name: India + country: IN status: url: https://status.kavin.rocks/786132664 display_content: https://img.shields.io/uptimerobot/status/m786132664-f9fa738fba1c4dc2f7364f71 @@ -75,14 +64,11 @@ instances: privacy_policy: ddos_mitm_protection: Cloudflare owner: https://github.com/FireMasterK - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://invidious-us.kavin.rocks - country: - flag: 🇺🇸 - name: USA + country: US status: url: https://status.kavin.rocks/788216947 display_content: https://img.shields.io/uptimerobot/status/m788216947-f3f63d30899a10dbe9a0338a @@ -91,30 +77,11 @@ instances: privacy_policy: ddos_mitm_protection: Cloudflare owner: https://github.com/FireMasterK - modified: - is_modified: False - source_url: null - - - url: https://invidious.048596.xyz - country: - flag: 🇨🇦 - name: Canada - status: - url: https://status.048596.xyz/786792286 - display_content: https://img.shields.io/uptimerobot/status/m786792286-b5894e4e11c42b8332375076 - display_content_is_image: True - display_content_image_fallback: Uptime Robot status - privacy_policy: - ddos_mitm_protection: - owner: https://github.com/tenpura-shrimp - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://vid.puffyan.us - country: - flag: 🇺🇸 - name: USA + country: US status: url: https://stats.uptimerobot.com/n7A08HGVl6/786947233 display_content: https://img.shields.io/uptimerobot/status/m786947233-1131c3f67b9a20621b1926d3 @@ -123,42 +90,20 @@ instances: privacy_policy: ddos_mitm_protection: owner: https://github.com/ItsSt0ne - modified: - is_modified: False - source_url: null - - - url: https://ytprivate.com - country: - flag: 🇺🇸 - name: USA - status: - url: https://status.ytprivate.com/786947505 - display_content: https://img.shields.io/uptimerobot/status/m786947505-2a50cf3262906bb28c6cf8fc - display_content_is_image: True - display_content_image_fallback: Uptime Robot status - privacy_policy: - ddos_mitm_protection: DDoS-Guard - owner: https://github.com/ytprivatecom - modified: - is_modified: True - source_url: https://github.com/ytprivatecom/invidious + is_modified: false + source: null - url: https://invidious.namazso.eu - country: - flag: 🇩🇪 - name: Germany + country: DE status: privacy_policy: https://namazso.eu/privacy.html ddos_mitm_protection: owner: https://github.com/namazso - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://invidious.silkky.cloud - country: - flag: 🇫🇮 - name: Finland + country: FI status: url: https://status.silkky.cloud/787784614 display_content: https://img.shields.io/uptimerobot/status/m787784614-79d1acc4b425d1ed813fc793 @@ -167,14 +112,12 @@ instances: privacy_policy: ddos_mitm_protection: owner: https://github.com/TheSilkky - modified: - is_modified: False - source_url: null + is_modified: false + source: null + - url: https://invidious.exonip.de - country: - flag: 🇩🇪 - name: Germany + country: DE status: url: https://status.exonip.de/ display_content: Uptime Robot status @@ -182,65 +125,48 @@ instances: privacy_policy: ddos_mitm_protection: owner: https://github.com/Exonip - modified: - is_modified: True - source_url: https://github.com/exonip-de/invidious-source-modifications + is_modified: True + source: https://github.com/exonip-de/invidious-source-modifications - url: https://inv.riverside.rocks - country: - flag: 🇺🇸 - name: USA + country: US status: privacy_policy: ddos_mitm_protection: owner: https://github.com/RiversideRocks - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://invidious.blamefran.net - country: - flag: 🇺🇸 - name: USA + country: US status: privacy_policy: ddos_mitm_protection: owner: https://github.com/Aidan16 - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://invidious.moomoo.me - country: - flag: 🇩🇪 - name: Germany + country: DE status: privacy_policy: ddos_mitm_protection: Cloudflare owner: https://github.com/moom0o - modified: - is_modified: False - source_url: null - + is_modified: false + source: null - url: https://ytb.trom.tf - - country: - flag: 🇩🇪 - name: Germany + country: DE status: privacy_policy: ddos_mitm_protection: owner: https://github.com/TROMsite - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://yt.cyberhost.uk/ - country: - flag: 🇫🇮 - name: Finland + country: FI status: url: https://stats.uptimerobot.com/JlM0qH8Ygn display_content: https://img.shields.io/uptimerobot/status/m788432154-c8801112193f349268ea6104 @@ -250,86 +176,66 @@ instances: ddos_mitm_protection: owner: https://github.com/cyberhost-uk modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: https://y.com.cm/ - country: - flag: 🇩🇪 - name: Germany + country: DE status: privacy_policy: ddos_mitm_protection: Cloudflare owner: https://github.com/Showfom - modified: - is_modified: False - source_url: null + is_modified: false + source: null onion: # Template onion instance - # - url: http://example.onion - # country: - # flag: 🇺🇸 - # name: USA + # url: http://example.onion + # country: FI # associated_clearnet_instance: https://example.com # privacy_policy: https://privacy.example.com # owner: https://github.com/example # modified: - # is_modified: False - # source_url: null + # is_modified: false + # source: null - url: http://c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid.onion - country: - flag: 🇫🇮 - name: Finland + country: FI associated_clearnet_instance: https://invidious.snopyta.org privacy_policy: owner: https://github.com/Perflyst - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: http://w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd.onion/ - country: - flag: 🇮🇳 - name: India + country: IN associated_clearnet_instance: https://invidious.kavin.rocks privacy_policy: owner: https://github.com/FireMasterK - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: http://kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad.onion/ - country: - flag: 🇳🇱 - name: Netherlands + country: NL associated_clearnet_instance: privacy_policy: owner: https://github.com/tirz - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: http://grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad.onion/ - country: - flag: 🇺🇸 - name: USA + country: US associated_clearnet_instance: https://vid.puffyan.us privacy_policy: owner: https://github.com/ItsSt0ne - modified: - is_modified: False - source_url: null + is_modified: false + source: null - url: http://hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid.onion/ - country: - flag: 🇺🇸 - name: USA + country: US associated_clearnet_instance: https://invidious-us.kavin.rocks/ privacy_policy: owner: https://github.com/FireMasterK - modified: - is_modified: False - source_url: null + is_modified: false + source: null diff --git a/src/create-instance-md.py b/src/create-instance-md.py index eef0a7f4..1fa6b80c 100644 --- a/src/create-instance-md.py +++ b/src/create-instance-md.py @@ -1,92 +1,156 @@ """Extremely quick and dirty module for creating a markdown file from the instances.yaml file""" from urllib.parse import urlparse +import pycountry import yaml from mdutils.mdutils import MdUtils -def create_table(table_data, instance_data): - rows = [] - for field, value in instance_data.items(): - if value is None: - rows.append("") - - # Use markdown links for Addresses - elif field in ["url", "associated_clearnet_instance"]: - url = urlparse(value).hostname - rows.append(f"[{url}]({value})") - - elif field == "status" and value is not None: - if value.get("display_content_is_image"): - rows.append(f"[![{value['display_content_image_fallback']}]({value['display_content']})]({value['url']})") - else: - rows.append(f"[{value['display_content']}]({value['url']})") - - elif field == "country" and value: - rows.append(f"{value['flag']} {value['name']}") - - elif field == "modified": - if value.get("is_modified"): - rows.append(f"[Yes]({value['source_url']})") - else: - rows.append("No") - - # We're going to use a markdown link here - elif field == "privacy_policy": - rows.append(f"[Here]({value})") - - # Handle author name - elif field == "owner": - # Assuming github url - author_name = value.split("/") - rows.append(f"[@{author_name[-1]}]({value})") - else: - rows.append(value) - - table_data.extend(rows) - +class ColumnBuilder: + def route(self, name): + router = { + "url": self._create_url_column, + "associated_clearnet_instance": self._create_url_column, + "country": self._create_country_column, + "status": self._create_status_column, + "is_modified": self._create_modified_column, + "privacy_policy": self._create_privacy_policy_column, + "ddos_mitm_protection": self.ddos_protection, + "owner": self._create_owner_column + } -with open("instances.yaml") as configuration_yaml: - data = yaml.safe_load(configuration_yaml) + return router[name] -instance_list = data["instances"] + @staticmethod + def _create_url_column(url): + if not url: + return "" -# Initial information -md_instance_list = MdUtils(file_name='Invidious-Instances.md') -md_instance_list.new_header(level=1, title='Public Instances') -md_instance_list.new_paragraph("Uptime History: [uptime.invidious.io](https://uptime.invidious.io)") -md_instance_list.new_paragraph("Instances API: [api.invidious.io](api.invidious.io)") + hostname = urlparse(url).hostname + return f"[{hostname}]({url})" + @staticmethod + def _create_country_column(country_code): + country_name = pycountry.countries.get(alpha_2=country_code).name + # https://stackoverflow.com/a/42235254 + flag = (lambda s, e: chr(ord(s.upper()) - 0x41 + 0x1F1E6) + + chr(ord(e.upper()) - 0x41 + 0x1F1E6) + )(*list(country_code)) -# Clearnet instances -md_instance_list.new_header(level=1, title='Instances list') -table_data = ["Address", "Country", "Status", "Privacy policy", "DDos Protection / MITM", "Owner", "Modified"] -for instance_data in instance_list["https"]: - create_table(table_data, instance_data) + return f"{flag} {country_name}" -md_instance_list.new_line() -md_instance_list.new_table(columns=7, rows=len(instance_list["https"]) + 1, text=table_data, text_align='center') + @staticmethod + def _create_status_column(status_dict): + if not status_dict: + return "" + status_url = status_dict["url"] + display_content = status_dict["display_content"] -# Onion instances -md_instance_list.new_header(level=1, title='Tor onion instances list') -table_data = ["Address", "Country", "Associated clearnet instance", "Privacy policy", "Owner", "Modified"] -for instance_data in instance_list["onion"]: - create_table(table_data, instance_data) - -md_instance_list.new_line() -md_instance_list.new_table(columns=6, rows=len(instance_list["onion"]) + 1, text=table_data, text_align='center') + if status_dict.get("display_content_is_image"): + fallback = status_dict["display_content_image_fallback"] + return f"[![{fallback}]({display_content})]({status_url})" + else: + return f"[{display_content}]({status_url})" + + @staticmethod + def _create_modified_column(is_modified, source): + if is_modified: + return f"[Yes]({source})" + return f"No" + + @staticmethod + def _create_privacy_policy_column(privacy): + return f"[Here]({privacy})" + + @staticmethod + def ddos_protection(ddos): + return f"{ddos}" + + @staticmethod + def _create_owner_column(owner): + author_name = owner.split("/") + return f"[@{author_name[-1]}]({owner})" + + +class MDInstanceListBuilder: + def __init__(self, instance_list_config): + self.config = instance_list_config + self.md = MdUtils(file_name='Invidious-Instances.md') + self.builder = ColumnBuilder() + + def _generate_heading(self): + self.md.new_header(level=1, title='Public Instances') + self.md.new_paragraph("Uptime History: [uptime.invidious.io](https://uptime.invidious.io)") + self.md.new_paragraph("Instances API: [api.invidious.io](api.invidious.io)") + + def _create_instance_tables(self): + # HTTPS + self.md.new_header(level=1, title='Instances list') + rows = ["Address", "Country", "Status", "Privacy policy", "DDos Protection / MITM", "Owner", "Modified"] + for instance in self.config["instances"]["https"]: + rows.extend(self._create_http_row(instance)) + self.md.new_table(columns=7, rows=len(self.config["instances"]["https"]) + 1, text=rows, text_align='center') + + self.md.new_line() + + # Onion + self.md.new_header(level=1, title='Onion instances list') + rows = ["Address", "Country", "Associated clearnet instance", "Privacy policy", "Owner", "Modified"] + for instance in self.config["instances"]["onion"]: + rows.extend(self._create_onion_row(instance)) + self.md.new_table(columns=6, rows=len(self.config["instances"]["onion"]) + 1, text=rows, text_align='center') + + def _create_prerequisite_list(self): + self.md.new_header(level=2, title='Prerequisites') + self.md.new_list(self.config["adding_instance"]["prerequisites"]) + self.md.new_line() + + def _create_direction_list(self): + self.md.new_header(level=2, title='Directions') + self.md.new_list(self.config["adding_instance"]["directions"], marked_with="1") + self.md.create_md_file() + + def _create_http_row(self, instance): + return [ + self.builder.route("url")(instance["url"]), + self.builder.route("country")(instance["country"]), + self.builder.route("status")(instance["status"]), + self.builder.route("privacy_policy")(instance["privacy_policy"]), + self.builder.route("ddos_mitm_protection")(instance["ddos_mitm_protection"]), + self.builder.route("owner")(instance["owner"]), + self.builder.route("is_modified")(instance["is_modified"], instance["source"]) + ] + + def _create_onion_row(self, instance): + return [ + self.builder.route("url")(instance["url"]), + self.builder.route("country")(instance["country"]), + self.builder.route("associated_clearnet_instance")(instance["associated_clearnet_instance"]), + self.builder.route("privacy_policy")(instance["privacy_policy"]), + self.builder.route("owner")(instance["owner"]), + self.builder.route("is_modified")(instance["is_modified"], instance["source"]) + ] + + def create(self): + self._generate_heading() + self.md.new_line() + + self._create_instance_tables() + self.md.new_line() + + # Instance adding directions and prerequisites + self.md.new_header(level=1, title='Adding your instance') + self._create_prerequisite_list() + self.md.new_line() + self._create_direction_list() + + self.md.create_md_file() -# Instance adding directions and prerequisites -md_instance_list.new_header(level=1, title='Adding your instance') -# Prerequisites -md_instance_list.new_header(level=2, title='Prerequisites') -md_instance_list.new_list(data["adding_instance"]["prerequisites"]) -md_instance_list.new_line() +with open("instances.yaml") as configuration_yaml: + data = yaml.safe_load(configuration_yaml) -# Directions -md_instance_list.new_header(level=2, title='Directions') -md_instance_list.new_list(data["adding_instance"]["directions"], marked_with="1") -md_instance_list.create_md_file() +md_builder = MDInstanceListBuilder(data) +md_builder.create() \ No newline at end of file diff --git a/src/requirements.txt b/src/requirements.txt index 9f30f687..c607ad16 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,2 +1,3 @@ mdutils==1.3.0 +pycountry==20.7.3 PyYAML==5.4.1