Rewrite parser for new instance list fmt

iv-org/documentation#76

This also contains a breaking changes to the API:
  - The flag and region values has been moved. They're now within the
    country dict on the JSON response

src/fetch.cr

@@ -0,0 +1,79 @@
+private def fetch_country(instance_data)
+  region = instance_data["country"]["flag"]?.try { |region| region.to_s.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }.to_s
+  flag = instance_data["country"]["flag"].to_s
+  country_name = instance_data["country"]["name"].to_s
+
+  return {flag: flag, region: region, name: country_name}
+end
+
+def prepare_http_instance(instance_data, instances_storage, monitors)
+  uri = URI.parse(instance_data["url"].to_s)
+  host = uri.host
+
+  country = fetch_country(instance_data)
+
+  begin
+    status_url = instance_data["status"].as_h["url"].to_s
+  rescue TypeCastError
+    status_url = nil
+  end
+
+  privacy_policy = instance_data["privacy_policy"].to_s
+  ddos_mitm_protection = instance_data["ddos_mitm_protection"].to_s
+  owner = {name: instance_data["owner"].to_s.split("/")[-1].to_s, url: instance_data["owner"].to_s}
+
+  is_modified = instance_data["modified"]["is_modified"].as_bool
+  source_url = instance_data["modified"]["source_url"]?.try &.to_s || nil
+
+  client = HTTP::Client.new(uri)
+  client.connect_timeout = 5.seconds
+  client.read_timeout = 5.seconds
+
+  begin
+    stats = JSON.parse(client.get("/api/v1/stats").body)
+  rescue ex
+    stats = nil
+  end
+
+  monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
+  return {country: country, stats: stats, type: "https", uri: uri.to_s, status_url: status_url,
+          privacy_policy: privacy_policy, ddos_mitm_protection: ddos_mitm_protection,
+          owner: owner, modified: {is_modified: is_modified, source_url: source_url},
+          monitor: monitor || instances_storage[host]?.try &.[:monitor]?}
+end
+
+def prepare_onion_instance(instance_data, instances_storage)
+  uri = URI.parse(instance_data["url"].to_s)
+  host = uri.host
+
+  country = fetch_country(instance_data)
+
+  associated_clearnet_instance = instance_data["associated_clearnet_instance"]?.try &.to_s || nil
+  privacy_policy = instance_data["privacy_policy"].to_s
+  owner = {name: instance_data["owner"].to_s.split("/")[-1].to_s, url: instance_data["owner"].to_s}
+
+  is_modified = instance_data["modified"]["is_modified"].as_bool
+  source_url = instance_data["modified"]["source_url"]?.try &.to_s || nil
+
+  if CONFIG["fetch_onion_instance_stats"]?
+    begin
+      args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
+      response = nil
+      Process.run("curl", args: args) do |result|
+        data = result.output.read_line
+        response = JSON.parse(data)
+      end
+
+      stats = response
+    rescue ex
+      stats = nil
+    end
+  else
+    stats = nil
+  end
+
+  return {country: country, stats: stats, type: "https", uri: uri.to_s, associated_clearnet_instance: associated_clearnet_instance,
+          privacy_policy: privacy_policy,
+          owner: owner, modified: {is_modified: is_modified, source_url: source_url},
+          monitor: nil}
+end

src/helpers/helpers.cr

@@ -1,6 +1,9 @@
 require "yaml"
 
 def load_config
-  config = YAML.parse(File.read("config.yml"))
-  return config
+  return YAML.parse(File.read("config.yml"))
+end
+
+def load_instance_yaml(contents)
+  return YAML.parse(contents)
 end

src/instances.cr

@@ -18,6 +18,7 @@ require "http/client"
 require "kemal"
 require "uri"
 
+require "./fetch.cr"
 require "./helpers/*"
 
 CONFIG = load_config()
@@ -28,9 +29,14 @@ macro rendered(filename)
   render "src/instances/views/#{{{filename}}}.ecr"
 end
 
-alias Instance = NamedTuple(flag: String?, region: String?, stats: JSON::Any?, type: String, uri: String, monitor: JSON::Any?)
+alias Owner = NamedTuple(name: String, url: String)
+alias Country = NamedTuple(flag: String, region: String, name: String)
+alias Modified = NamedTuple(is_modified: Bool, source_url: String?)
 
-INSTANCES = {} of String => Instance
+alias ClearNetInstance = NamedTuple(country: Country, stats: JSON::Any?, type: String, uri: String, status_url: String?, privacy_policy: String?, ddos_mitm_protection: String?, owner: Owner, modified: Modified, monitor: JSON::Any?)
+alias OnionInstance = NamedTuple(country: Country, stats: JSON::Any?, type: String, uri: String, associated_clearnet_instance: String?, privacy_policy: String?, owner: Owner, modified: Modified, monitor: JSON::Any?)
+
+INSTANCES = {} of String => ClearNetInstance | OnionInstance
 
 spawn do
   loop do
@@ -54,59 +60,23 @@ spawn do
         break
       end
     end
+
     begin
-      body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md")).body
+      # Needs to be replaced once merged!
+      body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/syeopite/documentation/alt-instance-list/instances.yaml")).body
     rescue ex
       body = ""
     end
 
-    instances = {} of String => Instance
-
-    body = body.split("### Blocked:")[0]
-    body.scan(/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)( .(?<region>[\x{1f100}-\x{1f1ff}]{2}))?/mx).each do |md|
-      region = md["region"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
-      flag = md["region"]?
-
-      uri = URI.parse(md["uri"])
-      host = md["host"]
-
-      case type = host.split(".")[-1]
-      when "onion"
-        type = "onion"
-
-        if CONFIG["fetch_onion_instance_stats"]?
-          begin
-            args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
-            response = nil
-            Process.run("curl", args: args) do |result|
-              data = result.output.read_line
-              response = JSON.parse(data)
-            end
-
-            stats = response
-          rescue ex
-            stats = nil
-          end
-        end
-      when "i2p"
-      else
-        type = uri.scheme.not_nil!
-        client = HTTP::Client.new(uri)
-        client.connect_timeout = 5.seconds
-        client.read_timeout = 5.seconds
-        begin
-          stats = JSON.parse(client.get("/api/v1/stats").body)
-        rescue ex
-          stats = nil
-        end
-      end
+    instance_yaml = load_instance_yaml(body)
 
-      monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
-      instances[host] = {flag: flag, region: region, stats: stats, type: type, uri: uri.to_s, monitor: monitor || instances[host]?.try &.[:monitor]?}
-    end
+    instance_storage = {} of String => ClearNetInstance | OnionInstance
+
+    instance_yaml["instances"]["https"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_http_instance(i, instance_storage, monitors) }
+    instance_yaml["instances"]["onion"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_onion_instance(i, instance_storage) }
 
     INSTANCES.clear
-    INSTANCES.merge! instances
+    INSTANCES.merge! instance_storage
 
     sleep CONFIG["minutes_between_refresh"].as_i.minutes
   end
@@ -154,13 +124,13 @@ static_headers do |response, filepath, filestat|
 end
 
 SORT_PROCS = {
-  "health"   => ->(name : String, instance : Instance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
-  "location" => ->(name : String, instance : Instance) { instance[:region]? || "ZZ" },
-  "name"     => ->(name : String, instance : Instance) { name },
-  "signup"   => ->(name : String, instance : Instance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
-  "type"     => ->(name : String, instance : Instance) { instance[:type] },
-  "users"    => ->(name : String, instance : Instance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
-  "version"  => ->(name : String, instance : Instance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
+  "health"   => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
+  "location" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:country][:region]? || "ZZ" },
+  "name"     => ->(name : String, instance : ClearNetInstance | OnionInstance) { name },
+  "signup"   => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
+  "type"     => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:type] },
+  "users"    => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
+  "version"  => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
 }
 
 def sort_instances(instances, sort_by)

src/instances/views/index.ecr

@@ -118,7 +118,7 @@
               <td><%= instance[:type] %></td>
               <td><%= instance[:stats]?.try &.["usage"]?.try &.["users"]["total"] || "-" %></td>
               <td><%= instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? "✔" : "❌" } || "-" %></td>
-              <td><%= instance[:flag]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %></td>
+              <td><%= instance[:country]? ? "#{instance[:country][:flag]} #{instance[:country][:name]}" : "-" %></td>
               <td><%= instance[:monitor]?.try &.["30dRatio"]["ratio"] || "-" %></td>
             </tr>
           <% end %>