Cria spider para Duque de Caxias RJ no padrão atual do projeto

data_collection/gazette/spiders/rj/rj_duque_de_caxias.py

@@ -0,0 +1,56 @@
+import re
+from datetime import date, datetime
+
+from scrapy import Request
+
+from gazette.items import Gazette
+from gazette.spiders.base import BaseGazetteSpider
+from gazette.utils import date_from_text_with_fuzzy_match
+
+
+class RjDuqueDeCaxiasSpider(BaseGazetteSpider):
+    name = "rj_duque_de_caxias"
+    TERRITORY_ID = "3301702"
+    allowed_domains = ["duquedecaxias.rj.gov.br"]
+    start_date = date(2017, 1, 2)
+
+    def start_requests(self):
+        current_year = datetime.today().year
+        for year in range(self.start_date.year, self.end_date.year + 1):
+            if year == current_year:
+                yield Request(
+                    "https://duquedecaxias.rj.gov.br/portal/boletim-oficial.html"
+                )
+            else:
+                yield Request(f"https://duquedecaxias.rj.gov.br/portal/{year}.html")
+
+    def parse(self, response):
+        pdf_divs = response.xpath(
+            "//i[contains(@class, 'fa-file-pdf')]/ancestor::div[1]"
+        )
+        # use descending order to reduce iterations in the daily crawl
+        for pdf_div in pdf_divs[::-1]:
+            raw_gazette_date = pdf_div.xpath("./preceding-sibling::div[1]/text()").get()
+            gazette_date = date_from_text_with_fuzzy_match(raw_gazette_date)
+            if not gazette_date or gazette_date > self.end_date:
+                continue
+            if gazette_date < self.start_date:
+                return
+
+            raw_gazette_edtion = pdf_div.xpath("./preceding-sibling::div[2]/text()")
+            is_extra_edition = bool(
+                re.search(r"extra|esp", raw_gazette_edtion.get(), re.IGNORECASE)
+            )
+            edition_number = (
+                None if is_extra_edition else raw_gazette_edtion.re_first(r"\d+")
+            )
+
+            gazette_url = response.urljoin(pdf_div.xpath(".//a/@href").get())
+
+            yield Gazette(
+                date=gazette_date,
+                edition_number=edition_number,
+                is_extra_edition=is_extra_edition,
+                file_urls=[gazette_url],
+                power="executive_legislative",
+            )

data_collection/gazette/utils.py

@@ -1,8 +1,27 @@
+import re
+
+import dateparser
+from fuzzywuzzy import process
 from sqlalchemy import create_engine, select
 from sqlalchemy.orm import sessionmaker
 
 from gazette.database.models import QueridoDiarioSpider
 
+MONTHS = [
+    "janeiro",
+    "fevereiro",
+    "março",
+    "abril",
+    "maio",
+    "junho",
+    "julho",
+    "agosto",
+    "setembro",
+    "outubro",
+    "novembro",
+    "dezembro",
+]
+
 
 def get_enabled_spiders(*, database_url, start_date=None, end_date=None):
     """Return list of all currently enabled spiders within date period.
@@ -22,3 +41,30 @@ def get_enabled_spiders(*, database_url, start_date=None, end_date=None):
     result = session.execute(stmt)
     for spider in result.scalars():
         yield spider.spider_name
+
+
+def date_from_text_with_fuzzy_match(text):
+    """Extract a date from a text. This method attempts to correct typing errors in the month.
+
+    Args:
+        text: A text containing a date with the name of the month full version (%B)
+    Returns:
+        The date, if match. Otherwise, returns None.
+    """
+
+    text = re.sub(" +", " ", text).strip()
+    match_date = re.search(r"\d{1,2}º?(\sde)? +(\w+)(\sde)? +\d{4}", text)
+    if not match_date:
+        return None
+
+    raw_date = match_date.group(0)
+    raw_date = raw_date.replace("º", "").replace("°", "")
+    month = match_date.group(2)
+    if month.lower() not in MONTHS:
+        match_month, score = process.extractOne(month, MONTHS)
+        if score < 70:
+            return None
+        raw_date = raw_date.replace(month, match_month)
+
+    parsed_datetime = dateparser.parse(raw_date, languages=["pt"])
+    return parsed_datetime.date() if parsed_datetime else None

data_collection/requirements-dev.txt

@@ -257,6 +257,9 @@ flake8==6.1.0 \
 fqdn==1.5.1 \
     --hash=sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f \
     --hash=sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014
+fuzzywuzzy==0.18.0 \
+    --hash=sha256:45016e92264780e58972dca1b3d939ac864b78437422beecebb3095f8efd00e8 \
+    --hash=sha256:928244b28db720d1e0ee7587acf660ea49d7e4c632569cad4f1cd7e68a5f0993
 greenlet==3.0.0 \
     --hash=sha256:02a807b2a58d5cdebb07050efe3d7deaf915468d112dfcf5e426d0564aa3aa4a \
     --hash=sha256:0b72b802496cccbd9b31acea72b6f87e7771ccfd7f7927437d592e5c92ed703c \
@@ -359,6 +362,95 @@ jsonschema[format]==4.19.1 \
 jsonschema-specifications==2023.7.1 \
     --hash=sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1 \
     --hash=sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb
+levenshtein==0.26.1 \
+    --hash=sha256:04b7cabb82edf566b1579b3ed60aac0eec116655af75a3c551fee8754ffce2ea \
+    --hash=sha256:07227281e12071168e6ae59238918a56d2a0682e529f747b5431664f302c0b42 \
+    --hash=sha256:0ae7cd6e4312c6ef34b2e273836d18f9fff518d84d823feff5ad7c49668256e0 \
+    --hash=sha256:0d19ba22330d50609b2349021ec3cf7d905c6fe21195a2d0d876a146e7ed2575 \
+    --hash=sha256:0eb3059f826f6cb0a5bca4a85928070f01e8202e7ccafcba94453470f83e49d4 \
+    --hash=sha256:1010814b1d7a60833a951f2756dfc5c10b61d09976ce96a0edae8fecdfb0ea7c \
+    --hash=sha256:13946e37323728695ba7a22f3345c2e907d23f4600bc700bf9b4352fb0c72a48 \
+    --hash=sha256:18f00a3eebf68a82fb651d8d0e810c10bfaa60c555d21dde3ff81350c74fb4c2 \
+    --hash=sha256:219c30be6aa734bf927188d1208b7d78d202a3eb017b1c5f01ab2034d2d4ccca \
+    --hash=sha256:2532f8a13b68bf09f152d906f118a88da2063da22f44c90e904b142b0a53d534 \
+    --hash=sha256:270d36c5da04a0d89990660aea8542227cbd8f5bc34e9fdfadd34916ff904520 \
+    --hash=sha256:2c23e749b68ebc9a20b9047317b5cd2053b5856315bc8636037a8adcbb98bed1 \
+    --hash=sha256:3351ddb105ef010cc2ce474894c5d213c83dddb7abb96400beaa4926b0b745bd \
+    --hash=sha256:33e2517e8d3c221de2d1183f400aed64211fcfc77077b291ed9f3bb64f141cdc \
+    --hash=sha256:33fa329d1bb65ce85e83ceda281aea31cee9f2f6e167092cea54f922080bcc66 \
+    --hash=sha256:3535ecfd88c9b283976b5bc61265855f59bba361881e92ed2b5367b6990c93fe \
+    --hash=sha256:397e245e77f87836308bd56305bba630010cd8298c34c4c44bd94990cdb3b7b1 \
+    --hash=sha256:42d6fa242e3b310ce6bfd5af0c83e65ef10b608b885b3bb69863c01fb2fcff98 \
+    --hash=sha256:44c51f5d33b3cfb9db518b36f1288437a509edd82da94c4400f6a681758e0cb6 \
+    --hash=sha256:480674c05077eeb0b0f748546d4fcbb386d7c737f9fff0010400da3e8b552942 \
+    --hash=sha256:48825c9f967f922061329d1481b70e9fee937fc68322d6979bc623f69f75bc91 \
+    --hash=sha256:488a945312f2f16460ab61df5b4beb1ea2254c521668fd142ce6298006296c98 \
+    --hash=sha256:4e9de292f2c51a7d34a0ae23bec05391b8f61f35781cd3e4c6d0533e06250c55 \
+    --hash=sha256:4f052c55046c2a9c9b5f742f39e02fa6e8db8039048b8c1c9e9fdd27c8a240a1 \
+    --hash=sha256:51ba374de7a1797d04a14a4f0ad3602d2d71fef4206bb20a6baaa6b6a502da58 \
+    --hash=sha256:55b1024516c59df55f1cf1a8651659a568f2c5929d863d3da1ce8893753153bd \
+    --hash=sha256:56b93203e725f9df660e2afe3d26ba07d71871b6d6e05b8b767e688e23dfb076 \
+    --hash=sha256:69229d651c97ed5b55b7ce92481ed00635cdbb80fbfb282a22636e6945dc52d5 \
+    --hash=sha256:6c389e44da12d6fb1d7ba0a709a32a96c9391e9be4160ccb9269f37e040599ee \
+    --hash=sha256:6cf8f1efaf90ca585640c5d418c30b7d66d9ac215cee114593957161f63acde0 \
+    --hash=sha256:6f53f9173ae21b650b4ed8aef1d0ad0c37821f367c221a982f4d2922b3044e0d \
+    --hash=sha256:790374a9f5d2cbdb30ee780403a62e59bef51453ac020668c1564d1e43438f0e \
+    --hash=sha256:798f2b525a2e90562f1ba9da21010dde0d73730e277acaa5c52d2a6364fd3e2a \
+    --hash=sha256:79dcd157046d62482a7719b08ba9e3ce9ed3fc5b015af8ea989c734c702aedd4 \
+    --hash=sha256:79dd6ad799784ea7b23edd56e3bf94b3ca866c4c6dee845658ee75bb4aefdabf \
+    --hash=sha256:7b05c0415c386d00efda83d48db9db68edd02878d6dbc6df01194f12062be1bb \
+    --hash=sha256:7b50c3620ff47c9887debbb4c154aaaac3e46be7fc2e5789ee8dbe128bce6a17 \
+    --hash=sha256:7e82f2ea44a81ad6b30d92a110e04cd3c8c7c6034b629aca30a3067fa174ae89 \
+    --hash=sha256:8191241cd8934feaf4d05d0cc0e5e72877cbb17c53bbf8c92af9f1aedaa247e9 \
+    --hash=sha256:81c57e1135c38c5e6e3675b5e2077d8a8d3be32bf0a46c57276c092b1dffc697 \
+    --hash=sha256:821ace3b4e1c2e02b43cf5dc61aac2ea43bdb39837ac890919c225a2c3f2fea4 \
+    --hash=sha256:8adcc90e3a5bfb0a463581d85e599d950fe3c2938ac6247b29388b64997f6e2d \
+    --hash=sha256:8dc4a4aecad538d944a1264c12769c99e3c0bf8e741fc5e454cc954913befb2e \
+    --hash=sha256:8f80dd7432d4b6cf493d012d22148db7af769017deb31273e43406b1fb7f091c \
+    --hash=sha256:90236e93d98bdfd708883a6767826fafd976dac8af8fc4a0fb423d4fa08e1bf0 \
+    --hash=sha256:9092b622765c7649dd1d8af0f43354723dd6f4e570ac079ffd90b41033957438 \
+    --hash=sha256:91d5e7d984891df3eff7ea9fec8cf06fdfacc03cd074fd1a410435706f73b079 \
+    --hash=sha256:9895b3a98f6709e293615fde0dcd1bb0982364278fa2072361a1a31b3e388b7a \
+    --hash=sha256:9d12c8390f156745e533d01b30773b9753e41d8bbf8bf9dac4b97628cdf16314 \
+    --hash=sha256:9d87215113259efdca8716e53b6d59ab6d6009e119d95d45eccc083148855f33 \
+    --hash=sha256:9e70d7ee157a9b698c73014f6e2b160830e7d2d64d2e342fefc3079af3c356fc \
+    --hash=sha256:9f942104adfddd4b336c3997050121328c39479f69de702d7d144abb69ea7ab9 \
+    --hash=sha256:9fb859da90262eb474c190b3ca1e61dee83add022c676520f5c05fdd60df902a \
+    --hash=sha256:a19862e3539a697df722a08793994e334cd12791e8144851e8a1dee95a17ff63 \
+    --hash=sha256:a323161dfeeac6800eb13cfe76a8194aec589cd948bcf1cdc03f66cc3ec26b72 \
+    --hash=sha256:a3777de1d8bfca054465229beed23994f926311ce666f5a392c8859bb2722f16 \
+    --hash=sha256:ae382af8c76f6d2a040c0d9ca978baf461702ceb3f79a0a3f6da8d596a484c5b \
+    --hash=sha256:aeff6ea3576f72e26901544c6c55c72a7b79b9983b6f913cba0e9edbf2f87a97 \
+    --hash=sha256:b01061d377d1944eb67bc40bef5d4d2f762c6ab01598efd9297ce5d0047eb1b5 \
+    --hash=sha256:b3554c1b59de63d05075577380340c185ff41b028e541c0888fddab3c259a2b4 \
+    --hash=sha256:b817376de4195a207cc0e4ca37754c0e1e1078c2a2d35a6ae502afde87212f9e \
+    --hash=sha256:b8b68295808893a81e0a1dbc2274c30dd90880f14d23078e8eb4325ee615fc68 \
+    --hash=sha256:b952b3732c4631c49917d4b15d78cb4a2aa006c1d5c12e2a23ba8e18a307a055 \
+    --hash=sha256:bb6c7347424a91317c5e1b68041677e4c8ed3e7823b5bbaedb95bffb3c3497ea \
+    --hash=sha256:c1d8f85b2672939f85086ed75effcf768f6077516a3e299c2ba1f91bc4644c22 \
+    --hash=sha256:c2599407e029865dc66d210b8804c7768cbdbf60f061d993bb488d5242b0b73e \
+    --hash=sha256:c3114586032361722ddededf28401ce5baf1cf617f9f49fb86b8766a45a423ff \
+    --hash=sha256:cc741ca406d3704dc331a69c04b061fc952509a069b79cab8287413f434684bd \
+    --hash=sha256:ceb673f572d1d0dc9b1cd75792bb8bad2ae8eb78a7c6721e23a3867d318cb6f2 \
+    --hash=sha256:d5b2953978b8c158dd5cd93af8216a5cfddbf9de66cf5481c2955f44bb20767a \
+    --hash=sha256:d8ec137170b95736842f99c0e7a9fd8f5641d0c1b63b08ce027198545d983e2b \
+    --hash=sha256:dc3b5a64f57c3c078d58b1e447f7d68cad7ae1b23abe689215d03fc434f8f176 \
+    --hash=sha256:dc54ced948fc3feafce8ad4ba4239d8ffc733a0d70e40c0363ac2a7ab2b7251e \
+    --hash=sha256:dcdad740e841d791b805421c2b20e859b4ed556396d3063b3aa64cd055be648c \
+    --hash=sha256:e07afb1613d6f5fd99abd4e53ad3b446b4efaa0f0d8e9dfb1d6d1b9f3f884d32 \
+    --hash=sha256:e1e83af732726987d2c4cd736f415dae8b966ba17b7a2239c8b7ffe70bfb5543 \
+    --hash=sha256:e4370733967f5994ceeed8dc211089bedd45832ee688cecea17bfd35a9eb22b9 \
+    --hash=sha256:e52575cbc6b9764ea138a6f82d73d3b1bc685fe62e207ff46a963d4c773799f6 \
+    --hash=sha256:e6516f69213ae393a220e904332f1a6bfc299ba22cf27a6520a1663a08eba0fb \
+    --hash=sha256:ec108f368c12b25787c8b1a4537a1452bc53861c3ee4abc810cc74098278edcd \
+    --hash=sha256:f1add8f1d83099a98ae4ac472d896b7e36db48c39d3db25adf12b373823cdeff \
+    --hash=sha256:f3956f3c5c229257dbeabe0b6aacd2c083ebcc1e335842a6ff2217fe6cc03b6b \
+    --hash=sha256:f48abff54054b4142ad03b323e80aa89b1d15cabc48ff49eb7a6ff7621829a56 \
+    --hash=sha256:f4cfea4eada1746d0c75a864bc7e9e63d4a6e987c852d6cec8d9cb0c83afe25b \
+    --hash=sha256:f7aa5c3327dda4ef952769bacec09c09ff5bf426e07fdc94478c37955681885b \
+    --hash=sha256:f92694c9396f55d4c91087efacf81297bef152893806fc54c289fc0254b45384 \
+    --hash=sha256:fc16796c85d7d8b259881d59cc8b5e22e940901928c2ff6924b2c967924e8a0b \
+    --hash=sha256:fd091209798cfdce53746f5769987b4108fe941c54fb2e058c016ffc47872918
 lxml==4.9.3 \
     --hash=sha256:05186a0f1346ae12553d66df1cfce6f251589fea3ad3da4f3ef4e34b2d58c6a3 \
     --hash=sha256:075b731ddd9e7f68ad24c635374211376aa05a281673ede86cbe1d1b3455279d \
@@ -643,6 +735,9 @@ python-dateutil==2.8.2 \
 python-decouple==3.8 \
     --hash=sha256:ba6e2657d4f376ecc46f77a3a615e058d93ba5e465c01bbe57289bfb7cce680f \
     --hash=sha256:d0d45340815b25f4de59c974b855bb38d03151d81b037d9e3f463b0c9f8cbd66
+python-levenshtein==0.26.1 \
+    --hash=sha256:24ba578e28058ebb4afa2700057e1678d7adf27e43cd1f17700c09a9009d5d3a \
+    --hash=sha256:8ef5e529dd640fb00f05ee62d998d2ee862f19566b641ace775d5ae16167b2ef
 python-slugify==8.0.1 \
     --hash=sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395 \
     --hash=sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27

data_collection/requirements.in

@@ -4,12 +4,14 @@ click
 chompjs
 dateparser
 filetype
+fuzzywuzzy
 itemadapter
 jinja2
 psycopg2-binary
 pyyaml==5.3.1
 python-dateutil
 python-decouple
+python-Levenshtein
 scrapy
 scrapy-zyte-smartproxy
 SQLAlchemy