.woodpecker/lint.yml

@@ -2,7 +2,7 @@ when:
   branch: main
 steps:
   - name: lint
-    image: python:3-slim
+    image: python:3-alpine
     commands:
       - python -m pip install --upgrade pip
       - python -m pip install -r requirements.txt

config-sample.yml

@@ -3,7 +3,3 @@ mastodon:
   client_secret: 
   access_token: 
   api_base_url: 
-languages:
-  - en
-  - es
-default_language: en

kuow_fetcher.py

@@ -6,7 +6,6 @@ from typing import Optional
 import requests
 import yaml
 from bs4 import BeautifulSoup
-from lingua import IsoCode639_1, Language, LanguageDetectorBuilder
 from mastodon import Mastodon
 from sqlalchemy import create_engine, select
 from sqlalchemy.exc import NoResultFound
@@ -22,27 +21,13 @@ class KuowStory(Base):
     pageview_story_id: Mapped[str] = mapped_column(primary_key=True, unique=True)
     dfp_targeting_id: Mapped[str] = mapped_column()
     article_link: Mapped[str] = mapped_column()
-    article_language: Mapped[Optional[str]] = mapped_column()
     last_updated_time: Mapped[datetime] = mapped_column()
     post_id: Mapped[Optional[str]] = mapped_column()
 
     def __repr__(self) -> str:
-        return f"KuowStory(pageview_story_id={self.pageview_story_id!r}, dfp_targeting_id={self.dfp_targeting_id!r}, article_link={self.article_link!r}, article_language={self.article_language!r}, last_updated_time={self.last_updated_time!r}, post_id={self.post_id!r})"
+        return f"KuowStory(pageview_story_id={self.pageview_story_id!r}, dfp_targeting_id={self.dfp_targeting_id!r}, article_link={self.article_link!r}, last_updated_time={self.last_updated_time!r}, post_id={self.post_id!r})"
 
 
-def get_language_from_iso_code_639_1_str(iso_code_639_1_str: str) -> Language:
-    iso_code_369_1 = getattr(IsoCode639_1, iso_code_639_1_str.upper())
-    return Language.from_iso_code_639_1(iso_code_369_1)
-
-
-def detect_article_language(article_description: str) -> str:
-    detector = LanguageDetectorBuilder.from_languages(*languages).build()
-    try:
-        language = detector.detect_language_of(article_description)
-        return language.iso_code_639_1.name
-    except AttributeError:
-        return default_language.iso_code_639_1.name
-
 engine = create_engine("sqlite:///kuow.db")
 Base.metadata.create_all(engine)
 
@@ -59,11 +44,6 @@ mastodon = Mastodon(
     api_base_url=config["mastodon"]["api_base_url"],
 )
 
-languages = [
-    get_language_from_iso_code_639_1_str(language) for language in config["languages"]
-]
-default_language = get_language_from_iso_code_639_1_str(config["default_language"])
-
 kuow_response = requests.get(url)
 soup = BeautifulSoup(kuow_response.content, "html.parser")
 articles = soup.find_all("span", class_="txt")
@@ -127,14 +107,14 @@ with Session(engine) as session:
                 article_record.dfp_targeting_id = article_soup.find(
                     "script", {"class": "dfp_targeting", "data-key": "id"}
                 )["data-value"]
-            except (NameError, TypeError):
+            except NameError:
                 print("Could not find or load IDs for this post")
 
             try:
                 tags = article_soup.find(
                     "script", {"class": "dfp_targeting", "data-key": "tags"}
                 )["data-value"].split("|")
-            except (NameError, TypeError):
+            except NameError:
                 print("Could not find or load any tags for this article")
                 tags = []
 
@@ -151,10 +131,6 @@ with Session(engine) as session:
                 )
                 if not article_record.post_id:
                     print("Posting to Mastodon")
-
-                    article_language = detect_article_language(article_description)
-                    article_record.article_language = article_language
-
                     mastodon_post_result = mastodon.status_post(
                         status=article_description
                         + "\n"
@@ -162,7 +138,6 @@ with Session(engine) as session:
                         + article_link
                         + "\n#KUOW #News{}".format(additional_tag_string),
                         visibility="public",
-                        language=article_language,
                     )
                     article_record.post_id = mastodon_post_result["id"]
                 else:

requirements.txt

@@ -5,7 +5,6 @@ charset-normalizer==3.3.2
 decorator==5.1.1
 greenlet==3.0.3
 idna==3.6
-lingua-language-detector==2.0.2
 Mastodon.py==1.8.1
 python-dateutil==2.8.2
 python-magic==0.4.27
@@ -13,6 +12,6 @@ PyYAML==6.0.1
 requests==2.31.0
 six==1.16.0
 soupsieve==2.5
-SQLAlchemy==2.0.27
+SQLAlchemy==2.0.26
 typing_extensions==4.9.0
 urllib3==2.2.0