Reactivate FilmAffinity scraping

.github/workflows/publish-to-test-pypi.yml

@@ -1,4 +1,4 @@
-name: Upload Python Package
+name: Publish Python Package
 
 on:
   release:
@@ -7,20 +7,22 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v6
       with:
-        python-version: '3.x'
+        python-version: '3.14'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+        python -m pip install build twine
+    - name: Build
+      run: python -m build
+    - name: Check distributions
+      run: twine check dist/*
+    - name: Publish
+      if: ${{ vars.PUBLISH_TO_PYPI == 'true' }}
+      uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/python-test.yml

@@ -1,10 +1,24 @@
 name: Tests
-on: [push]
+
+on:
+  push:
+  pull_request:
+
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
-      - name: Prepare repo
-        uses: actions/checkout@master
-      - name: Test
-        uses: onichandame/python-test-action@master
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install package
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+      - name: Run tests
+        run: python -m pytest

CHANGES.rst

@@ -1,6 +1,19 @@
 Changelog
 =========
 
+v0.0.21 (17-06-2026)
+********************
+
+- Updated scraping for the current FilmAffinity layout.
+- Replaced the deprecated user-agent dependency with an internal header pool.
+- Added fast parser tests and optional live integration tests.
+- Modernized packaging metadata and GitHub Actions.
+
+v0.0.20 (28-09-2023)
+********************
+
+- Scrapping updated
+
 v0.0.19 (22-06-2021)
 ********************
 

README.rst

@@ -213,6 +213,14 @@ recommend HBO, Netflix, Filmin, Movistar, Rakuten
 
 Changelog
 =========
+v0.0.21 (17-06-2026)
+********************
+
+- Updated scraping for the current FilmAffinity layout.
+- Replaced the deprecated user-agent dependency with an internal header pool.
+- Added fast parser tests and optional live integration tests.
+- Modernized packaging metadata and GitHub Actions.
+
 v0.0.20 (28-09-2023)
 ********************
 

pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test.py", "test_*.py"]
+markers = [
+    "integration: tests that call filmaffinity.com",
+]

python_filmaffinity/__meta__.py

@@ -5,16 +5,15 @@
 __summary__ = 'Python wrapper for FilmAffinity'
 __url__ = 'https://github.com/sergiormb/python_filmaffinity'
 
-__version__ = '0.0.20'
+__version__ = '0.0.21'
 
 __install_requires__ = [
+    'beautifulsoup4>=4.9.1',
     'requests>=2.24.0',
-    'requests-cache>=0.5.2',
-    'bs4>=0.0.1',
-    'user_agent==0.1.9',
+    'requests-cache>=1.0.0',
 ]
 
 __author__ = 'sergiormb'
 __email__ = 'sergiormb88@gmail.com'
 
-__license__ = 'MIT License'
+__license__ = 'MIT'

python_filmaffinity/client.py

@@ -7,7 +7,6 @@
 import random
 
 from bs4 import BeautifulSoup
-from user_agent import generate_user_agent
 from inspect import getsourcefile
 from os.path import join, dirname, abspath
 from .config import FIELDS_MOVIE
@@ -24,6 +23,14 @@
 
 current_folder = dirname(abspath(getsourcefile(lambda: 0)))
 supported_languages = ['en', 'es', 'mx', 'ar', 'cl', 'co']
+USER_AGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/125.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 '
+    '(KHTML, like Gecko) Version/17.5 Safari/605.1.15',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 '
+    '(KHTML, like Gecko) Chrome/125.0 Safari/537.36',
+]
 
 
 class Client:
@@ -82,24 +89,20 @@ def __init__(self, lang='es', cache_path=None,
         self.cache_remove_expired = cache_remove_expired
         self.session = None
         self.session_headers = {
-            'User-Agent': generate_user_agent(
-                device_type="desktop", os=('mac', 'linux')
-            )
+            'User-Agent': random.choice(USER_AGENTS)
         }
 
     def _generate_new_session_headers(self):
         self.session_headers = {
-            'User-Agent': generate_user_agent(
-                device_type="desktop", os=('mac', 'linux')
-            )
+            'User-Agent': random.choice(USER_AGENTS)
         }
 
     def _get_cache_file(self, cache_path=None):
         """Returns the cache file used by requests-cache
         """
         c = None
         if self.cache_backend in ['memory']:
-            p = 'cache'
+            c = 'cache-film-affinity'
         elif cache_path:
             c = join(cache_path, "cache-film-affinity")
         else:
@@ -116,7 +119,16 @@ def _get_requests_session(self):
             old_data_on_error=True
             )
         if self.cache_remove_expired:
-            self.session.remove_expired_responses()
+            if hasattr(self.session.cache, 'delete'):
+                self.session.cache.delete(expired=True)
+            else:
+                remove_expired = getattr(
+                    self.session, 'remove_expired_responses', None)
+                if remove_expired is None:
+                    remove_expired = getattr(
+                        self.session.cache, 'remove_expired_responses', None)
+                if remove_expired:
+                    remove_expired()
 
     def _load_url(self, url, headers=None, verify=None,
                   timeout=3, force_server_response=False):
@@ -142,7 +154,7 @@ def _load_url(self, url, headers=None, verify=None,
                     response = self.session.get(url, **kwargs)
         except requests.exceptions.ConnectionError as er:
             raise FilmAffinityConnectionError(er)
-        logging.warn(f"Filmaffinty Client: GET {url}")
+        logging.warning(f"Filmaffinty Client: GET {url}")
         return response
 
     def _get_trailer(self, fa_id):
@@ -155,7 +167,7 @@ def _get_movie_images(self, fa_id):
         url = self.url_images + str(fa_id)
         r = self._load_url(url)
         soup = BeautifulSoup(r.content, "html.parser")
-        exist = soup.findAll("div", {"id": 'main-image-wrapper'})
+        exist = soup.find_all("div", {"id": 'main-image-wrapper'})
         if not exist:
             return {
                 'posters': [],
@@ -328,15 +340,16 @@ def _return_list_movies(self, page, method, top=10):
         soup = BeautifulSoup(page.content, "html.parser")
         if method == 'top':
             movies_list = soup.find("ul", {"id": 'top-movies'})
-            movies_cell = movies_list.find_all(
-                "li", {"class": None, "id": None}
-            )
+            movies_cell = movies_list.select(
+                ".movie-card[data-movie-id]"
+            ) if movies_list else []
             class_ = TopPage
         if method == 'search':
-            movies_cell = soup.find_all("div", {"class": 'se-it'})
+            movies_cell = soup.select(".movie-card[data-movie-id]")
             class_ = SearchPage
         if method == 'top_service':
-            movies_cell = soup.find_all("div", {"class": 'top-movie'})
+            wrapper = soup.select(".top-movie .movie-card[data-movie-id]")
+            movies_cell = wrapper or soup.select(".movie-card[data-movie-id]")
             class_ = TopServicePage
         for cell in movies_cell[:top]:
             page = class_(cell)
@@ -349,7 +362,9 @@ def _recommend(self, service, trailer=False, images=False):
         url = self.url + 'topcat.php?id=' + service
         page = self._load_url(url)
         soup = BeautifulSoup(page.content, "html.parser")
-        movies_cell = soup.find_all("div", {"class": 'movie-card'})
+        movies_cell = soup.select(".movie-card[data-movie-id]")
+        if not movies_cell:
+            return movie
         cell = random.choice(movies_cell)
         id = str(cell['data-movie-id'])
         movie = self._get_movie_by_id(id, trailer, images)

python_filmaffinity/pages/detail.py

@@ -27,10 +27,13 @@ def get_original_title(self):
         info_cell = self.soup.find("dl", {"class": 'movie-info'})
         if info_cell:
             original_title_cell = info_cell.find('dd')
+            if not original_title_cell:
+                return None
             inner_span_tag = original_title_cell.span
             if inner_span_tag:
                 inner_span_tag.decompose()
             return original_title_cell.text.strip()
+        return None
 
     def get_year(self):
         """Get the year."""
@@ -62,6 +65,7 @@ def get_rating(self):
     def get_number_of_votes(self):
         """Get the number of votes."""
         votes = self.soup.find("span", {"itemprop": 'ratingCount'})
+        votes_checked = None
         if votes:
             try:
                 votes = votes['content']
@@ -133,7 +137,7 @@ def get_music(self):
         for dt in self.soup.find_all('dt'):
             if dt.get_text() == 'Música':
                 dd = dt.next_sibling.next_sibling
-                for nb in dd.find_all('a', {'class': 'nb'}):
+                for nb in dd.find_all('span', {'class': 'nb'}):
                     music.append(nb.find('a').get_text())
         return music
 
@@ -160,7 +164,8 @@ def get_genre(self):
         """Get the genre."""
         genres = []
         for i in self.soup.find_all("span", {"itemprop": 'genre'}):
-            genres.append(i.find('a').get_text())
+            link = i.find('a')
+            genres.append(link.get_text() if link else i.get_text().strip())
         return genres
 
     def get_awards(self):
@@ -174,8 +179,10 @@ def get_awards(self):
         if not ac:
             return awards
         for i in ac.find_all("a"):
+            award = ''.join(str(s) for s in i.next_siblings).strip()
+            award = award[2:] if award.startswith('- ') else award
             awards.append({'year': i.get_text(),
-                           'award': i.next_sibling[2:]})
+                           'award': award})
         return awards
 
     def get_reviews(self):
@@ -193,4 +200,3 @@ def get_reviews(self):
                      "itemprop": 'reviewBody'}).get_text(),
                  'url': i.a['href'] if i.a else None})
         return reviews
-

python_filmaffinity/pages/images.py

@@ -2,7 +2,8 @@
 import re
 from .page import Page
 pattern_thumbnail = re.compile(r'\((.*?)\)', re.IGNORECASE)
-pattern_country = re.compile(r'>País: </strong>(.*?)</div>')
+pattern_country = re.compile(r'>Pa[ií]s: </strong>(.*?)</div>')
+pattern_country_div = re.compile(r'<div>(.*?)</div>')
 
 
 class ImagesPage(Page):
@@ -18,15 +19,18 @@ def _get_list_of_images(self, imgs_soup):
         all_imgs = imgs_soup.find_all("div", {"class": 'colorbox-image'})
         imgs = []
         for i in all_imgs:
-            if hasattr(i, 'a'):
-                im = i.a['href']
+            link = i.find('a')
+            if link:
+                im = link.get('href')
                 th = None
-                country = pattern_country.search(i.a['title'])
-                if country:
-                    country = country.group(1)
-                if hasattr(i.a, 'div'):
-                    re_th = pattern_thumbnail.search(
-                        i.a.div['style'])
+                title = link.get('title') or link.get('data-bs-title') or ''
+                country = pattern_country.search(title)
+                if not country:
+                    country = pattern_country_div.search(title)
+                country = country.group(1) if country else None
+                div = link.find('div')
+                if div and div.get('style'):
+                    re_th = pattern_thumbnail.search(div['style'])
                     if re_th:
                         th = re_th.group(1)
                 imgs.append({'image': im,