updates

.github/workflows/lint.yml

@@ -12,7 +12,7 @@ permissions:
 
 jobs:
   Linting:
-    name: Linting & Type Checking
+    name: Linting
     runs-on: ubuntu-latest
     timeout-minutes: 10
 

.github/workflows/tests.yml

@@ -14,14 +14,14 @@ permissions:
 
 jobs:
   Tests:
-    name: Python ${{ matrix.python-version }}
+    name: Tests
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.10']
 
     steps:
     - uses: actions/checkout@v3
@@ -62,7 +62,7 @@ jobs:
           -m "not slow"
 
     - name: Run integration tests
-      if: matrix.python-version == '3.11'
+      if: matrix.python-version == '3.10'
       env:
         MOCK_BROWSER: "true"
         DATABASE_URL: "sqlite:///:memory:"
@@ -74,7 +74,7 @@ jobs:
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v4
-      if: matrix.python-version == '3.11'
+      if: matrix.python-version == '3.10'
       with:
         file: ./coverage.xml
         flags: unittests
@@ -84,15 +84,15 @@ jobs:
 
     - name: Archive coverage HTML report
       uses: actions/upload-artifact@v4
-      if: matrix.python-version == '3.11'
+      if: matrix.python-version == '3.10'
       with:
         name: coverage-report-py${{ matrix.python-version }}
         path: htmlcov/
         retention-days: 30
 
     - name: Pytest coverage comment
       uses: MishaKav/pytest-coverage-comment@main
-      if: matrix.python-version == '3.11' && github.event_name == 'pull_request'
+      if: matrix.python-version == '3.10' && github.event_name == 'pull_request'
       continue-on-error: true
       with:
         pytest-coverage-path: ./coverage.xml
@@ -126,7 +126,7 @@ jobs:
 
     - name: Comment PR with coverage
       uses: actions/github-script@v7
-      if: matrix.python-version == '3.11' && github.event_name == 'pull_request'
+      if: matrix.python-version == '3.10' && github.event_name == 'pull_request'
       continue-on-error: true
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -176,7 +176,7 @@ jobs:
               '<details>',
               '<summary>📈 View detailed coverage report</summary>',
               '',
-              'Download the `coverage-report-py3.11` artifact from the workflow run to view the full HTML report.',
+              'Download the `coverage-report-py3.10` artifact from the workflow run to view the full HTML report.',
               '',
               '</details>',
               '',

app/scrapers/base_scraper.py

@@ -1,11 +1,13 @@
 from abc import ABC, abstractmethod
 from DrissionPage import ChromiumPage, ChromiumOptions
+from DrissionPage.errors import ElementNotFoundError, PageDisconnectedError
 from typing import List, Dict, Optional
 import time
 import random
 import logging
+import asyncio
 from datetime import datetime, timedelta
 from app.core.database import Listing, ScrapingState
 from sqlalchemy.orm import Session
 import json
 import os
@@ -240,6 +242,19 @@
         Check if error is due to browser disconnection.
         Returns True if browser is disconnected, False otherwise.
         """
+        # Check for DrissionPage-specific disconnection errors
+        try:
+            from DrissionPage.errors import PageDisconnectedError, ConnectionError as DPConnectionError
+            if isinstance(error, (PageDisconnectedError, DPConnectionError)):
+                logger.error(
+                    f"[{self.source_name}] ❌ Browser connection lost. Stopping run. "
+                    f"Please restart Chrome on port {settings.chrome_debug_port}."
+                )
+                self.browser_alive = False
+                return True
+        except ImportError:
+            pass
+
         error_str = str(error).lower()
         # Check for Chinese disconnection message or common connection errors
         disconnection_indicators = [
@@ -257,7 +272,8 @@
         for indicator in disconnection_indicators:
             if indicator in error_str:
                 logger.error(
-                    f"[{self.source_name}] ❌ Browser connection lost. Stopping scraper run."
+                    f"[{self.source_name}] ❌ Browser connection lost. Stopping run. "
+                    f"Please restart Chrome on port {settings.chrome_debug_port}."
                 )
                 self.browser_alive = False
                 return True
@@ -623,10 +639,10 @@
 class ScraperWithRetry:
     """Wrapper to add retry logic to scrapers"""
 
-    def __init__(self, scraper: BaseScraper, max_retries: int = 3, retry_delay: int = 60, shutdown_event = None):
+    def __init__(self, scraper: BaseScraper, max_retries: Optional[int] = None, retry_delay: Optional[int] = None, shutdown_event = None):
         self.scraper = scraper
-        self.max_retries = max_retries
-        self.retry_delay = retry_delay
+        self.max_retries = max_retries if max_retries is not None else settings.scraper_max_retries
+        self.retry_delay = retry_delay if retry_delay is not None else settings.retry_delay_seconds
         self.shutdown_event = shutdown_event
 
     def scrape_with_retry(self) -> List[Dict]:
@@ -636,7 +652,7 @@
         last_error = None
 
         for attempt in range(self.max_retries):
-            # Check for shutdown signal
+            # SHUTDOWN CHECK #1: At the very beginning of the scrape attempt
             if self.shutdown_event and self.shutdown_event.is_set():
                 logger.info(f"[Scraper Retry] Shutdown signal received, aborting scrape for {source}")
                 return []
@@ -668,7 +684,7 @@
             except Exception as e:
                 last_error = e
 
-                # Check for shutdown signal immediately after exception
+                # SHUTDOWN CHECK #2: Immediately after catching an exception (before retrying)
                 if self.shutdown_event and self.shutdown_event.is_set():
                     logger.info(f"[Scraper Retry] Shutdown detected, cancelling retries for {source}")
                     return []
@@ -690,22 +706,48 @@
                     logger.warning(f"[Scraper Retry] Cleanup failed, source: {source}, error: {cleanup_error}")
 
                 if attempt < self.max_retries - 1:
-                    # Check for shutdown before initiating retry delay
+                    # Use asyncio.sleep() for retry delay with shutdown checks
+                    logger.info(f"[Scraper Retry] Retrying in {self.retry_delay} seconds, source: {source}")
+                    self._sleep_with_shutdown_check(self.retry_delay)
+
+                    # SHUTDOWN CHECK #3: Immediately after waking up from sleep
                     if self.shutdown_event and self.shutdown_event.is_set():
-                        logger.info(f"[Scraper Retry] Shutdown signal received, aborting retry for {source}")
+                        logger.info(f"[Scraper Retry] Shutdown signal received after retry delay for {source}")
                         return []
 
-                    # Check for shutdown during retry delay
-                    logger.info(f"[Scraper Retry] Retrying in {self.retry_delay} seconds, source: {source}")
-                    for _ in range(self.retry_delay):
-                        if self.shutdown_event and self.shutdown_event.is_set():
-                            logger.info(f"[Scraper Retry] Shutdown signal received during retry delay for {source}")
-                            return []
-                        time.sleep(settings.shutdown_check_interval)
-
         # All retries failed
         error_msg = f"Failed after {self.max_retries} attempts: {last_error}"
         self.scraper.update_scraping_state(success=False, error_msg=error_msg)
         logger.error(f"[Scraper Retry] ❌ All retries exhausted, source: {source}, error: {error_msg}")
 
         return []
+
+    def _sleep_with_shutdown_check(self, duration: int):
+        """
+        Sleep for the specified duration using asyncio.sleep(), checking for shutdown signals.
+        This replaces time.sleep() to enable faster shutdown response.
+        """
+        try:
+            # Try to get the running event loop
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                # We're in a thread pool, so we need to use run_coroutine_threadsafe
+                # But since we can't await in a sync function, we'll use a different approach
+                # Split the sleep into smaller chunks for responsive shutdown
+                for _ in range(duration):
+                    if self.shutdown_event and self.shutdown_event.is_set():
+                        logger.info(f"[Scraper Retry] Shutdown signal received during sleep for {self.scraper.source_name}")
+                        return
+                    time.sleep(1)
+            else:
+                # No event loop running, fall back to chunked time.sleep
+                for _ in range(duration):
+                    if self.shutdown_event and self.shutdown_event.is_set():
+                        return
+                    time.sleep(1)
+        except RuntimeError:
+            # No event loop, use chunked time.sleep for responsive shutdown
+            for _ in range(duration):
+                if self.shutdown_event and self.shutdown_event.is_set():
+                    return
+                time.sleep(1)