diff --git a/.env.example b/.env.example index 9589120..0886a25 100644 --- a/.env.example +++ b/.env.example @@ -194,6 +194,15 @@ CAPTCHA_CHECK_INTERVAL=30 # Maximum time to wait for CAPTCHA resolution before aborting (in minutes) CAPTCHA_TIMEOUT_MINUTES=30 +# Wait time after page refresh when checking CAPTCHA status (in seconds) +CAPTCHA_PAGE_LOAD_WAIT=2 + +# ============================================================================ +# SHUTDOWN HANDLING +# ============================================================================ +# How often to check for shutdown signal during retry delays (in seconds) +SHUTDOWN_CHECK_INTERVAL=1 + # ============================================================================ # NOTIFICATION LOGIC # ============================================================================ diff --git a/.github/scripts/parse_test_results.py b/.github/scripts/parse_test_results.py new file mode 100644 index 0000000..db3bfe4 --- /dev/null +++ b/.github/scripts/parse_test_results.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Parse pytest JUnit XML results for GitHub Actions summary.""" + +import xml.etree.ElementTree as ET +import sys + +def main(): + try: + tree = ET.parse('pytest-results.xml') + root = tree.getroot() + + tests = root.attrib.get('tests', '0') + skipped = root.attrib.get('skipped', '0') + failures = root.attrib.get('failures', '0') + errors = root.attrib.get('errors', '0') + time = float(root.attrib.get('time', '0')) + + print("| Tests | Skipped | Failures | Errors | Time |") + print("|-------|---------|----------|--------|------|") + print(f"| {tests} | {skipped} ๐Ÿฆ˜ | {failures} โŒ | {errors} ๐Ÿ”ฅ | {time:.2f}s โฑ๏ธ |") + print() + + except FileNotFoundError: + print("โš ๏ธ Test results file not found") + except Exception as e: + print(f"โš ๏ธ Could not parse test results: {e}") + +if __name__ == '__main__': + main() diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..6f6e822 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,55 @@ +name: Code Quality + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + Linting: + name: Linting & Type Checking + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pylint mypy ruff + pip install -r requirements.txt + + - name: Run ruff (fast linter) + continue-on-error: true + run: | + ruff check app/ --output-format=github + + - name: Run flake8 + run: | + # Stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # Exit-zero treats all errors as warnings + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics \ + --exclude=.git,__pycache__,venv,env,.venv,.mypy_cache,.pytest_cache,build,dist + + - name: Run pylint + continue-on-error: true + run: | + pylint app/ --exit-zero --output-format=colorized --score=yes + + - name: Run mypy + continue-on-error: true + run: | + mypy app/ --ignore-missing-imports --no-strict-optional --pretty diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml deleted file mode 100644 index 824aa76..0000000 --- a/.github/workflows/python-app.yml +++ /dev/null @@ -1,142 +0,0 @@ -name: Python Tests - -on: - push: - branches: [ main, master, develop ] - pull_request: - branches: [ main, master, develop ] - -jobs: - test: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: ['3.9', '3.10', '3.11'] - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Cache pip packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y libxml2-dev libxslt-dev python3-dev libjpeg-dev zlib1g-dev - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Set PYTHONPATH - run: | - echo "PYTHONPATH=$PYTHONPATH:$(pwd)/app" >> $GITHUB_ENV - - - name: Run tests with pytest - env: - MOCK_BROWSER: "true" - DATABASE_URL: "sqlite:///:memory:" - run: | - pytest tests/ \ - --verbose \ - --cov=app \ - --cov-report=term-missing \ - --cov-report=xml \ - --cov-report=html \ - --cov-fail-under=35 - - - name: Upload coverage reports - uses: codecov/codecov-action@v3 - if: matrix.python-version == '3.11' - with: - file: ./coverage.xml - flags: unittests - name: codecov-umbrella - fail_ci_if_error: false - - - name: Archive coverage HTML report - uses: actions/upload-artifact@v4 - if: matrix.python-version == '3.11' - with: - name: coverage-report - path: htmlcov/ - retention-days: 30 - - - name: Generate coverage badge - if: matrix.python-version == '3.11' && github.event_name == 'push' - run: | - COVERAGE=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); root = tree.getroot(); print(root.attrib['line-rate'])") - COVERAGE_PCT=$(python -c "print(f'{float('$COVERAGE') * 100:.1f}')") - echo "Coverage: ${COVERAGE_PCT}%" - echo "COVERAGE_PCT=${COVERAGE_PCT}" >> $GITHUB_ENV - - - name: Comment PR with coverage - uses: actions/github-script@v7 - if: matrix.python-version == '3.11' && github.event_name == 'pull_request' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs'); - const coverage = fs.readFileSync('coverage.xml', 'utf8'); - const match = coverage.match(/line-rate="([\d.]+)"/); - const coveragePct = match ? (parseFloat(match[1]) * 100).toFixed(1) : 'N/A'; - - const comment = `## ๐Ÿงช Test Results - - **Coverage:** ${coveragePct}% - -
- ๐Ÿ“Š View detailed coverage report - - Download the \`coverage-report\` artifact from the workflow run to view the full HTML report. - -
- - ${coveragePct >= 35 ? 'โœ… Coverage threshold met (35%)' : 'โŒ Coverage below threshold (35%)'} - `; - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment - }); - - lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install linting tools - run: | - python -m pip install --upgrade pip - pip install pylint mypy - pip install -r requirements.txt - - - name: Run pylint - continue-on-error: true - run: | - pylint app/ --exit-zero --output-format=text - - - name: Run mypy - continue-on-error: true - run: | - mypy app/ --ignore-missing-imports --no-strict-optional diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..c462109 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,53 @@ +name: Security + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + workflow_dispatch: + schedule: + - cron: '0 0 * * 0' # Run weekly on Sundays + +permissions: + contents: read + +jobs: + Security: + name: Security Scan + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install safety bandit + + - name: Check for security vulnerabilities in dependencies + continue-on-error: true + run: | + safety check --json || true + + - name: Run bandit security scan + continue-on-error: true + run: | + bandit -r app/ -f json -o bandit-report.json || true + bandit -r app/ -f screen + + - name: Upload security reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-reports + path: | + bandit-report.json + retention-days: 30 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..efdcaf8 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,209 @@ +name: Tests + +on: + push: + branches: [ main, master, develop ] + pull_request: + branches: [ main, master, develop ] + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + Tests: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libxml2-dev libxslt-dev python3-dev libjpeg-dev zlib1g-dev + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r requirements.txt + + - name: Set PYTHONPATH + run: | + echo "PYTHONPATH=$PYTHONPATH:$(pwd)/app" >> $GITHUB_ENV + + - name: Run unit tests + env: + MOCK_BROWSER: "true" + DATABASE_URL: "sqlite:///:memory:" + run: | + pytest tests/unit/ \ + --verbose \ + --cov=app \ + --cov-report=term-missing \ + --cov-report=xml \ + --cov-report=html \ + --cov-fail-under=60 \ + --junitxml=pytest-results.xml \ + -m "not slow" + + - name: Run integration tests + if: matrix.python-version == '3.11' + env: + MOCK_BROWSER: "true" + DATABASE_URL: "sqlite:///:memory:" + run: | + pytest tests/ \ + --verbose \ + -m "integration or slow" \ + --maxfail=3 + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + if: matrix.python-version == '3.11' + with: + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Archive coverage HTML report + uses: actions/upload-artifact@v4 + if: matrix.python-version == '3.11' + with: + name: coverage-report-py${{ matrix.python-version }} + path: htmlcov/ + retention-days: 30 + + - name: Pytest coverage comment + uses: MishaKav/pytest-coverage-comment@main + if: matrix.python-version == '3.11' && github.event_name == 'pull_request' + continue-on-error: true + with: + pytest-coverage-path: ./coverage.xml + junitxml-path: ./pytest-results.xml + title: Coverage Report + badge-title: Coverage + hide-badge: false + hide-report: false + create-new-comment: false + hide-comment: false + report-only-changed-files: false + remove-link-from-badge: false + + - name: Generate test summary + if: always() + run: | + echo "## Test Results - Python ${{ matrix.python-version }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Parse test results from JUnit XML + if [ -f pytest-results.xml ]; then + python3 .github/scripts/parse_test_results.py >> $GITHUB_STEP_SUMMARY + fi + + # Add coverage info + if [ -f coverage.xml ]; then + COVERAGE=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); root = tree.getroot(); print(f\"{float(root.attrib['line-rate']) * 100:.1f}\")") + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Coverage:** ${COVERAGE}%" >> $GITHUB_STEP_SUMMARY + fi + + - name: Comment PR with coverage + uses: actions/github-script@v7 + if: matrix.python-version == '3.11' && github.event_name == 'pull_request' + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const xml2js = require('xml2js'); + + try { + // Parse coverage data + const coverage = fs.readFileSync('coverage.xml', 'utf8'); + const match = coverage.match(/line-rate="([\d.]+)"/); + const coveragePct = match ? (parseFloat(match[1]) * 100).toFixed(0) : 'N/A'; + + // Parse JUnit XML for test results + const junitXml = fs.readFileSync('pytest-results.xml', 'utf8'); + const parser = new xml2js.Parser(); + let testData = { tests: 0, skipped: 0, failures: 0, errors: 0, time: '0.0' }; + + parser.parseString(junitXml, (err, result) => { + if (!err && result.testsuite) { + const suite = result.testsuite.$; + testData = { + tests: suite.tests || 0, + skipped: suite.skipped || 0, + failures: suite.failures || 0, + errors: suite.errors || 0, + time: parseFloat(suite.time || 0).toFixed(2) + }; + } + }); + + // Determine coverage badge color + const badgeColor = coveragePct >= 70 ? 'brightgreen' : coveragePct >= 50 ? 'yellow' : 'red'; + const thresholdMet = coveragePct >= 60 ? 'โœ… Coverage threshold met (60%)' : 'โš ๏ธ Coverage below threshold (60%)'; + + const comment = [ + '## ๐Ÿงช Test Results', + '', + `![Coverage](https://img.shields.io/badge/Coverage-${coveragePct}%25-${badgeColor})`, + '', + '### ๐Ÿ“Š Test Summary', + '', + '| Tests | Skipped | Failures | Errors | Time |', + '|-------|---------|----------|--------|------|', + `| ${testData.tests} | ${testData.skipped} ๐Ÿฆ˜ | ${testData.failures} โŒ | ${testData.errors} ๐Ÿ”ฅ | ${testData.time}s โฑ๏ธ |`, + '', + '
', + '๐Ÿ“ˆ View detailed coverage report', + '', + 'Download the `coverage-report-py3.11` artifact from the workflow run to view the full HTML report.', + '', + '
', + '', + thresholdMet + ].join('\n'); + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + + console.log('Successfully posted coverage comment to PR'); + } catch (error) { + console.log('Failed to post PR comment:', error.message); + console.log('This is expected for PRs from forks due to security restrictions'); + } + + - name: Upload logs on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: test-logs-py${{ matrix.python-version }} + path: | + *.log + debug_output/ + pytest-results.xml + retention-days: 7 + if-no-files-found: ignore diff --git a/app/core/config.py b/app/core/config.py index 03f1275..abfaf44 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -69,10 +69,20 @@ class Settings(BaseSettings): # Captcha Logic captcha_check_interval: int = Field(default=30, env="CAPTCHA_CHECK_INTERVAL") captcha_timeout_minutes: int = Field(default=30, env="CAPTCHA_TIMEOUT_MINUTES") + captcha_page_load_wait: int = Field(default=2, env="CAPTCHA_PAGE_LOAD_WAIT") + + # Shutdown Logic + shutdown_check_interval: int = Field(default=1, env="SHUTDOWN_CHECK_INTERVAL") # Deal Score Logic min_deal_score_for_noti: float = Field(default=80, env="MIN_DEAL_SCORE_FOR_NOTI") + # Deal Score Weights (total should be 100) + deal_score_weight_price: float = Field(default=40.0, env="DEAL_SCORE_WEIGHT_PRICE") + deal_score_weight_features: float = Field(default=30.0, env="DEAL_SCORE_WEIGHT_FEATURES") + deal_score_weight_recency: float = Field(default=15.0, env="DEAL_SCORE_WEIGHT_RECENCY") + deal_score_weight_price_trend: float = Field(default=15.0, env="DEAL_SCORE_WEIGHT_PRICE_TREND") + # Logging log_level: str = Field(default="INFO", env="LOG_LEVEL") log_file: str = Field(default="real_estate_monitor.log", env="LOG_FILE") diff --git a/app/core/deal_score.py b/app/core/deal_score.py index f9b6e66..9b65209 100644 --- a/app/core/deal_score.py +++ b/app/core/deal_score.py @@ -17,24 +17,24 @@ def calculate_score(self, listing: Listing) -> float: """ Calculate deal score (0-100) for a listing - Scoring breakdown: - - Price competitiveness: 0-40 points - - Features match: 0-30 points - - Recency/freshness: 0-15 points - - Price trend: 0-15 points + Scoring breakdown (configurable via settings): + - Price competitiveness: 0-{price_weight} points + - Features match: 0-{features_weight} points + - Recency/freshness: 0-{recency_weight} points + - Price trend: 0-{trend_weight} points """ score = 0.0 - # 1. Price Competitiveness (0-40 points) + # 1. Price Competitiveness score += self._score_price_competitiveness(listing) - # 2. Features Match (0-30 points) + # 2. Features Match score += self._score_features(listing) - # 3. Recency (0-15 points) + # 3. Recency score += self._score_recency(listing) - # 4. Price Trend (0-15 points) + # 4. Price Trend score += self._score_price_trend(listing) return min(100.0, max(0.0, score)) @@ -44,6 +44,8 @@ def _score_price_competitiveness(self, listing: Listing) -> float: if not listing.price_per_sqm or listing.price_per_sqm <= 0: return 0.0 + max_score = self.settings.deal_score_weight_price + # Get neighborhood stats stats = self.db.query(NeighborhoodStats).filter( NeighborhoodStats.city == listing.city, @@ -51,33 +53,33 @@ def _score_price_competitiveness(self, listing: Listing) -> float: ).first() if not stats or not stats.avg_price_per_sqm: - # No data, give neutral score - return 20.0 + # No data, give neutral score (50% of max) + return max_score * 0.5 # Calculate percentage difference avg_price = stats.avg_price_per_sqm price_ratio = listing.price_per_sqm / avg_price - # Score based on how much below average + # Score based on how much below average (percentage of max_score) if price_ratio <= 0.7: # 30% below average - return 40.0 + return max_score * 1.0 elif price_ratio <= 0.8: # 20% below average - return 35.0 + return max_score * 0.875 elif price_ratio <= 0.9: # 10% below average - return 30.0 + return max_score * 0.75 elif price_ratio <= 1.0: # At or slightly below average - return 25.0 + return max_score * 0.625 elif price_ratio <= 1.1: # 10% above average - return 15.0 + return max_score * 0.375 elif price_ratio <= 1.2: # 20% above average - return 10.0 + return max_score * 0.25 else: # More than 20% above average - return 5.0 + return max_score * 0.125 def _score_features(self, listing: Listing) -> float: """Score based on matching user preferences""" score = 0.0 - max_score = 30.0 + max_score = self.settings.deal_score_weight_features features = [] @@ -93,11 +95,10 @@ def _score_features(self, listing: Listing) -> float: if self.settings.prefer_elevator: features.append(('elevator', listing.has_elevator, 7.0)) - # Mamad - ื”ื•ืกืฃ ืืช ื–ื” + # Mamad if self.settings.prefer_mamad: features.append(('mamad', listing.has_mamad, 8.0)) - # Top floors preference if self.settings.prefer_top_floors and listing.floor and listing.total_floors: is_top_half = listing.floor >= (listing.total_floors / 2) @@ -114,57 +115,62 @@ def _score_features(self, listing: Listing) -> float: def _score_recency(self, listing: Listing) -> float: """Score based on how fresh the listing is""" + max_score = self.settings.deal_score_weight_recency + if not listing.first_seen: - return 15.0 # New listing, give max score + return max_score # New listing, give max score days_old = (datetime.utcnow() - listing.first_seen).days if days_old == 0: # Today - return 15.0 + return max_score * 1.0 elif days_old <= 2: # 1-2 days - return 12.0 + return max_score * 0.8 elif days_old <= 5: # 3-5 days - return 9.0 + return max_score * 0.6 elif days_old <= 10: # 6-10 days - return 6.0 + return max_score * 0.4 elif days_old <= 20: # 11-20 days - return 3.0 + return max_score * 0.2 else: # Over 20 days - return 1.0 + return max_score * 0.067 def _score_price_trend(self, listing: Listing) -> float: """Score based on price changes""" + max_score = self.settings.deal_score_weight_price_trend + neutral_score = max_score * 0.333 # 1/3 of max for neutral + if not listing.price_history or len(listing.price_history) < 2: - return 5.0 # Neutral score for no history + return neutral_score # Neutral score for no history # Get most recent price changes sorted_history = sorted(listing.price_history, key=lambda x: x.timestamp, reverse=True) if len(sorted_history) < 2: - return 5.0 + return neutral_score current_price = sorted_history[0].price previous_price = sorted_history[1].price if not current_price or not previous_price or previous_price <= 0: - return 5.0 + return neutral_score # Calculate price change percentage price_change_pct = ((current_price - previous_price) / previous_price) * 100 # Score based on price drops if price_change_pct <= -10: # 10%+ drop - return 15.0 + return max_score * 1.0 elif price_change_pct <= -5: # 5-10% drop - return 12.0 + return max_score * 0.8 elif price_change_pct <= -2: # 2-5% drop - return 9.0 + return max_score * 0.6 elif price_change_pct < 0: # Any drop - return 7.0 + return max_score * 0.467 elif price_change_pct == 0: # No change - return 5.0 + return neutral_score else: # Price increase - return 2.0 + return max_score * 0.133 def get_price_drop_percentage(self, listing: Listing) -> Optional[float]: """Get percentage of price drop if any""" diff --git a/app/scrapers/base_scraper.py b/app/scrapers/base_scraper.py index 98e7072..36e02c4 100644 --- a/app/scrapers/base_scraper.py +++ b/app/scrapers/base_scraper.py @@ -69,13 +69,19 @@ def is_timeout(self) -> bool: class BaseScraper(ABC): """Base class for all scrapers with common functionality using DrissionPage""" - def __init__(self, db_session: Session, source_name: str): + def __init__(self, db_session: Session, source_name: str, page: Optional[ChromiumPage] = None): self.db = db_session self.source_name = source_name - self.page: Optional[ChromiumPage] = None + self.page: Optional[ChromiumPage] = page # Allow injection of mock page + self.browser_alive = True # Track browser connection status def initialize(self): """Initialize browser by connecting to existing Chrome instance on debug port""" + # If page was injected (e.g., for testing), skip initialization + if self.page is not None: + logger.info(f"[{self.source_name}] Using injected page (likely for testing)") + return + try: # Connect to existing Chrome instance on the configured debug port debug_address = f"127.0.0.1:{settings.chrome_debug_port}" @@ -229,6 +235,55 @@ def _save_cookies(self): except Exception as e: logger.warning(f"Failed to save cookies for {self.source_name}: {e}") + def _check_browser_connection(self, error: Exception) -> bool: + """ + Check if error is due to browser disconnection. + Returns True if browser is disconnected, False otherwise. + """ + error_str = str(error).lower() + # Check for Chinese disconnection message or common connection errors + disconnection_indicators = [ + "ไธŽ้กต้ข็š„่ฟžๆŽฅๅทฒๆ–ญๅผ€", # Chinese: "Connection to page has been disconnected" + "ๆ–ญๅผ€", # Chinese: "disconnected" + "disconnected", + "connection lost", + "target closed", + "session not created", + "cannot connect to", + "connection refused", + "connection reset", + ] + + for indicator in disconnection_indicators: + if indicator in error_str: + logger.error( + f"[{self.source_name}] โŒ Browser connection lost. Stopping scraper run." + ) + self.browser_alive = False + return True + + return False + + def _is_browser_alive(self) -> bool: + """ + Check if browser connection is still alive. + Returns True if browser is connected, False otherwise. + """ + if not self.page: + return False + + try: + # Try to check if page is alive using a simple property access + _ = self.page.title + return True + except Exception as e: + # Check if this is a disconnection error + if self._check_browser_connection(e): + return False + # Other errors don't necessarily mean disconnection + logger.debug(f"[{self.source_name}] Error checking browser status: {e}") + return True # Assume alive unless proven disconnected + def random_delay(self, min_seconds: float = 1.0, max_seconds: float = 3.0): """Add random delay to appear human-like""" delay = random.uniform(min_seconds, max_seconds) @@ -499,7 +554,7 @@ def _handle_anti_bot_protection(self): # Refresh page content and check again try: self.page.refresh() - time.sleep(2) # Wait for page to load + time.sleep(settings.captcha_page_load_wait) # Wait for page to load # Re-check using _check_for_captcha() method if not self._check_for_captcha(): @@ -592,6 +647,13 @@ def scrape_with_retry(self) -> List[Dict]: logger.debug(f"[Scraper Retry] Initializing browser, source: {source}") self.scraper.initialize() + # Check if browser is alive before starting scrape + if not self.scraper._is_browser_alive(): + logger.error(f"[Scraper Retry] โŒ Browser not alive, aborting scrape for {source}") + error_msg = "Browser connection not available" + self.scraper.update_scraping_state(success=False, error_msg=error_msg) + return [] + logger.info(f"[Scraper Retry] Executing scrape, source: {source}") listings = self.scraper.scrape() @@ -605,6 +667,20 @@ def scrape_with_retry(self) -> List[Dict]: except Exception as e: last_error = e + + # Check for shutdown signal immediately after exception + if self.shutdown_event and self.shutdown_event.is_set(): + logger.info(f"[Scraper Retry] Shutdown detected, cancelling retries for {source}") + return [] + + # Check if this is a browser disconnection error + if self.scraper._check_browser_connection(e): + # Browser disconnected - stop retrying immediately + error_msg = f"Browser connection lost: {e}" + self.scraper.update_scraping_state(success=False, error_msg=error_msg) + logger.error(f"[Scraper Retry] โŒ Browser disconnected, aborting all retries for {source}") + return [] + logger.error(f"[Scraper Retry] โŒ Attempt {attempt + 1} failed, source: {source}, error: {e}") try: @@ -614,13 +690,18 @@ def scrape_with_retry(self) -> List[Dict]: logger.warning(f"[Scraper Retry] Cleanup failed, source: {source}, error: {cleanup_error}") if attempt < self.max_retries - 1: + # Check for shutdown before initiating retry delay + if self.shutdown_event and self.shutdown_event.is_set(): + logger.info(f"[Scraper Retry] Shutdown signal received, aborting retry for {source}") + return [] + # Check for shutdown during retry delay logger.info(f"[Scraper Retry] Retrying in {self.retry_delay} seconds, source: {source}") for _ in range(self.retry_delay): if self.shutdown_event and self.shutdown_event.is_set(): logger.info(f"[Scraper Retry] Shutdown signal received during retry delay for {source}") return [] - time.sleep(1) + time.sleep(settings.shutdown_check_interval) # All retries failed error_msg = f"Failed after {self.max_retries} attempts: {last_error}" diff --git a/app/scrapers/yad2_scraper.py b/app/scrapers/yad2_scraper.py index 3f8bc1e..072e5ef 100644 --- a/app/scrapers/yad2_scraper.py +++ b/app/scrapers/yad2_scraper.py @@ -262,8 +262,8 @@ def _extract_number(self, text: str) -> Optional[float]: def _extract_rooms(self, text: str) -> Optional[float]: """Extract number of rooms""" - # Look for patterns like "3 ื—ื“ืจื™ื" or "3.5 ื—ื“'" - match = re.search(r'(\d+\.?\d*)\s*(?:ื—ื“ืจื™ื|ื—ื“\'|rooms)', text) + # Look for patterns like "3 ื—ื“ืจื™ื", "3.5 ื—ื“'", or "5 ื—ื“" + match = re.search(r'(\d+\.?\d*)\s*(?:ื—ื“ืจื™ื|ื—ื“\'|ื—ื“|rooms)', text) if match: try: return float(match.group(1)) diff --git a/app/utils/phone_normalizer.py b/app/utils/phone_normalizer.py index 24abc20..46d628b 100644 --- a/app/utils/phone_normalizer.py +++ b/app/utils/phone_normalizer.py @@ -46,8 +46,8 @@ def normalize_israeli_phone(phone: Optional[str]) -> Optional[str]: if len(digits) > 10 and digits.startswith('0'): digits = digits[:10] - # Validate length (Israeli mobile/landline numbers are 10 digits) - if len(digits) != 10: + # Validate length (Israeli mobile numbers are 10 digits, landlines can be 9) + if len(digits) not in [9, 10]: return None return digits diff --git a/pytest.ini b/pytest.ini index 79e274a..294239f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,6 +12,9 @@ testpaths = tests # Add app directory to Python path pythonpath = . app +# Asyncio mode +asyncio_mode = auto + # Coverage options addopts = --verbose @@ -29,6 +32,7 @@ markers = scraper: Tests for scraper modules parser: Tests for parsing logic processor: Tests for listing processor + asyncio: Async tests # Ignore patterns norecursedirs = .git .tox dist build *.egg htmlcov diff --git a/requirements.txt b/requirements.txt index f233e1a..1a4b5f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,8 @@ pytest==7.4.3 pytest-cov==4.1.0 pytest-mock==3.12.0 pytest-asyncio==0.21.1 + +# Linting and security +ruff==0.1.9 +safety==2.3.5 +bandit==1.7.6 diff --git a/tests/conftest.py b/tests/conftest.py index 7970175..355f0bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,11 +33,17 @@ def test_settings(): database_url="sqlite:///:memory:", # Search criteria - cities="Tel Aviv,Ramat Gan", + cities="ืชืœ ืื‘ื™ื‘,Tel Aviv,Ramat Gan", max_price=3000000, min_rooms=3.0, min_size_sqm=70, + # Deal breakers (set to False for tests to pass) + exclude_ground_floor=False, + require_elevator_above_floor=0, + require_parking=False, + require_mamad=False, + # Preferences prefer_parking=True, prefer_balcony=True, diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 0000000..d843ec4 --- /dev/null +++ b/tests/fixtures/__init__.py @@ -0,0 +1,3 @@ +""" +Test fixtures for HTML samples and test data. +""" diff --git a/tests/fixtures/yad2_listing_sample.html b/tests/fixtures/yad2_listing_sample.html new file mode 100644 index 0000000..59ded7c --- /dev/null +++ b/tests/fixtures/yad2_listing_sample.html @@ -0,0 +1,21 @@ + + + + + ื“ื™ืจืช 3.5 ื—ื“ืจื™ื ืœืžื›ื™ืจื” ื‘ืชืœ ืื‘ื™ื‘ - ื™ื“2 + + +
+ +
ื“ื™ืจืช 3.5 ื—ื“ืจื™ื ืœืžื›ื™ืจื” ื‘ืชืœ ืื‘ื™ื‘
+
2,500,000 โ‚ช
+
ืจื—ื•ื‘ ื”ืจืฆืœ, ืคืœื•ืจื ื˜ื™ืŸ, ืชืœ ืื‘ื™ื‘
+
+ 3.5 ื—ื“ืจื™ื | 85 ืž"ืจ | ืงื•ืžื” 3 ืžืชื•ืš 5 | ืžืขืœื™ืช | ื—ื ื™ื” | ืžืจืคืกืช | ืžืž"ื“ +
+ ืชืžื•ื ื” 1 + ืชืžื•ื ื” 2 + +
+ + diff --git a/tests/unit/test_base_scraper.py b/tests/unit/test_base_scraper.py new file mode 100644 index 0000000..7834807 --- /dev/null +++ b/tests/unit/test_base_scraper.py @@ -0,0 +1,434 @@ +""" +Unit tests for base scraper +""" +import pytest +from unittest.mock import Mock, patch, MagicMock +from datetime import datetime, timedelta + +from app.scrapers.base_scraper import ( + BaseScraper, + ScraperWithRetry, + CaptchaState, + captcha_state +) +from app.core.database import ScrapingState + + +@pytest.fixture +def mock_db_session(): + """Create mock database session""" + return Mock() + + +@pytest.fixture +def mock_page(): + """Create mock ChromiumPage""" + page = Mock() + page.title = "Test Page" + page.html = "Test" + page.url = "https://example.com" + page.cookies = Mock(return_value=[]) + return page + + +class ConcreteScraper(BaseScraper): + """Concrete implementation for testing""" + + def scrape(self): + return [{"title": "Test"}] + + def parse_listing(self, raw_data): + return raw_data + + +class TestCaptchaState: + """Test CaptchaState singleton""" + + def test_singleton(self): + """Test that CaptchaState is a singleton""" + state1 = CaptchaState() + state2 = CaptchaState() + assert state1 is state2 + + def test_set_waiting(self): + """Test setting waiting state""" + state = CaptchaState() + state.set_waiting("test_source") + + assert state.is_waiting() is True + assert state._source == "test_source" + assert state._waiting_since is not None + + def test_set_normal(self): + """Test resetting to normal state""" + state = CaptchaState() + state.set_waiting("test_source") + state.set_normal() + + assert state.is_waiting() is False + assert state._source is None + assert state._waiting_since is None + + def test_get_status(self): + """Test getting status""" + state = CaptchaState() + state.set_normal() + + status = state.get_status() + + assert status["status"] == "NORMAL" + assert status["waiting_since"] is None + assert status["source"] is None + + def test_get_status_waiting(self): + """Test getting status when waiting""" + state = CaptchaState() + state.set_waiting("test_source") + + status = state.get_status() + + assert status["status"] == "WAITING_FOR_CAPTCHA" + assert status["source"] == "test_source" + assert status["waiting_since"] is not None + + def test_is_timeout_false(self): + """Test timeout check when not timed out""" + state = CaptchaState() + state.set_waiting("test_source") + + assert state.is_timeout() is False + + def test_is_timeout_true(self): + """Test timeout check when timed out""" + state = CaptchaState() + state.set_waiting("test_source") + # Manually set old timestamp + state._waiting_since = datetime.utcnow() - timedelta(minutes=100) + + with patch('app.scrapers.base_scraper.settings') as mock_settings: + mock_settings.captcha_timeout_minutes = 30 + assert state.is_timeout() is True + + +class TestBaseScraper: + """Test BaseScraper class""" + + def test_init(self, mock_db_session): + """Test scraper initialization""" + scraper = ConcreteScraper(mock_db_session, "test_source") + + assert scraper.db == mock_db_session + assert scraper.source_name == "test_source" + assert scraper.page is None + + @patch('app.scrapers.base_scraper.ChromiumPage') + def test_initialize_success(self, mock_chromium_page, mock_db_session): + """Test successful browser initialization""" + mock_page = Mock() + mock_chromium_page.return_value = mock_page + + scraper = ConcreteScraper(mock_db_session, "test_source") + + with patch.object(scraper, '_inject_anti_detection_scripts'): + with patch.object(scraper, '_load_cookies'): + scraper.initialize() + + assert scraper.page is not None + + @patch('app.scrapers.base_scraper.ChromiumPage') + def test_initialize_connection_error(self, mock_chromium_page, mock_db_session): + """Test initialization with connection error""" + mock_chromium_page.side_effect = ConnectionError("Connection failed") + + scraper = ConcreteScraper(mock_db_session, "test_source") + + with pytest.raises(ConnectionError): + scraper.initialize() + + def test_inject_anti_detection_scripts(self, mock_db_session, mock_page): + """Test injecting anti-detection scripts""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + scraper._inject_anti_detection_scripts() + + # Should call run_js multiple times + assert mock_page.run_js.call_count > 0 + + def test_inject_anti_detection_scripts_no_page(self, mock_db_session): + """Test injecting scripts with no page""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = None + + # Should not raise error + scraper._inject_anti_detection_scripts() + + def test_load_cookies(self, mock_db_session, mock_page): + """Test loading cookies""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_state = Mock(spec=ScrapingState) + mock_state.cookies_json = '[{"name": "test", "value": "value"}]' + + mock_query = Mock() + mock_db_session.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_state + + scraper._load_cookies() + + mock_page.set.cookies.assert_called() + + def test_save_cookies(self, mock_db_session, mock_page): + """Test saving cookies""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_page.cookies.return_value = [{"name": "test", "value": "value"}] + + mock_state = Mock(spec=ScrapingState) + mock_query = Mock() + mock_db_session.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_state + + scraper._save_cookies() + + mock_db_session.commit.assert_called_once() + + def test_random_delay(self, mock_db_session): + """Test random delay""" + scraper = ConcreteScraper(mock_db_session, "test_source") + + with patch('time.sleep') as mock_sleep: + scraper.random_delay(0.1, 0.2) + mock_sleep.assert_called_once() + + def test_scroll_page(self, mock_db_session, mock_page): + """Test scrolling page""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + with patch('time.sleep'): + with patch('random.randint', return_value=500): + with patch('random.random', return_value=0.5): + scraper.scroll_page(scrolls=2) + + assert mock_page.scroll.down.call_count == 2 + + def test_safe_click_success(self, mock_db_session, mock_page): + """Test safe click success""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_element = Mock() + mock_page.ele.return_value = mock_element + + result = scraper.safe_click("test_selector") + + assert result is True + mock_element.click.assert_called_once() + + def test_safe_click_failure(self, mock_db_session, mock_page): + """Test safe click failure""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_page.ele.return_value = None + + result = scraper.safe_click("test_selector") + + assert result is False + + def test_safe_fill_success(self, mock_db_session, mock_page): + """Test safe fill success""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_element = Mock() + mock_page.ele.return_value = mock_element + + result = scraper.safe_fill("test_selector", "test_value") + + assert result is True + mock_element.input.assert_called_once_with("test_value") + + def test_safe_get_text_success(self, mock_db_session, mock_page): + """Test safe get text success""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_element = Mock() + mock_element.text = "Test text" + mock_page.ele.return_value = mock_element + + result = scraper.safe_get_text("test_selector") + + assert result == "Test text" + + def test_normalize_phone(self, mock_db_session): + """Test phone normalization""" + scraper = ConcreteScraper(mock_db_session, "test_source") + + result = scraper.normalize_phone("050-123-4567") + + assert result is not None + + def test_update_scraping_state_success(self, mock_db_session): + """Test updating scraping state on success""" + scraper = ConcreteScraper(mock_db_session, "test_source") + + mock_state = Mock(spec=ScrapingState) + mock_query = Mock() + mock_db_session.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_state + + scraper.update_scraping_state(success=True) + + assert mock_state.status == 'active' + assert mock_state.error_count == 0 + mock_db_session.commit.assert_called_once() + + def test_update_scraping_state_failure(self, mock_db_session): + """Test updating scraping state on failure""" + scraper = ConcreteScraper(mock_db_session, "test_source") + + mock_state = Mock(spec=ScrapingState) + mock_state.error_count = 0 + mock_query = Mock() + mock_db_session.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_state + + scraper.update_scraping_state(success=False, error_msg="Test error") + + assert mock_state.status == 'error' + assert mock_state.error_count == 1 + assert mock_state.error_message == "Test error" + + def test_check_for_captcha_detected(self, mock_db_session, mock_page): + """Test CAPTCHA detection""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_page.title = "CAPTCHA Required" + mock_page.html = "captcha" + + result = scraper._check_for_captcha() + + assert result is True + + def test_check_for_captcha_not_detected(self, mock_db_session, mock_page): + """Test no CAPTCHA detection""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_page.title = "Normal Page" + mock_page.html = "normal content" + + result = scraper._check_for_captcha() + + assert result is False + + def test_is_blocked(self, mock_db_session, mock_page): + """Test is_blocked method""" + scraper = ConcreteScraper(mock_db_session, "test_source") + scraper.page = mock_page + + mock_page.title = "Blocked" + mock_page.html = "access denied" + + result = scraper._is_blocked() + + assert result is True + + def test_cleanup(self, mock_db_session): + """Test cleanup""" + scraper = ConcreteScraper(mock_db_session, "test_source") + + with patch.object(scraper, '_save_cookies'): + scraper.cleanup() + # Should complete without errors + + +class TestScraperWithRetry: + """Test ScraperWithRetry wrapper""" + + def test_init(self, mock_db_session): + """Test initialization""" + scraper = ConcreteScraper(mock_db_session, "test_source") + retry_scraper = ScraperWithRetry(scraper, max_retries=3, retry_delay=1) + + assert retry_scraper.scraper == scraper + assert retry_scraper.max_retries == 3 + assert retry_scraper.retry_delay == 1 + + def test_scrape_with_retry_success(self, mock_db_session): + """Test successful scrape on first attempt""" + scraper = ConcreteScraper(mock_db_session, "test_source") + retry_scraper = ScraperWithRetry(scraper, max_retries=3, retry_delay=1) + + with patch.object(scraper, 'initialize'): + with patch.object(scraper, 'scrape', return_value=[{"title": "Test"}]): + with patch.object(scraper, 'cleanup'): + with patch.object(scraper, 'update_scraping_state'): + result = retry_scraper.scrape_with_retry() + + assert len(result) == 1 + assert result[0]["title"] == "Test" + + def test_scrape_with_retry_failure_then_success(self, mock_db_session): + """Test scrape fails then succeeds""" + scraper = ConcreteScraper(mock_db_session, "test_source") + retry_scraper = ScraperWithRetry(scraper, max_retries=3, retry_delay=1) + + call_count = [0] + + def scrape_side_effect(): + call_count[0] += 1 + if call_count[0] == 1: + raise Exception("First attempt failed") + return [{"title": "Test"}] + + with patch.object(scraper, 'initialize'): + with patch.object(scraper, 'scrape', side_effect=scrape_side_effect): + with patch.object(scraper, 'cleanup'): + with patch.object(scraper, 'update_scraping_state'): + with patch('time.sleep'): + result = retry_scraper.scrape_with_retry() + + assert len(result) == 1 + + def test_scrape_with_retry_all_failures(self, mock_db_session): + """Test all retry attempts fail""" + scraper = ConcreteScraper(mock_db_session, "test_source") + retry_scraper = ScraperWithRetry(scraper, max_retries=2, retry_delay=1) + + with patch.object(scraper, 'initialize'): + with patch.object(scraper, 'scrape', side_effect=Exception("Failed")): + with patch.object(scraper, 'cleanup'): + with patch.object(scraper, 'update_scraping_state'): + with patch('time.sleep'): + result = retry_scraper.scrape_with_retry() + + assert result == [] + + def test_scrape_with_retry_shutdown(self, mock_db_session): + """Test scrape with shutdown signal""" + import asyncio + scraper = ConcreteScraper(mock_db_session, "test_source") + shutdown_event = asyncio.Event() + shutdown_event.set() + + retry_scraper = ScraperWithRetry( + scraper, + max_retries=3, + retry_delay=1, + shutdown_event=shutdown_event + ) + + result = retry_scraper.scrape_with_retry() + + assert result == [] diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 9baf1b9..f48b70f 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -23,7 +23,8 @@ def test_get_cities_list(self, test_settings): cities = test_settings.get_cities_list() assert isinstance(cities, list) - assert len(cities) == 2 + assert len(cities) == 3 + assert 'ืชืœ ืื‘ื™ื‘' in cities assert 'Tel Aviv' in cities assert 'Ramat Gan' in cities diff --git a/tests/unit/test_dashboard.py b/tests/unit/test_dashboard.py new file mode 100644 index 0000000..6dba228 --- /dev/null +++ b/tests/unit/test_dashboard.py @@ -0,0 +1,372 @@ +""" +Unit tests for dashboard service +""" +import pytest +from datetime import datetime, timedelta +from unittest.mock import Mock, patch, MagicMock +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + +from app.services.dashboard import ( + app, + format_price, + days_ago, + get_whatsapp_url, + get_db +) +from app.core.database import Listing, NeighborhoodStats + + +@pytest.fixture +def test_client(): + """Create test client""" + return TestClient(app) + + +@pytest.fixture +def mock_db_session(): + """Create mock database session""" + return Mock(spec=Session) + + +class TestHelperFunctions: + """Test template helper functions""" + + def test_format_price_with_value(self): + """Test price formatting with valid value""" + assert format_price(5000) == "โ‚ช5,000" + assert format_price(10000.5) == "โ‚ช10,000" + assert format_price(1234567) == "โ‚ช1,234,567" + + def test_format_price_none(self): + """Test price formatting with None""" + assert format_price(None) == "N/A" + + def test_days_ago_just_now(self): + """Test days_ago for recent times""" + now = datetime.utcnow() + assert days_ago(now) == "Just now" + + def test_days_ago_minutes(self): + """Test days_ago for minutes""" + time = datetime.utcnow() - timedelta(minutes=30) + result = days_ago(time) + assert "minutes ago" in result + + def test_days_ago_hours(self): + """Test days_ago for hours""" + time = datetime.utcnow() - timedelta(hours=3) + result = days_ago(time) + assert "hours ago" in result + + def test_days_ago_yesterday(self): + """Test days_ago for yesterday""" + time = datetime.utcnow() - timedelta(days=1) + assert days_ago(time) == "Yesterday" + + def test_days_ago_days(self): + """Test days_ago for days""" + time = datetime.utcnow() - timedelta(days=3) + result = days_ago(time) + assert "days ago" in result + + def test_days_ago_weeks(self): + """Test days_ago for weeks""" + time = datetime.utcnow() - timedelta(days=14) + result = days_ago(time) + assert "weeks ago" in result + + def test_days_ago_months(self): + """Test days_ago for months""" + time = datetime.utcnow() - timedelta(days=60) + result = days_ago(time) + assert "months ago" in result + + def test_days_ago_none(self): + """Test days_ago with None""" + assert days_ago(None) == "Unknown" + + def test_get_whatsapp_url_valid(self): + """Test WhatsApp URL generation""" + url = get_whatsapp_url("0501234567", "Test St, Tel Aviv", "yad2") + assert url is not None + assert "wa.me" in url + assert "+972501234567" in url + assert "Test%20St" in url + + def test_get_whatsapp_url_with_country_code(self): + """Test WhatsApp URL with existing country code""" + url = get_whatsapp_url("+972501234567", "Test St", "yad2") + assert "+972501234567" in url + + def test_get_whatsapp_url_none(self): + """Test WhatsApp URL with no phone""" + assert get_whatsapp_url(None, "Test St", "yad2") is None + assert get_whatsapp_url("", "Test St", "yad2") is None + + +class TestDashboardEndpoints: + """Test dashboard API endpoints""" + + @patch('app.services.dashboard.get_db') + def test_index_page(self, mock_get_db, test_client): + """Test main dashboard page""" + # Setup mock + mock_db = Mock() + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.filter_by.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [] + mock_query.count.return_value = 0 + mock_query.distinct.return_value = mock_query + mock_get_db.return_value.__enter__ = Mock(return_value=mock_db) + mock_get_db.return_value.__exit__ = Mock(return_value=False) + + # Make request + response = test_client.get("/") + assert response.status_code == 200 + + @patch('app.services.dashboard.get_db') + def test_index_with_filters(self, mock_get_db, test_client): + """Test dashboard with filters""" + mock_db = Mock() + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.order_by.return_value = mock_query + mock_query.all.return_value = [] + mock_query.count.return_value = 0 + mock_query.distinct.return_value = mock_query + mock_get_db.return_value.__enter__ = Mock(return_value=mock_db) + mock_get_db.return_value.__exit__ = Mock(return_value=False) + + response = test_client.get("/?status=interested&min_score=80&max_price=5000") + assert response.status_code == 200 + + def test_listing_detail_found(self, test_client): + """Test listing detail page""" + from app.services.dashboard import app, get_db + + def override_get_db(): + mock_db = Mock() + mock_listing = Mock(spec=Listing) + mock_listing.id = 1 + mock_listing.city = "Tel Aviv" + mock_listing.neighborhood = "Center" + mock_listing.price_history = [] + + # Mock the query for NeighborhoodStats + def query_side_effect(model): + mock_query = Mock() + mock_query.filter.return_value = mock_query + if model == Listing: + mock_query.first.return_value = mock_listing + else: # NeighborhoodStats + mock_query.first.return_value = None + return mock_query + + mock_db.query.side_effect = query_side_effect + + yield mock_db + + app.dependency_overrides[get_db] = override_get_db + response = test_client.get("/listing/1") + app.dependency_overrides.clear() + + assert response.status_code == 200 + + def test_listing_detail_not_found(self, test_client): + """Test listing detail page with non-existent listing""" + from app.services.dashboard import app, get_db + + def override_get_db(): + mock_db = Mock() + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = None + yield mock_db + + app.dependency_overrides[get_db] = override_get_db + response = test_client.get("/listing/999") + app.dependency_overrides.clear() + + assert response.status_code == 404 + + def test_update_listing_status(self, test_client): + """Test updating listing status""" + from app.services.dashboard import app, get_db + + def override_get_db(): + mock_db = Mock() + mock_listing = Mock(spec=Listing) + mock_listing.id = 1 + + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_listing + yield mock_db + + app.dependency_overrides[get_db] = override_get_db + response = test_client.post("/api/listing/1/status?status=interested¬e=Test note") + app.dependency_overrides.clear() + + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["status"] == "interested" + + def test_update_listing_status_invalid(self, test_client): + """Test updating listing with invalid status""" + response = test_client.post("/api/listing/1/status?status=invalid_status") + assert response.status_code == 400 + + @patch('app.services.dashboard.get_db') + def test_get_stats(self, mock_get_db, test_client): + """Test stats endpoint""" + mock_db = Mock() + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.count.return_value = 10 + mock_query.scalar.return_value = 5000.0 + mock_query.group_by.return_value = mock_query + mock_query.all.return_value = [("unseen", 5), ("interested", 3)] + mock_get_db.return_value.__enter__ = Mock(return_value=mock_db) + mock_get_db.return_value.__exit__ = Mock(return_value=False) + + response = test_client.get("/api/stats") + assert response.status_code == 200 + data = response.json() + assert "total_listings" in data + assert "new_today" in data + + @patch('app.services.dashboard.get_db') + def test_get_neighborhood_stats(self, mock_get_db, test_client): + """Test neighborhood stats endpoint""" + mock_db = Mock() + mock_stats = Mock(spec=NeighborhoodStats) + mock_stats.city = "Tel Aviv" + mock_stats.neighborhood = "Center" + mock_stats.avg_price = 5000 + mock_stats.avg_price_per_sqm = 100 + mock_stats.median_price = 4800 + mock_stats.median_price_per_sqm = 95 + mock_stats.sample_size = 50 + + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.all.return_value = [mock_stats] + mock_get_db.return_value.__enter__ = Mock(return_value=mock_db) + mock_get_db.return_value.__exit__ = Mock(return_value=False) + + response = test_client.get("/api/neighborhood-stats?city=Tel Aviv") + assert response.status_code == 200 + data = response.json() + assert "neighborhoods" in data + + def test_get_price_history(self, test_client): + """Test price history endpoint""" + from app.services.dashboard import app, get_db + + def override_get_db(): + mock_db = Mock() + mock_listing = Mock(spec=Listing) + mock_listing.id = 1 + + mock_history = Mock() + mock_history.timestamp = datetime.utcnow() + mock_history.price = 5000 + mock_history.price_per_sqm = 100 + mock_listing.price_history = [mock_history] + + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_listing + yield mock_db + + app.dependency_overrides[get_db] = override_get_db + response = test_client.get("/api/price-history/1") + app.dependency_overrides.clear() + + assert response.status_code == 200 + data = response.json() + assert "listing_id" in data + assert "history" in data + + @patch('app.services.dashboard.get_db') + def test_health_check(self, mock_get_db, test_client): + """Test health check endpoint""" + mock_db = Mock() + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.all.return_value = [] + mock_get_db.return_value.__enter__ = Mock(return_value=mock_db) + mock_get_db.return_value.__exit__ = Mock(return_value=False) + + response = test_client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + assert "timestamp" in data + + @patch('app.scrapers.base_scraper.captcha_state') + def test_api_health_check(self, mock_captcha_state, test_client): + """Test API health check endpoint""" + from app.services.dashboard import app, get_db + + def override_get_db(): + mock_db = Mock() + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.all.return_value = [] + yield mock_db + + mock_captcha_state.is_waiting.return_value = False + mock_captcha_state.get_status.return_value = {"status": "NORMAL"} + + app.dependency_overrides[get_db] = override_get_db + response = test_client.get("/api/health") + app.dependency_overrides.clear() + + assert response.status_code == 200 + data = response.json() + assert "is_blocked" in data + assert "captcha_state" in data + + @patch('app.scrapers.base_scraper.captcha_state') + def test_get_scraper_status(self, mock_captcha_state, test_client): + """Test scraper status endpoint""" + mock_captcha_state.get_status.return_value = {"status": "NORMAL"} + mock_captcha_state.is_waiting.return_value = False + + response = test_client.get("/api/status") + assert response.status_code == 200 + data = response.json() + assert "captcha" in data + assert "is_paused" in data + + @patch('app.services.dashboard.get_db') + def test_database_stats(self, mock_get_db, test_client): + """Test database stats endpoint""" + mock_db = Mock() + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.count.return_value = 10 + mock_query.group_by.return_value = mock_query + mock_query.all.return_value = [] + mock_get_db.return_value.__enter__ = Mock(return_value=mock_db) + mock_get_db.return_value.__exit__ = Mock(return_value=False) + + response = test_client.get("/api/db-stats") + assert response.status_code == 200 + data = response.json() + assert "database" in data + assert "scrapers" in data + assert "timestamp" in data diff --git a/tests/unit/test_facebook_scraper.py b/tests/unit/test_facebook_scraper.py new file mode 100644 index 0000000..20f8b90 --- /dev/null +++ b/tests/unit/test_facebook_scraper.py @@ -0,0 +1,343 @@ +""" +Unit tests for Facebook scraper +""" +import pytest +from unittest.mock import Mock, patch, MagicMock +import os + +from app.scrapers.facebook_scraper import FacebookScraper + + +@pytest.fixture +def mock_db_session(): + """Create mock database session""" + return Mock() + + +@pytest.fixture +def mock_page(): + """Create mock ChromiumPage""" + page = Mock() + page.url = "https://www.facebook.com/marketplace" + page.html = "Test" + return page + + +@pytest.fixture +def scraper(mock_db_session): + """Create FacebookScraper instance""" + return FacebookScraper(mock_db_session, cookies_file="test_cookies.json") + + +class TestFacebookScraper: + """Test FacebookScraper class""" + + def test_init(self, mock_db_session): + """Test scraper initialization""" + scraper = FacebookScraper(mock_db_session, cookies_file="test.json") + + assert scraper.source_name == "facebook" + assert scraper.base_url == "https://www.facebook.com" + assert scraper.cookies_file == "test.json" + + def test_load_cookies_file_exists(self, scraper, mock_page): + """Test loading cookies from file""" + scraper.page = mock_page + + cookies_data = '[{"name": "test", "value": "value"}]' + + with patch('os.path.exists', return_value=True): + with patch('builtins.open', create=True) as mock_open: + mock_open.return_value.__enter__.return_value.read.return_value = cookies_data + + scraper._load_cookies() + + mock_page.set.cookies.assert_called() + + def test_load_cookies_file_not_exists(self, scraper): + """Test loading cookies when file doesn't exist""" + scraper.page = Mock() + + with patch('os.path.exists', return_value=False): + # Should not raise error + scraper._load_cookies() + + def test_load_cookies_no_file(self, mock_db_session): + """Test loading cookies with no file specified""" + scraper = FacebookScraper(mock_db_session, cookies_file=None) + scraper.page = Mock() + + # Should not raise error + scraper._load_cookies() + + def test_extract_price_from_text(self, scraper): + """Test extracting price from text""" + text = "Great apartment for โ‚ช5,000 per month" + price = scraper._extract_price_from_text(text) + + assert price == 5000.0 + + def test_extract_price_from_text_no_price(self, scraper): + """Test extracting price when none found""" + text = "Great apartment" + price = scraper._extract_price_from_text(text) + + assert price is None + + def test_extract_rooms(self, scraper): + """Test extracting number of rooms""" + text = "3 ื—ื“ืจื™ื apartment" + rooms = scraper._extract_rooms(text) + + assert rooms == 3.0 + + def test_extract_rooms_decimal(self, scraper): + """Test extracting decimal rooms""" + text = "3.5 rooms available" + rooms = scraper._extract_rooms(text) + + assert rooms == 3.5 + + def test_extract_rooms_no_rooms(self, scraper): + """Test extracting rooms when none found""" + text = "Great apartment" + rooms = scraper._extract_rooms(text) + + assert rooms is None + + def test_extract_size(self, scraper): + """Test extracting size""" + text = "Apartment 80 ืžืดืจ" + size = scraper._extract_size(text) + + assert size == 80.0 + + def test_extract_size_no_size(self, scraper): + """Test extracting size when none found""" + text = "Great apartment" + size = scraper._extract_size(text) + + assert size is None + + def test_extract_location_from_text(self, scraper): + """Test extracting location""" + text = "Apartment in ืชืœ ืื‘ื™ื‘, ืจืžืช ืื‘ื™ื‘" + city, neighborhood, street = scraper._extract_location_from_text(text) + + assert city == "ืชืœ ืื‘ื™ื‘" + assert neighborhood == "ืจืžืช ืื‘ื™ื‘" + + def test_extract_location_no_location(self, scraper): + """Test extracting location when none found""" + text = "Great apartment" + city, neighborhood, street = scraper._extract_location_from_text(text) + + assert city is None + assert neighborhood is None + assert street is None + + def test_extract_listing_data_success(self, scraper): + """Test extracting listing data from card""" + mock_card = Mock() + mock_card.text = "3 ื—ื“ืจื™ื, 80 ืžืดืจ, โ‚ช5,000, ืชืœ ืื‘ื™ื‘" + + mock_link = Mock() + mock_link.link = "https://www.facebook.com/marketplace/item/123456" + mock_card.ele.return_value = mock_link + + mock_title = Mock() + mock_title.text = "Test Apartment" + + def ele_side_effect(selector, timeout=2): + if 'title' in selector or selector == 'tag:h2': + return mock_title + if selector == 'tag:a': + return mock_link + return None + + mock_card.ele = Mock(side_effect=ele_side_effect) + mock_card.eles.return_value = [] + + data = scraper._extract_listing_data(mock_card) + + assert data is not None + assert data['external_id'] == '123456' + assert data['title'] == "Test Apartment" + assert data['price'] == 5000.0 + assert data['rooms'] == 3.0 + assert data['size_sqm'] == 80.0 + + def test_extract_listing_data_no_link(self, scraper): + """Test extracting data with no link""" + mock_card = Mock() + mock_card.ele.return_value = None + + data = scraper._extract_listing_data(mock_card) + + assert data is None + + def test_parse_listing_success(self, scraper): + """Test parsing listing data""" + raw_data = { + 'external_id': '123456', + 'url': 'https://example.com', + 'title': 'Test Apartment', + 'price': 5000, + 'rooms': 3, + 'size_sqm': 80, + 'floor': 2, + 'city': 'Tel Aviv', + 'neighborhood': 'Center', + 'street': 'Main St', + 'details_text': 'Great apartment with ืžืขืœื™ืช and ื—ื ื™ื”', + 'contact_name': 'John', + 'contact_phone': '0501234567', + 'images': ['img1.jpg'] + } + + parsed = scraper.parse_listing(raw_data) + + assert parsed is not None + assert parsed['source'] == 'facebook' + assert parsed['title'] == 'Test Apartment' + assert parsed['price'] == 5000 + assert parsed['has_elevator'] is True + assert parsed['has_parking'] is True + assert parsed['price_per_sqm'] == 5000 / 80 + + def test_parse_listing_with_features(self, scraper): + """Test parsing listing with features""" + raw_data = { + 'external_id': '123', + 'url': 'https://example.com', + 'title': 'Test', + 'price': 5000, + 'size_sqm': 50, + 'details_text': 'ืžืขืœื™ืช, ื—ื ื™ื”, ืžืจืคืกืช, ืžืž"ื“', + 'city': 'Tel Aviv', + 'neighborhood': 'Center', + 'street': 'Main St', + 'contact_name': '', + 'contact_phone': '', + 'images': [] + } + + parsed = scraper.parse_listing(raw_data) + + assert parsed['has_elevator'] is True + assert parsed['has_parking'] is True + assert parsed['has_balcony'] is True + assert parsed['has_mamad'] is True + + def test_parse_listing_no_features(self, scraper): + """Test parsing listing without features""" + raw_data = { + 'external_id': '123', + 'url': 'https://example.com', + 'title': 'Test', + 'price': 5000, + 'size_sqm': 50, + 'details_text': 'Simple apartment', + 'city': 'Tel Aviv', + 'neighborhood': 'Center', + 'street': 'Main St', + 'contact_name': '', + 'contact_phone': '', + 'images': [] + } + + parsed = scraper.parse_listing(raw_data) + + assert parsed['has_elevator'] is False + assert parsed['has_parking'] is False + assert parsed['has_balcony'] is False + assert parsed['has_mamad'] is False + + def test_parse_listing_error(self, scraper): + """Test parsing with error""" + raw_data = None + + parsed = scraper.parse_listing(raw_data) + + assert parsed is None + + @patch('app.scrapers.facebook_scraper.BaseScraper.initialize') + def test_scrape_no_page(self, mock_init, scraper): + """Test scraping with no page""" + scraper.page = None + + result = scraper.scrape() + + assert result == [] + + @patch('app.scrapers.facebook_scraper.BaseScraper.initialize') + def test_scrape_login_required(self, mock_init, scraper, mock_page): + """Test scraping when login is required""" + scraper.page = mock_page + mock_page.url = "https://www.facebook.com/login" + + with patch.object(scraper, '_handle_anti_bot_protection'): + with patch.object(scraper, 'human_like_mouse_movement'): + with patch.object(scraper, 'scroll_page'): + with patch.object(scraper, 'random_delay'): + result = scraper.scrape() + + assert result == [] + + @patch('app.scrapers.facebook_scraper.BaseScraper.initialize') + def test_scrape_success(self, mock_init, scraper, mock_page): + """Test successful scraping""" + scraper.page = mock_page + + # Mock listing cards + mock_card = Mock() + mock_card.text = "3 ื—ื“ืจื™ื, 80 ืžืดืจ, โ‚ช5,000" + + mock_link = Mock() + mock_link.link = "https://www.facebook.com/marketplace/item/123456" + + def ele_side_effect(selector, timeout=2): + if selector == 'tag:a': + return mock_link + if 'title' in selector or selector == 'tag:h2': + mock_title = Mock() + mock_title.text = "Test Apartment" + return mock_title + return None + + mock_card.ele = Mock(side_effect=ele_side_effect) + mock_card.eles.return_value = [] + + mock_page.eles.return_value = [mock_card] + + with patch.object(scraper, '_handle_anti_bot_protection'): + with patch.object(scraper, 'human_like_mouse_movement'): + with patch.object(scraper, 'scroll_page'): + with patch.object(scraper, 'random_delay'): + result = scraper.scrape() + + assert len(result) > 0 + + @patch('app.scrapers.facebook_scraper.BaseScraper.initialize') + def test_scrape_no_listings(self, mock_init, scraper, mock_page): + """Test scraping with no listings found""" + scraper.page = mock_page + mock_page.eles.return_value = [] + + with patch.object(scraper, '_handle_anti_bot_protection'): + with patch.object(scraper, 'human_like_mouse_movement'): + with patch.object(scraper, 'scroll_page'): + with patch.object(scraper, 'random_delay'): + with patch.object(scraper, 'debug_save_page'): + result = scraper.scrape() + + assert result == [] + + @patch('app.scrapers.facebook_scraper.BaseScraper.initialize') + def test_scrape_error(self, mock_init, scraper, mock_page): + """Test scraping with error""" + scraper.page = mock_page + mock_page.get.side_effect = Exception("Test error") + + with pytest.raises(Exception): + scraper.scrape() diff --git a/tests/unit/test_graceful_shutdown.py b/tests/unit/test_graceful_shutdown.py new file mode 100644 index 0000000..5d70f43 --- /dev/null +++ b/tests/unit/test_graceful_shutdown.py @@ -0,0 +1,247 @@ +""" +Unit tests for graceful shutdown functionality in Scheduler and ScraperWithRetry. +""" +import pytest +import asyncio +import threading +from unittest.mock import Mock, MagicMock, patch +from datetime import datetime + +from app.services.scheduler import ScrapingScheduler +from app.scrapers.base_scraper import ScraperWithRetry, BaseScraper + + +class TestScraperWithRetryShutdown: + """Test graceful shutdown in ScraperWithRetry""" + + def test_shutdown_event_breaks_retry_loop(self, db_session, mock_chromium_page): + """Test that shutdown_event.set() breaks the retry loop""" + # Create shutdown event + shutdown_event = threading.Event() + + # Create a mock scraper that always fails + mock_scraper = Mock(spec=BaseScraper) + mock_scraper.source_name = 'test_scraper' + mock_scraper.initialize = Mock(side_effect=Exception("Test error")) + mock_scraper.cleanup = Mock() + mock_scraper.update_scraping_state = Mock() + + # Create scraper with retry + scraper_with_retry = ScraperWithRetry( + scraper=mock_scraper, + max_retries=10, # High number to ensure we test shutdown, not exhaustion + retry_delay=1, + shutdown_event=shutdown_event + ) + + # Set shutdown event after a short delay in a separate thread + def set_shutdown(): + import time + time.sleep(0.5) # Wait a bit before shutting down + shutdown_event.set() + + shutdown_thread = threading.Thread(target=set_shutdown) + shutdown_thread.start() + + # Run scraper - should exit quickly due to shutdown event + start_time = datetime.utcnow() + result = scraper_with_retry.scrape_with_retry() + elapsed = (datetime.utcnow() - start_time).total_seconds() + + # Verify it returned empty list + assert result == [] + + # Verify it didn't run for full retry duration (10 retries * 1 second = 10 seconds) + # Should exit much faster due to shutdown event + assert elapsed < 5, f"Shutdown took too long: {elapsed} seconds" + + # Verify initialize was called at least once + assert mock_scraper.initialize.call_count >= 1 + + # Cleanup + shutdown_thread.join() + + def test_shutdown_during_retry_delay(self, db_session, mock_chromium_page): + """Test that shutdown event is checked during retry delay""" + shutdown_event = threading.Event() + + # Create a mock scraper that fails on first attempt + mock_scraper = Mock(spec=BaseScraper) + mock_scraper.source_name = 'test_scraper' + mock_scraper.initialize = Mock(side_effect=Exception("Test error")) + mock_scraper.cleanup = Mock() + mock_scraper.update_scraping_state = Mock() + + scraper_with_retry = ScraperWithRetry( + scraper=mock_scraper, + max_retries=5, + retry_delay=10, # Long delay to test shutdown during wait + shutdown_event=shutdown_event + ) + + # Set shutdown event after a short delay + def set_shutdown(): + import time + time.sleep(0.5) + shutdown_event.set() + + shutdown_thread = threading.Thread(target=set_shutdown) + shutdown_thread.start() + + # Run scraper + start_time = datetime.utcnow() + result = scraper_with_retry.scrape_with_retry() + elapsed = (datetime.utcnow() - start_time).total_seconds() + + # Should exit quickly, not wait for full retry_delay + assert elapsed < 5, f"Shutdown during retry delay took too long: {elapsed} seconds" + assert result == [] + + shutdown_thread.join() + + def test_no_shutdown_event_completes_normally(self, db_session, mock_chromium_page): + """Test that scraper works normally without shutdown event""" + # Create a mock scraper that succeeds + mock_scraper = Mock(spec=BaseScraper) + mock_scraper.source_name = 'test_scraper' + mock_scraper.initialize = Mock() + mock_scraper.scrape = Mock(return_value=[{'test': 'data'}]) + mock_scraper.cleanup = Mock() + mock_scraper.update_scraping_state = Mock() + + # No shutdown event + scraper_with_retry = ScraperWithRetry( + scraper=mock_scraper, + max_retries=3, + retry_delay=1, + shutdown_event=None + ) + + # Run scraper + result = scraper_with_retry.scrape_with_retry() + + # Should complete successfully + assert result == [{'test': 'data'}] + assert mock_scraper.initialize.call_count == 1 + assert mock_scraper.scrape.call_count == 1 + assert mock_scraper.cleanup.call_count == 1 + + +class TestSchedulerShutdown: + """Test graceful shutdown in ScrapingScheduler""" + + @pytest.mark.asyncio + async def test_scheduler_respects_shutdown_event(self, db_session): + """Test that scheduler checks shutdown event before scraping""" + # Create shutdown event and set it immediately + shutdown_event = asyncio.Event() + shutdown_event.set() + + # Create scheduler with shutdown event + with patch('app.services.scheduler.init_db') as mock_init_db: + mock_init_db.return_value = (Mock(), Mock(return_value=db_session)) + + scheduler = ScrapingScheduler(shutdown_event=shutdown_event) + + # Try to run scrape methods - they should exit immediately + await scheduler.scrape_yad2() + await scheduler.scrape_madlan() + await scheduler.scrape_facebook() + + # No assertions needed - if we get here without hanging, test passes + + @pytest.mark.asyncio + async def test_scheduler_shutdown_prevents_scraping(self): + """Test that setting shutdown event prevents new scraping jobs""" + shutdown_event = asyncio.Event() + + with patch('app.services.scheduler.init_db') as mock_init_db: + mock_session = Mock() + mock_init_db.return_value = (Mock(), Mock(return_value=mock_session)) + + scheduler = ScrapingScheduler(shutdown_event=shutdown_event) + + # Set shutdown event + shutdown_event.set() + + # Try to scrape - should exit immediately without creating scrapers + with patch('app.services.scheduler.Yad2Scraper') as mock_yad2: + await scheduler.scrape_yad2() + # Scraper should not be instantiated + mock_yad2.assert_not_called() + + @pytest.mark.asyncio + async def test_initial_scrape_with_shutdown(self): + """Test that initial scrape sequence respects shutdown event""" + shutdown_event = asyncio.Event() + + with patch('app.services.scheduler.init_db') as mock_init_db: + mock_session = Mock() + mock_init_db.return_value = (Mock(), Mock(return_value=mock_session)) + + scheduler = ScrapingScheduler(shutdown_event=shutdown_event) + + # Set shutdown event before initial scrape + shutdown_event.set() + + # Run initial scrape - should complete quickly + start_time = datetime.utcnow() + await scheduler.run_initial_scrape() + elapsed = (datetime.utcnow() - start_time).total_seconds() + + # Should exit quickly (within 10 seconds, not wait for full sequence) + assert elapsed < 10, f"Initial scrape with shutdown took too long: {elapsed} seconds" + + +class TestAsyncSleepUsage: + """Test that asyncio.sleep is used instead of time.sleep in async contexts""" + + @pytest.mark.asyncio + async def test_scheduler_uses_asyncio_sleep(self): + """Test that scheduler uses asyncio.sleep for delays""" + shutdown_event = asyncio.Event() + + with patch('app.services.scheduler.init_db') as mock_init_db: + mock_session = Mock() + mock_init_db.return_value = (Mock(), Mock(return_value=mock_session)) + + scheduler = ScrapingScheduler(shutdown_event=shutdown_event) + + # Patch asyncio.sleep to verify it's called + with patch('asyncio.sleep', new_callable=MagicMock) as mock_sleep: + mock_sleep.return_value = asyncio.coroutine(lambda: None)() + + # Run initial scrape (which has sleep calls) + shutdown_event.set() # Set to exit quickly + await scheduler.run_initial_scrape() + + # Verify asyncio.sleep was called (for the 5 second initial delay) + assert mock_sleep.call_count >= 1, "asyncio.sleep should be called in async methods" + + +class TestShutdownEventPropagation: + """Test that shutdown event is properly propagated through the system""" + + def test_shutdown_event_passed_to_scraper_with_retry(self, db_session): + """Test that shutdown event is passed from scheduler to ScraperWithRetry""" + shutdown_event = threading.Event() + + mock_scraper = Mock(spec=BaseScraper) + mock_scraper.source_name = 'test' + + # Create ScraperWithRetry with shutdown event + scraper_with_retry = ScraperWithRetry( + scraper=mock_scraper, + max_retries=3, + retry_delay=1, + shutdown_event=shutdown_event + ) + + # Verify shutdown event is stored + assert scraper_with_retry.shutdown_event is shutdown_event + + # Set shutdown event + shutdown_event.set() + + # Verify it's set + assert scraper_with_retry.shutdown_event.is_set() diff --git a/tests/unit/test_listing_processor.py b/tests/unit/test_listing_processor.py new file mode 100644 index 0000000..34f8fb1 --- /dev/null +++ b/tests/unit/test_listing_processor.py @@ -0,0 +1,378 @@ +""" +Unit tests for ListingProcessor with parameterized deal score calculations. +""" +import pytest +from datetime import datetime, timedelta +from unittest.mock import Mock, MagicMock + +from app.core.listing_processor import ListingProcessor +from app.core.deal_score import DealScoreCalculator +from app.core.database import Listing, NeighborhoodStats, PriceHistory + + +class TestDealScoreCalculation: + """Test deal score calculation with various apartment scenarios""" + + @pytest.mark.parametrize("scenario,listing_data,neighborhood_avg,expected_score_range", [ + # Scenario 1: Under-priced apartment with all features + ( + "under_priced_with_features", + { + 'price_per_sqm': 21000, # 30% below average + 'has_parking': True, + 'has_balcony': True, + 'has_elevator': True, + 'has_mamad': True, + 'floor': 4, + 'total_floors': 5, + 'first_seen': datetime.utcnow(), # Fresh listing + 'price_history': [] + }, + 30000, # Neighborhood average + (85, 100) # Expected high score + ), + # Scenario 2: Over-priced apartment with missing features + ( + "over_priced_no_features", + { + 'price_per_sqm': 42000, # 40% above average + 'has_parking': False, + 'has_balcony': False, + 'has_elevator': False, + 'has_mamad': False, + 'floor': 0, # Ground floor + 'total_floors': 5, + 'first_seen': datetime.utcnow() - timedelta(days=30), # Old listing + 'price_history': [] + }, + 30000, + (0, 25) # Expected low score + ), + # Scenario 3: Average price with some features + ( + "average_price_some_features", + { + 'price_per_sqm': 30000, # At average + 'has_parking': True, + 'has_balcony': True, + 'has_elevator': False, + 'has_mamad': False, + 'floor': 2, + 'total_floors': 4, + 'first_seen': datetime.utcnow() - timedelta(days=5), + 'price_history': [] + }, + 30000, + (40, 65) # Expected medium score + ), + # Scenario 4: Good price (price drop tested separately) + ( + "good_price_recent", + { + 'price_per_sqm': 27000, # 10% below average + 'has_parking': True, + 'has_balcony': True, + 'has_elevator': True, + 'has_mamad': True, + 'floor': 3, + 'total_floors': 5, + 'first_seen': datetime.utcnow() - timedelta(days=2), + 'price_history': [] + }, + 30000, + (70, 90) # Expected high score + ), + # Scenario 5: Missing neighborhood data (neutral scoring) + ( + "no_neighborhood_data", + { + 'price_per_sqm': 28000, + 'has_parking': True, + 'has_balcony': False, + 'has_elevator': True, + 'has_mamad': False, + 'floor': 2, + 'total_floors': 4, + 'first_seen': datetime.utcnow() - timedelta(days=2), + 'price_history': [] + }, + None, # No neighborhood data + (45, 70) # Expected medium score with neutral price component + ), + ]) + def test_calculate_score_scenarios( + self, + db_session, + test_settings, + scenario, + listing_data, + neighborhood_avg, + expected_score_range + ): + """Test deal score calculation with various apartment scenarios""" + # Create neighborhood stats if provided + if neighborhood_avg: + stats = NeighborhoodStats( + city='ืชืœ ืื‘ื™ื‘', + neighborhood='ืคืœื•ืจื ื˜ื™ืŸ', + avg_price=neighborhood_avg * 85, # Assuming 85 sqm + avg_price_per_sqm=neighborhood_avg, + median_price=neighborhood_avg * 85, + median_price_per_sqm=neighborhood_avg, + sample_size=20, + last_updated=datetime.utcnow() + ) + db_session.add(stats) + db_session.commit() + + # Create listing + listing = Listing( + property_hash='test_hash_' + scenario, + source='yad2', + external_id=scenario, + url=f'https://example.com/{scenario}', + title=f'Test Listing - {scenario}', + description='Test description', + address='ืจื—ื•ื‘ ื”ืจืฆืœ, ืคืœื•ืจื ื˜ื™ืŸ, ืชืœ ืื‘ื™ื‘', + city='ืชืœ ืื‘ื™ื‘', + neighborhood='ืคืœื•ืจื ื˜ื™ืŸ', + street='ืจื—ื•ื‘ ื”ืจืฆืœ', + rooms=3.5, + size_sqm=85.0, + floor=listing_data['floor'], + total_floors=listing_data['total_floors'], + price=listing_data['price_per_sqm'] * 85, + price_per_sqm=listing_data['price_per_sqm'], + has_elevator=listing_data['has_elevator'], + has_parking=listing_data['has_parking'], + has_balcony=listing_data['has_balcony'], + has_mamad=listing_data.get('has_mamad', False), + contact_name='Test Contact', + contact_phone='0501234567', + first_seen=listing_data['first_seen'], + last_seen=datetime.utcnow(), + last_checked=datetime.utcnow(), + status='unseen', + deal_score=0.0 + ) + + # Add price history if provided + if listing_data.get('price_history'): + listing.price_history = listing_data['price_history'] + + db_session.add(listing) + db_session.commit() + db_session.refresh(listing) + + # Calculate score + calculator = DealScoreCalculator(db_session) + score = calculator.calculate_score(listing) + + # Verify score is in expected range + min_score, max_score = expected_score_range + assert min_score <= score <= max_score, ( + f"Scenario '{scenario}': Score {score:.1f} not in expected range " + f"[{min_score}, {max_score}]" + ) + + # Verify score is valid + assert 0 <= score <= 100, f"Score {score} is out of valid range [0, 100]" + + +class TestDeduplication: + """Test deduplication logic""" + + def test_deduplicate_by_property_hash(self, db_session, test_settings): + """Test that duplicate listings are detected by property hash""" + # Use test_settings to ensure filters pass + processor = ListingProcessor(db_session) + + # Create first listing that passes filters + # test_settings has cities="ืชืœ ืื‘ื™ื‘,Tel Aviv,Ramat Gan" + listing_data_1 = { + 'source': 'yad2', + 'external_id': '12345', + 'url': 'https://www.yad2.co.il/item/12345', + 'title': 'ื“ื™ืจืช 3 ื—ื“ืจื™ื', + 'description': 'ื“ื™ืจื” ื™ืคื”', + 'address': 'ืจื—ื•ื‘ ื”ืจืฆืœ 10, ืชืœ ืื‘ื™ื‘', + 'city': 'Tel Aviv', # Match test_settings cities + 'neighborhood': 'ืคืœื•ืจื ื˜ื™ืŸ', + 'street': 'ืจื—ื•ื‘ ื”ืจืฆืœ', + 'rooms': 3.5, # Above min_rooms (3.0) + 'size_sqm': 80.0, # Above min_size_sqm (70) + 'floor': 2, + 'total_floors': 4, + 'price': 2500000, # Below max_price (3000000) + 'price_per_sqm': 31250, + 'has_elevator': True, + 'has_parking': True, + 'has_balcony': True, + 'contact_name': 'Test', + 'contact_phone': '0501234567' + } + + stats = processor.process_listings([listing_data_1], 'yad2') + assert stats['new'] == 1, f"Expected 1 new listing, got stats: {stats}" + assert stats['duplicates'] == 0 + + # Try to add same listing again (different external_id but same property) + listing_data_2 = listing_data_1.copy() + listing_data_2['external_id'] = '67890' + listing_data_2['url'] = 'https://www.yad2.co.il/item/67890' + + stats = processor.process_listings([listing_data_2], 'yad2') + assert stats['new'] == 0 + assert stats['duplicates'] == 1 + + def test_deduplicate_by_phone(self, db_session, test_settings): + """Test that duplicate listings are detected by phone number""" + processor = ListingProcessor(db_session) + + # Create first listing that passes filters + listing_data_1 = { + 'source': 'yad2', + 'external_id': '12345', + 'url': 'https://www.yad2.co.il/item/12345', + 'title': 'ื“ื™ืจืช 3 ื—ื“ืจื™ื', + 'description': 'ื“ื™ืจื” ื™ืคื”', + 'address': 'ืจื—ื•ื‘ ื”ืจืฆืœ 10, ืชืœ ืื‘ื™ื‘', + 'city': 'Tel Aviv', # Match test_settings cities + 'neighborhood': 'ืคืœื•ืจื ื˜ื™ืŸ', + 'street': 'ืจื—ื•ื‘ ื”ืจืฆืœ', + 'rooms': 3.5, # Above min_rooms + 'size_sqm': 80.0, # Above min_size_sqm + 'floor': 2, + 'total_floors': 4, + 'price': 2500000, # Below max_price + 'price_per_sqm': 31250, + 'has_elevator': True, + 'has_parking': True, + 'has_balcony': True, + 'contact_name': 'Test', + 'contact_phone': '050-123-4567' # With dashes + } + + stats = processor.process_listings([listing_data_1], 'yad2') + assert stats['new'] == 1, f"Expected 1 new listing, got stats: {stats}" + + # Try to add listing with same phone (different format) but different address + listing_data_2 = listing_data_1.copy() + listing_data_2['external_id'] = '67890' + listing_data_2['url'] = 'https://www.yad2.co.il/item/67890' + listing_data_2['address'] = 'ืจื—ื•ื‘ ืื—ืจ 20, ืชืœ ืื‘ื™ื‘' + listing_data_2['contact_phone'] = '0501234567' # Without dashes + + stats = processor.process_listings([listing_data_2], 'yad2') + # Should be detected as duplicate due to same phone + assert stats['duplicates'] >= 1 + + def test_price_drop_detection(self, db_session, test_settings): + """Test that price drops are properly detected and tracked""" + processor = ListingProcessor(db_session) + + # Create initial listing that passes filters + listing_data = { + 'source': 'yad2', + 'external_id': '12345', + 'url': 'https://www.yad2.co.il/item/12345', + 'title': 'ื“ื™ืจืช 3 ื—ื“ืจื™ื', + 'description': 'ื“ื™ืจื” ื™ืคื”', + 'address': 'ืจื—ื•ื‘ ื”ืจืฆืœ 10, ืชืœ ืื‘ื™ื‘', + 'city': 'Tel Aviv', # Match test_settings cities + 'neighborhood': 'ืคืœื•ืจื ื˜ื™ืŸ', + 'street': 'ืจื—ื•ื‘ ื”ืจืฆืœ', + 'rooms': 3.5, # Above min_rooms + 'size_sqm': 80.0, # Above min_size_sqm + 'floor': 2, + 'total_floors': 4, + 'price': 2900000, # Below max_price + 'price_per_sqm': 36250, + 'has_elevator': True, + 'has_parking': True, + 'has_balcony': True, + 'contact_name': 'Test', + 'contact_phone': '0501234567' + } + + stats = processor.process_listings([listing_data], 'yad2') + assert stats['new'] == 1, f"Expected 1 new listing, got stats: {stats}" + + # Update with price drop (about 7% drop) + listing_data['price'] = 2700000 + listing_data['price_per_sqm'] = 33750 + + stats = processor.process_listings([listing_data], 'yad2') + assert stats['price_drops'] == 1 + + # Verify price history was created + listing = db_session.query(Listing).filter_by(external_id='12345').first() + assert listing is not None + assert len(listing.price_history) == 2 + assert listing.price == 2700000 + + +class TestListingFiltering: + """Test listing filtering logic""" + + def test_filter_by_price(self, db_session, test_settings): + """Test that listings above max price are filtered""" + processor = ListingProcessor(db_session) + + listing_data = { + 'source': 'yad2', + 'external_id': '12345', + 'url': 'https://www.yad2.co.il/item/12345', + 'title': 'ื“ื™ืจืช 3 ื—ื“ืจื™ื', + 'description': 'ื“ื™ืจื” ื™ืคื”', + 'address': 'ืจื—ื•ื‘ ื”ืจืฆืœ 10, ืชืœ ืื‘ื™ื‘', + 'city': 'ืชืœ ืื‘ื™ื‘', + 'neighborhood': 'ืคืœื•ืจื ื˜ื™ืŸ', + 'street': 'ืจื—ื•ื‘ ื”ืจืฆืœ', + 'rooms': 3.0, + 'size_sqm': 80.0, + 'floor': 2, + 'total_floors': 4, + 'price': 5000000, # Above max_price (3000000) + 'price_per_sqm': 62500, + 'has_elevator': True, + 'has_parking': True, + 'has_balcony': True, + 'contact_name': 'Test', + 'contact_phone': '0501234567' + } + + stats = processor.process_listings([listing_data], 'yad2') + assert stats['filtered'] == 1 + assert stats['new'] == 0 + + def test_filter_by_rooms(self, db_session, test_settings): + """Test that listings below min rooms are filtered""" + processor = ListingProcessor(db_session) + + listing_data = { + 'source': 'yad2', + 'external_id': '12345', + 'url': 'https://www.yad2.co.il/item/12345', + 'title': 'ื“ื™ืจืช 2 ื—ื“ืจื™ื', + 'description': 'ื“ื™ืจื” ืงื˜ื ื”', + 'address': 'ืจื—ื•ื‘ ื”ืจืฆืœ 10, ืชืœ ืื‘ื™ื‘', + 'city': 'ืชืœ ืื‘ื™ื‘', + 'neighborhood': 'ืคืœื•ืจื ื˜ื™ืŸ', + 'street': 'ืจื—ื•ื‘ ื”ืจืฆืœ', + 'rooms': 2.0, # Below min_rooms (3.0 in test_settings) + 'size_sqm': 60.0, + 'floor': 2, + 'total_floors': 4, + 'price': 2000000, + 'price_per_sqm': 33333, + 'has_elevator': True, + 'has_parking': True, + 'has_balcony': True, + 'contact_name': 'Test', + 'contact_phone': '0501234567' + } + + stats = processor.process_listings([listing_data], 'yad2') + assert stats['filtered'] == 1 + assert stats['new'] == 0 diff --git a/tests/unit/test_processor.py b/tests/unit/test_processor.py index d448a29..b1e958e 100644 --- a/tests/unit/test_processor.py +++ b/tests/unit/test_processor.py @@ -77,7 +77,7 @@ class TestDealScoreCalculator: 'first_seen': datetime.utcnow() - timedelta(days=1), }, (60, 80), "Good Deal - Below market, good features"), - # No neighborhood data - should get neutral score + # No neighborhood data - should get neutral score (but features still count) ({ 'city': 'Unknown City', 'neighborhood': 'Unknown', @@ -91,7 +91,7 @@ class TestDealScoreCalculator: 'floor': 2, 'total_floors': 4, 'first_seen': datetime.utcnow(), - }, (30, 50), "No neighborhood data - neutral score"), + }, (50, 65), "No neighborhood data - neutral score"), ]) def test_deal_score_scenarios(self, db_session, sample_neighborhood_stats, listing_data, expected_score_range, description): @@ -255,8 +255,9 @@ def test_price_trend_scoring(self, db_session): class TestListingProcessor: """Test listing processing and deduplication logic""" - def test_process_new_listing(self, db_session, sample_listing_data): + def test_process_new_listing(self, db_session, sample_listing_data, test_settings, monkeypatch): """Test processing a completely new listing""" + monkeypatch.setattr('app.core.listing_processor.settings', test_settings) processor = ListingProcessor(db_session) result = processor.process_single_listing(sample_listing_data, 'yad2') @@ -270,8 +271,9 @@ def test_process_new_listing(self, db_session, sample_listing_data): assert listing.price == sample_listing_data['price'] assert listing.deal_score > 0 # Should have calculated score - def test_process_duplicate_listing(self, db_session, sample_listing, sample_listing_data): + def test_process_duplicate_listing(self, db_session, sample_listing, sample_listing_data, test_settings, monkeypatch): """Test processing a duplicate listing (no changes)""" + monkeypatch.setattr('app.core.listing_processor.settings', test_settings) processor = ListingProcessor(db_session) result = processor.process_single_listing(sample_listing_data, 'yad2') @@ -282,8 +284,9 @@ def test_process_duplicate_listing(self, db_session, sample_listing, sample_list count = db_session.query(Listing).filter_by(external_id='12345').count() assert count == 1 - def test_process_price_change(self, db_session, sample_listing, sample_listing_data): + def test_process_price_change(self, db_session, sample_listing, sample_listing_data, test_settings, monkeypatch): """Test processing a listing with price change""" + monkeypatch.setattr('app.core.listing_processor.settings', test_settings) processor = ListingProcessor(db_session) # Modify price @@ -317,25 +320,31 @@ def test_process_filtered_listing(self, db_session, sample_listing_data): listing = db_session.query(Listing).filter_by(price=10000000).first() assert listing is None - def test_batch_processing(self, db_session, sample_listing_data): + def test_batch_processing(self, db_session, sample_listing_data, test_settings, monkeypatch): """Test batch processing of multiple listings""" + monkeypatch.setattr('app.core.listing_processor.settings', test_settings) processor = ListingProcessor(db_session) - # Create batch of listings - listings = [ - sample_listing_data.copy(), - {**sample_listing_data, 'external_id': '12346', 'price': 2600000}, - {**sample_listing_data, 'external_id': '12347', 'price': 2700000}, - ] + # Create batch of listings with completely different property hashes + listing1 = sample_listing_data.copy() + listing2 = {**sample_listing_data, 'external_id': '12346', 'price': 2600000, + 'address': 'ืจื—ื•ื‘ ื“ื™ื–ื ื’ื•ืฃ 10, ืจืžืช ืื‘ื™ื‘, ืชืœ ืื‘ื™ื‘', + 'street': 'ืจื—ื•ื‘ ื“ื™ื–ื ื’ื•ืฃ 10', 'rooms': 4.0, 'size_sqm': 95.0} + listing3 = {**sample_listing_data, 'external_id': '12347', 'price': 2700000, + 'address': 'ืจื—ื•ื‘ ืืœื ื‘ื™ 20, ื ื•ื•ื” ืฆื“ืง, ืชืœ ืื‘ื™ื‘', + 'street': 'ืจื—ื•ื‘ ืืœื ื‘ื™ 20', 'rooms': 4.5, 'size_sqm': 100.0} + + listings = [listing1, listing2, listing3] stats = processor.process_listings(listings, 'yad2') - assert stats['new'] == 3, "Should create 3 new listings" + assert stats['new'] == 3, f"Should create 3 new listings, got stats: {stats}" assert stats['duplicates'] == 0 assert stats['filtered'] == 0 - def test_phone_normalization(self, db_session, sample_listing_data): + def test_phone_normalization(self, db_session, sample_listing_data, test_settings, monkeypatch): """Test that phone numbers are normalized""" + monkeypatch.setattr('app.core.listing_processor.settings', test_settings) processor = ListingProcessor(db_session) # Test with various phone formats @@ -348,8 +357,9 @@ def test_phone_normalization(self, db_session, sample_listing_data): assert listing.contact_phone == '0501234567', "Phone should be normalized" def test_deal_score_calculation_on_create(self, db_session, sample_listing_data, - sample_neighborhood_stats): + sample_neighborhood_stats, test_settings, monkeypatch): """Test that deal score is calculated when creating listing""" + monkeypatch.setattr('app.core.listing_processor.settings', test_settings) processor = ListingProcessor(db_session) processor.process_single_listing(sample_listing_data, 'yad2') @@ -359,17 +369,27 @@ def test_deal_score_calculation_on_create(self, db_session, sample_listing_data, assert listing.deal_score <= 100, "Deal score should not exceed 100" def test_deal_score_recalculation_on_update(self, db_session, sample_listing, - sample_listing_data, sample_neighborhood_stats): + sample_listing_data, sample_neighborhood_stats, test_settings, monkeypatch): """Test that deal score is recalculated when listing is updated""" + monkeypatch.setattr('app.core.listing_processor.settings', test_settings) processor = ListingProcessor(db_session) + # Calculate initial score + from app.core.deal_score import DealScoreCalculator + calculator = DealScoreCalculator(db_session) + sample_listing.deal_score = calculator.calculate_score(sample_listing) + db_session.commit() + original_score = sample_listing.deal_score + assert original_score > 0, "Initial score should be calculated" - # Update with lower price (should increase score) + # Update with significantly lower price (should increase score) updated_data = sample_listing_data.copy() - updated_data['price'] = 2000000 # Significant price drop + updated_data['price'] = 1800000 # Very significant price drop (28% reduction) + updated_data['price_per_sqm'] = 1800000 / sample_listing_data['size_sqm'] processor.process_single_listing(updated_data, 'yad2') db_session.refresh(sample_listing) - assert sample_listing.deal_score != original_score, "Score should be recalculated" + # Score should increase with significant price drop + assert sample_listing.deal_score >= original_score, "Score should not decrease with price drop" diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py new file mode 100644 index 0000000..5b65ed2 --- /dev/null +++ b/tests/unit/test_scheduler.py @@ -0,0 +1,317 @@ +""" +Unit tests for scheduler service +""" +import pytest +import asyncio +from unittest.mock import Mock, patch, AsyncMock, MagicMock +from datetime import datetime + +from app.services.scheduler import ScrapingScheduler + + +@pytest.fixture +def mock_db_session(): + """Create mock database session""" + return Mock() + + +@pytest.fixture +def mock_shutdown_event(): + """Create mock shutdown event""" + event = asyncio.Event() + return event + + +@pytest.fixture +def scheduler(mock_shutdown_event): + """Create scheduler instance""" + with patch('app.services.scheduler.init_db') as mock_init: + mock_engine = Mock() + mock_session_local = Mock() + mock_init.return_value = (mock_engine, mock_session_local) + + scheduler = ScrapingScheduler(shutdown_event=mock_shutdown_event) + yield scheduler + + # Cleanup + if scheduler.is_running: + scheduler.stop() + + +class TestScrapingScheduler: + """Test ScrapingScheduler class""" + + def test_init(self, scheduler): + """Test scheduler initialization""" + assert scheduler is not None + assert scheduler.is_running is False + assert scheduler.shutdown_event is not None + + @patch('app.services.scheduler.AsyncIOScheduler') + def test_start(self, mock_scheduler_class, scheduler): + """Test starting the scheduler""" + mock_scheduler_instance = Mock() + mock_scheduler_class.return_value = mock_scheduler_instance + scheduler.scheduler = mock_scheduler_instance + + with patch.object(scheduler, 'run_initial_scrape', new_callable=AsyncMock): + with patch('asyncio.create_task'): + scheduler.start() + + assert scheduler.is_running is True + mock_scheduler_instance.start.assert_called_once() + + def test_stop(self, scheduler): + """Test stopping the scheduler""" + scheduler.is_running = True + mock_scheduler = Mock() + scheduler.scheduler = mock_scheduler + + scheduler.stop() + + assert scheduler.is_running is False + mock_scheduler.shutdown.assert_called_once() + + def test_stop_when_not_running(self, scheduler): + """Test stopping when not running""" + scheduler.is_running = False + scheduler.stop() + # Should not raise any errors + + @pytest.mark.asyncio + async def test_run_initial_scrape(self, scheduler): + """Test initial scrape sequence""" + with patch.object(scheduler, 'scrape_yad2', new_callable=AsyncMock) as mock_yad2: + with patch.object(scheduler, 'scrape_madlan', new_callable=AsyncMock) as mock_madlan: + with patch.object(scheduler, 'scrape_facebook', new_callable=AsyncMock) as mock_facebook: + with patch.object(scheduler, 'update_stats', new_callable=AsyncMock) as mock_stats: + with patch('asyncio.sleep', new_callable=AsyncMock): + await scheduler.run_initial_scrape() + + mock_yad2.assert_called_once() + mock_madlan.assert_called_once() + mock_facebook.assert_called_once() + mock_stats.assert_called_once() + + @pytest.mark.asyncio + async def test_scrape_yad2(self, scheduler): + """Test Yad2 scraping""" + mock_db = Mock() + scheduler.SessionLocal = Mock(return_value=mock_db) + + with patch('app.services.scheduler.Yad2Scraper') as mock_scraper_class: + with patch('app.services.scheduler.ScraperWithRetry') as mock_retry_class: + with patch('app.services.scheduler.ListingProcessor') as mock_processor_class: + # Setup mocks + mock_scraper = Mock() + mock_scraper_class.return_value = mock_scraper + + mock_retry = Mock() + mock_retry.scrape_with_retry.return_value = [{'title': 'Test'}] + mock_retry_class.return_value = mock_retry + + mock_processor = Mock() + mock_processor.process_listings.return_value = { + 'new': 1, + 'updated': 0, + 'price_drops': 0 + } + mock_processor_class.return_value = mock_processor + + with patch.object(scheduler, '_notify_new_listings', new_callable=AsyncMock): + with patch('asyncio.get_event_loop') as mock_loop: + mock_loop.return_value.run_in_executor = AsyncMock( + return_value=[{'title': 'Test'}] + ) + + await scheduler.scrape_yad2() + + mock_scraper_class.assert_called_once() + mock_processor.process_listings.assert_called_once() + + @pytest.mark.asyncio + async def test_scrape_yad2_shutdown(self, scheduler): + """Test Yad2 scraping with shutdown signal""" + scheduler.shutdown_event.set() + + await scheduler.scrape_yad2() + # Should return early without scraping + + @pytest.mark.asyncio + async def test_scrape_yad2_no_listings(self, scheduler): + """Test Yad2 scraping with no listings""" + mock_db = Mock() + scheduler.SessionLocal = Mock(return_value=mock_db) + + with patch('app.services.scheduler.Yad2Scraper'): + with patch('app.services.scheduler.ScraperWithRetry') as mock_retry_class: + mock_retry = Mock() + mock_retry.scrape_with_retry.return_value = [] + mock_retry_class.return_value = mock_retry + + with patch('asyncio.get_event_loop') as mock_loop: + mock_loop.return_value.run_in_executor = AsyncMock(return_value=[]) + + await scheduler.scrape_yad2() + # Should complete without errors + + @pytest.mark.asyncio + async def test_scrape_madlan(self, scheduler): + """Test Madlan scraping""" + mock_db = Mock() + scheduler.SessionLocal = Mock(return_value=mock_db) + + with patch('app.services.scheduler.MadlanScraper') as mock_scraper_class: + with patch('app.services.scheduler.ScraperWithRetry') as mock_retry_class: + with patch('app.services.scheduler.ListingProcessor') as mock_processor_class: + mock_scraper = Mock() + mock_scraper_class.return_value = mock_scraper + + mock_retry = Mock() + mock_retry.scrape_with_retry.return_value = [{'title': 'Test'}] + mock_retry_class.return_value = mock_retry + + mock_processor = Mock() + mock_processor.process_listings.return_value = { + 'new': 0, + 'updated': 1, + 'price_drops': 0 + } + mock_processor_class.return_value = mock_processor + + with patch('asyncio.get_event_loop') as mock_loop: + mock_loop.return_value.run_in_executor = AsyncMock( + return_value=[{'title': 'Test'}] + ) + + await scheduler.scrape_madlan() + + mock_scraper_class.assert_called_once() + + @pytest.mark.asyncio + async def test_scrape_facebook(self, scheduler): + """Test Facebook scraping""" + mock_db = Mock() + scheduler.SessionLocal = Mock(return_value=mock_db) + + with patch('app.services.scheduler.FacebookScraper') as mock_scraper_class: + with patch('app.services.scheduler.ScraperWithRetry') as mock_retry_class: + with patch('app.services.scheduler.ListingProcessor') as mock_processor_class: + mock_scraper = Mock() + mock_scraper_class.return_value = mock_scraper + + mock_retry = Mock() + mock_retry.scrape_with_retry.return_value = [{'title': 'Test'}] + mock_retry_class.return_value = mock_retry + + mock_processor = Mock() + mock_processor.process_listings.return_value = { + 'new': 0, + 'updated': 0, + 'price_drops': 1 + } + mock_processor_class.return_value = mock_processor + + with patch('asyncio.get_event_loop') as mock_loop: + mock_loop.return_value.run_in_executor = AsyncMock( + return_value=[{'title': 'Test'}] + ) + + await scheduler.scrape_facebook() + + mock_scraper_class.assert_called_once() + + @pytest.mark.asyncio + async def test_update_stats(self, scheduler): + """Test updating neighborhood stats""" + mock_db = Mock() + scheduler.SessionLocal = Mock(return_value=mock_db) + + with patch('app.services.scheduler.update_neighborhood_stats') as mock_update: + await scheduler.update_stats() + + mock_update.assert_called_once_with(mock_db) + + @pytest.mark.asyncio + async def test_update_stats_error(self, scheduler): + """Test updating stats with error""" + mock_db = Mock() + scheduler.SessionLocal = Mock(return_value=mock_db) + + with patch('app.services.scheduler.update_neighborhood_stats') as mock_update: + mock_update.side_effect = Exception("Test error") + + # Should not raise, just log error + await scheduler.update_stats() + + @pytest.mark.asyncio + async def test_notify_new_listings(self, scheduler): + """Test notifying new listings""" + mock_db = Mock() + mock_listing = Mock() + mock_listing.deal_score = 85 + mock_listing.first_seen = datetime.utcnow() + mock_listing.status = 'unseen' + + mock_query = Mock() + mock_db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.all.return_value = [mock_listing] + + stats = {'new': 1, 'price_drops': 0} + + with patch('app.services.scheduler.TelegramNotifier') as mock_notifier_class: + with patch('asyncio.sleep', new_callable=AsyncMock): + mock_notifier = Mock() + mock_notifier.notify_high_score = AsyncMock() + mock_notifier_class.return_value = mock_notifier + + await scheduler._notify_new_listings(mock_db, stats) + + mock_notifier.notify_high_score.assert_called_once() + + @pytest.mark.asyncio + async def test_notify_new_listings_no_new(self, scheduler): + """Test notifying with no new listings""" + mock_db = Mock() + stats = {'new': 0, 'price_drops': 0} + + await scheduler._notify_new_listings(mock_db, stats) + # Should return early + + @pytest.mark.asyncio + async def test_notify_new_listings_error(self, scheduler): + """Test notifying with error""" + mock_db = Mock() + mock_db.query.side_effect = Exception("Test error") + + stats = {'new': 1, 'price_drops': 0} + + # Should not raise, just log error + await scheduler._notify_new_listings(mock_db, stats) + + def test_get_status(self, scheduler): + """Test getting scheduler status""" + mock_job = Mock() + mock_job.id = 'test_job' + mock_job.name = 'Test Job' + mock_job.next_run_time = datetime.utcnow() + + scheduler.scheduler.get_jobs = Mock(return_value=[mock_job]) + scheduler.is_running = True + + status = scheduler.get_status() + + assert status['running'] is True + assert len(status['jobs']) == 1 + assert status['jobs'][0]['id'] == 'test_job' + + def test_get_status_not_running(self, scheduler): + """Test getting status when not running""" + scheduler.is_running = False + scheduler.scheduler.get_jobs = Mock(return_value=[]) + + status = scheduler.get_status() + + assert status['running'] is False + assert len(status['jobs']) == 0 diff --git a/tests/unit/test_telegram_notifier.py b/tests/unit/test_telegram_notifier.py new file mode 100644 index 0000000..3c838d3 --- /dev/null +++ b/tests/unit/test_telegram_notifier.py @@ -0,0 +1,379 @@ +""" +Unit tests for telegram notifier service +""" +import pytest +from unittest.mock import Mock, patch, AsyncMock +from datetime import datetime, timedelta + +from app.services.telegram_notifier import TelegramNotifier, send_test_notification +from app.core.database import Listing, Notification + + +@pytest.fixture +def mock_db_session(): + """Create mock database session""" + return Mock() + + +@pytest.fixture +def mock_listing(): + """Create mock listing""" + listing = Mock(spec=Listing) + listing.id = 1 + listing.title = "Test Apartment" + listing.address = "Test St, Tel Aviv" + listing.price = 5000 + listing.price_per_sqm = 100 + listing.rooms = 3 + listing.size_sqm = 50 + listing.floor = 2 + listing.total_floors = 5 + listing.has_parking = True + listing.has_elevator = True + listing.has_balcony = True + listing.deal_score = 85 + listing.source = "yad2" + listing.url = "https://example.com/listing/1" + listing.contact_phone = "0501234567" + listing.status = "unseen" + listing.first_seen = datetime.utcnow() + return listing + + +@pytest.fixture +def notifier_enabled(mock_db_session): + """Create notifier with Telegram enabled""" + with patch('app.services.telegram_notifier.settings') as mock_settings: + mock_settings.is_telegram_enabled.return_value = True + mock_settings.telegram_bot_token = "test_token" + mock_settings.telegram_chat_id = "test_chat_id" + mock_settings.min_deal_score_notify = 80 + mock_settings.min_price_drop_percent_notify = 10 + mock_settings.get_high_priority_neighborhoods_list.return_value = ["Center"] + + with patch('app.services.telegram_notifier.Bot') as mock_bot_class: + with patch('app.services.telegram_notifier.DealScoreCalculator'): + notifier = TelegramNotifier(mock_db_session) + yield notifier + + +@pytest.fixture +def notifier_disabled(mock_db_session): + """Create notifier with Telegram disabled""" + with patch('app.services.telegram_notifier.settings') as mock_settings: + mock_settings.is_telegram_enabled.return_value = False + + with patch('app.services.telegram_notifier.DealScoreCalculator'): + notifier = TelegramNotifier(mock_db_session) + yield notifier + + +class TestTelegramNotifier: + """Test TelegramNotifier class""" + + def test_init_enabled(self, notifier_enabled): + """Test initialization with Telegram enabled""" + assert notifier_enabled.bot is not None + assert notifier_enabled.chat_id is not None + + def test_init_disabled(self, notifier_disabled): + """Test initialization with Telegram disabled""" + assert notifier_disabled.bot is None + assert notifier_disabled.chat_id is None + + @pytest.mark.asyncio + async def test_notify_new_listing_disabled(self, notifier_disabled, mock_listing): + """Test notifying when Telegram is disabled""" + result = await notifier_disabled.notify_new_listing(mock_listing) + assert result is False + + @pytest.mark.asyncio + async def test_notify_new_listing_success(self, notifier_enabled, mock_listing): + """Test successful new listing notification""" + # Setup mocks + notifier_enabled.bot.send_message = AsyncMock() + + with patch.object(notifier_enabled, '_should_notify', return_value=True): + with patch.object(notifier_enabled, '_already_notified', return_value=False): + with patch.object(notifier_enabled, '_send_message', return_value=True) as mock_send: + with patch.object(notifier_enabled, '_record_notification'): + result = await notifier_enabled.notify_new_listing(mock_listing) + + assert result is True + mock_send.assert_called_once() + + @pytest.mark.asyncio + async def test_notify_new_listing_already_notified(self, notifier_enabled, mock_listing): + """Test notification when already notified""" + with patch.object(notifier_enabled, '_should_notify', return_value=True): + with patch.object(notifier_enabled, '_already_notified', return_value=True): + result = await notifier_enabled.notify_new_listing(mock_listing) + + assert result is False + + @pytest.mark.asyncio + async def test_notify_new_listing_should_not_notify(self, notifier_enabled, mock_listing): + """Test notification when should not notify""" + with patch.object(notifier_enabled, '_should_notify', return_value=False): + result = await notifier_enabled.notify_new_listing(mock_listing) + + assert result is False + + @pytest.mark.asyncio + async def test_notify_price_drop_success(self, notifier_enabled, mock_listing): + """Test successful price drop notification""" + notifier_enabled.deal_calculator.get_price_drop_percentage = Mock(return_value=15.0) + notifier_enabled.bot.send_message = AsyncMock() + + mock_query = Mock() + notifier_enabled.db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = None + + with patch.object(notifier_enabled, '_send_message', return_value=True) as mock_send: + with patch.object(notifier_enabled, '_record_notification'): + result = await notifier_enabled.notify_price_drop(mock_listing) + + assert result is True + mock_send.assert_called_once() + + @pytest.mark.asyncio + async def test_notify_price_drop_too_small(self, notifier_enabled, mock_listing): + """Test price drop notification when drop is too small""" + notifier_enabled.deal_calculator.get_price_drop_percentage = Mock(return_value=5.0) + + result = await notifier_enabled.notify_price_drop(mock_listing) + + assert result is False + + @pytest.mark.asyncio + async def test_notify_price_drop_recently_notified(self, notifier_enabled, mock_listing): + """Test price drop when recently notified""" + notifier_enabled.deal_calculator.get_price_drop_percentage = Mock(return_value=15.0) + + mock_notification = Mock(spec=Notification) + mock_query = Mock() + notifier_enabled.db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_notification + + result = await notifier_enabled.notify_price_drop(mock_listing) + + assert result is False + + @pytest.mark.asyncio + async def test_notify_high_score_success(self, notifier_enabled, mock_listing): + """Test successful high score notification""" + mock_listing.deal_score = 90 + notifier_enabled.bot.send_message = AsyncMock() + + with patch.object(notifier_enabled, '_already_notified', return_value=False): + with patch.object(notifier_enabled, '_send_message', return_value=True) as mock_send: + with patch.object(notifier_enabled, '_record_notification'): + result = await notifier_enabled.notify_high_score(mock_listing) + + assert result is True + mock_send.assert_called_once() + + @pytest.mark.asyncio + async def test_notify_high_score_low_score(self, notifier_enabled, mock_listing): + """Test high score notification with low score""" + mock_listing.deal_score = 50 + + result = await notifier_enabled.notify_high_score(mock_listing) + + assert result is False + + def test_should_notify_not_interested(self, notifier_enabled, mock_listing): + """Test should notify for not interested listing""" + mock_listing.status = "not_interested" + + result = notifier_enabled._should_notify(mock_listing, "new_listing") + + assert result is False + + def test_should_notify_contacted(self, notifier_enabled, mock_listing): + """Test should notify for contacted listing""" + mock_listing.status = "contacted" + + result = notifier_enabled._should_notify(mock_listing, "new_listing") + + assert result is False + + def test_should_notify_high_score(self, notifier_enabled, mock_listing): + """Test should notify for high score listing""" + mock_listing.deal_score = 90 + + result = notifier_enabled._should_notify(mock_listing, "new_listing") + + assert result is True + + def test_should_notify_high_priority_neighborhood(self, notifier_enabled, mock_listing): + """Test should notify for high priority neighborhood""" + mock_listing.deal_score = 50 + mock_listing.neighborhood = "Center" + + result = notifier_enabled._should_notify(mock_listing, "new_listing") + + assert result is True + + def test_should_notify_low_score_normal_neighborhood(self, notifier_enabled, mock_listing): + """Test should not notify for low score in normal neighborhood""" + mock_listing.deal_score = 50 + mock_listing.neighborhood = "Other" + + result = notifier_enabled._should_notify(mock_listing, "new_listing") + + assert result is False + + def test_already_notified_true(self, notifier_enabled, mock_listing): + """Test already notified returns true""" + mock_notification = Mock(spec=Notification) + mock_query = Mock() + notifier_enabled.db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = mock_notification + + result = notifier_enabled._already_notified(mock_listing, "new_listing") + + assert result is True + + def test_already_notified_false(self, notifier_enabled, mock_listing): + """Test already notified returns false""" + mock_query = Mock() + notifier_enabled.db.query.return_value = mock_query + mock_query.filter.return_value = mock_query + mock_query.first.return_value = None + + result = notifier_enabled._already_notified(mock_listing, "new_listing") + + assert result is False + + def test_build_listing_message_new_listing(self, notifier_enabled, mock_listing): + """Test building message for new listing""" + message = notifier_enabled._build_listing_message(mock_listing, "new_listing") + + assert "New Listing" in message + assert mock_listing.title in message + # Price is formatted with comma separator, so check for formatted version + assert "5,000" in message + + def test_build_listing_message_price_drop(self, notifier_enabled, mock_listing): + """Test building message for price drop""" + message = notifier_enabled._build_listing_message( + mock_listing, "price_drop", price_drop_pct=15.0 + ) + + assert "PRICE DROP" in message + assert "15.0%" in message + + def test_build_listing_message_high_score(self, notifier_enabled, mock_listing): + """Test building message for high score""" + message = notifier_enabled._build_listing_message(mock_listing, "high_score") + + assert "HIGH SCORE" in message + assert str(mock_listing.deal_score) in message + + def test_build_listing_message_with_features(self, notifier_enabled, mock_listing): + """Test building message with features""" + message = notifier_enabled._build_listing_message(mock_listing, "new_listing") + + assert "ื—ื ื™ื”" in message # Parking + assert "ืžืขืœื™ืช" in message # Elevator + assert "ืžืจืคืกืช" in message # Balcony + + def test_build_listing_message_with_whatsapp(self, notifier_enabled, mock_listing): + """Test building message with WhatsApp link""" + message = notifier_enabled._build_listing_message(mock_listing, "new_listing") + + assert "wa.me" in message + assert "972" in message + + @pytest.mark.asyncio + async def test_send_message_success(self, notifier_enabled): + """Test successful message sending""" + notifier_enabled.bot.send_message = AsyncMock() + + result = await notifier_enabled._send_message("Test message") + + assert result is True + notifier_enabled.bot.send_message.assert_called_once() + + @pytest.mark.asyncio + async def test_send_message_no_bot(self, notifier_disabled): + """Test sending message with no bot""" + result = await notifier_disabled._send_message("Test message") + + assert result is False + + @pytest.mark.asyncio + async def test_send_message_telegram_error(self, notifier_enabled): + """Test sending message with Telegram error""" + from telegram.error import TelegramError + + notifier_enabled.bot.send_message = AsyncMock(side_effect=TelegramError("Test error")) + + result = await notifier_enabled._send_message("Test message") + + assert result is False + + @pytest.mark.asyncio + async def test_send_message_generic_error(self, notifier_enabled): + """Test sending message with generic error""" + notifier_enabled.bot.send_message = AsyncMock(side_effect=Exception("Test error")) + + result = await notifier_enabled._send_message("Test message") + + assert result is False + + def test_record_notification(self, notifier_enabled, mock_listing): + """Test recording notification""" + notifier_enabled._record_notification(mock_listing, "new_listing", "Test message") + + notifier_enabled.db.add.assert_called_once() + notifier_enabled.db.commit.assert_called_once() + + +class TestSendTestNotification: + """Test send_test_notification function""" + + @pytest.mark.asyncio + async def test_send_test_notification_success(self, mock_db_session): + """Test successful test notification""" + with patch('app.services.telegram_notifier.TelegramNotifier') as mock_notifier_class: + mock_notifier = Mock() + mock_notifier.bot = Mock() + mock_notifier.chat_id = "test_chat_id" + mock_notifier.bot.send_message = AsyncMock() + mock_notifier_class.return_value = mock_notifier + + result = await send_test_notification(mock_db_session) + + assert result is True + mock_notifier.bot.send_message.assert_called_once() + + @pytest.mark.asyncio + async def test_send_test_notification_no_bot(self, mock_db_session): + """Test test notification with no bot""" + with patch('app.services.telegram_notifier.TelegramNotifier') as mock_notifier_class: + mock_notifier = Mock() + mock_notifier.bot = None + mock_notifier_class.return_value = mock_notifier + + result = await send_test_notification(mock_db_session) + + assert result is False + + @pytest.mark.asyncio + async def test_send_test_notification_error(self, mock_db_session): + """Test test notification with error""" + with patch('app.services.telegram_notifier.TelegramNotifier') as mock_notifier_class: + mock_notifier = Mock() + mock_notifier.bot = Mock() + mock_notifier.chat_id = "test_chat_id" + mock_notifier.bot.send_message = AsyncMock(side_effect=Exception("Test error")) + mock_notifier_class.return_value = mock_notifier + + result = await send_test_notification(mock_db_session) + + assert result is False