parser.py

import dataclasses
import json
import os
import sys
import typing
from dataclasses import dataclass
from enum import StrEnum

import requests
from bs4 import BeautifulSoup, Tag

BASE_URL = "https://apps1.sis.yorku.ca/WebObjects/cdm.woa/Contents/WebServerResources"
COOKIE_FILE = "cookie.txt"

TERMS = {
    "all": {
        "F": {"start_date": "2026-09-09", "end_date": "2026-12-08"},
        "W": {"start_date": "2027-01-04", "end_date": "2027-04-05"},
        "Y": {"start_date": "2026-09-09", "end_date": "2027-04-05"},
    },
    "AP": {
        "WS": {"start_date": "2027-01-04", "end_date": "2027-08-03"},
    },
    "ED": {
        "B1": {"start_date": "2026-08-04", "end_date": "2026-09-18"},
        "B2": {"start_date": "2026-09-08", "end_date": "2026-12-23"},
        "B3": {"start_date": "2026-09-08", "end_date": "2027-04-30"},
        "B4": {"start_date": "2027-01-04", "end_date": "2027-04-23"},
        "FA": {"start_date": "2026-09-09", "end_date": "2026-10-27"},
        "FB": {"start_date": "2026-10-28", "end_date": "2026-12-08"},
        "FW": {"start_date": "2026-10-19", "end_date": "2027-04-30"},
        "RW": {"start_date": "2027-02-15", "end_date": "2027-02-19"},
        "WA": {"start_date": "2027-01-04", "end_date": "2027-02-12"},
        "WB": {"start_date": "2027-02-22", "end_date": "2027-04-05"},
        "WP": {"start_date": "2027-01-04", "end_date": "2027-06-04"},
    },
    "EU": {
        "O3": {"start_date": "2027-02-15", "end_date": "2027-03-26"},
    },
    "GL": {
        "WS": {"start_date": "2027-01-04", "end_date": "2027-08-03"},
    },
    "GS": {
        "E1": {"start_date": "2026-08-04", "end_date": "2026-08-28"},
        "FA": {"start_date": "2026-09-09", "end_date": "2026-10-27"},
        "FB": {"start_date": "2026-10-28", "end_date": "2026-12-08"},
        "FS": {"start_date": "2026-09-09", "end_date": "2027-08-03"},
        "NQ": {"start_date": "2026-08-24", "end_date": "2027-04-23"},
        "O3": {"start_date": "2027-02-15", "end_date": "2027-03-26"},
        "WA": {"start_date": "2027-01-04", "end_date": "2027-02-12"},
        "WB": {"start_date": "2027-02-22", "end_date": "2027-04-05"},
        "WS": {"start_date": "2027-01-04", "end_date": "2027-08-03"},
    },
    "HH": {
        "GH": {"start_date": "2027-04-07", "end_date": "2027-04-23"},
    },
    "LE": {
        "A1": {"start_date": "2026-09-09", "end_date": "2026-10-23"},
        "A2": {"start_date": "2026-10-26", "end_date": "2026-12-08"},
        "A3": {"start_date": "2026-09-09", "end_date": "2026-10-27"},
        "A4": {"start_date": "2026-10-28", "end_date": "2026-12-08"},
        "B1": {"start_date": "2027-01-04", "end_date": "2027-02-12"},
        "F1": {"start_date": "2026-09-09", "end_date": "2026-09-15"},
        "F2": {"start_date": "2026-09-16", "end_date": "2026-10-27"},
        "F3": {"start_date": "2026-10-29", "end_date": "2026-12-02"},
        "F4": {"start_date": "2026-12-08", "end_date": "2026-12-22"},
        "FA": {"start_date": "2026-09-09", "end_date": "2026-10-27"},
        "FB": {"start_date": "2026-10-28", "end_date": "2026-12-08"},
        "FS": {"start_date": "2026-09-09", "end_date": "2027-08-03"},
        "LF": {"start_date": "2026-08-24", "end_date": "2026-09-04"},
        "M3": {"start_date": "2026-09-09", "end_date": "2026-09-18"},
        "M4": {"start_date": "2026-09-21", "end_date": "2026-11-06"},
        "M5": {"start_date": "2026-11-09", "end_date": "2026-11-27"},
        "M6": {"start_date": "2026-11-30", "end_date": "2026-12-08"},
        "W1": {"start_date": "2027-01-04", "end_date": "2027-01-22"},
        "W2": {"start_date": "2027-01-28", "end_date": "2027-02-24"},
        "W3": {"start_date": "2027-03-02", "end_date": "2027-03-22"},
        "W4": {"start_date": "2027-03-29", "end_date": "2027-04-16"},
        "WS": {"start_date": "2027-01-04", "end_date": "2027-08-03"},
    },
    "LW": {
        "FO": {"start_date": "2026-08-31", "end_date": "2026-11-27"},
        "WO": {"start_date": "2027-01-04", "end_date": "2027-04-02"},
        "YO": {"start_date": "2026-08-24", "end_date": "2027-04-02"},
    },
    "SB": {
        "A": {"start_date": "2026-09-14", "end_date": "2026-10-26"},
        "C": {"start_date": "2027-01-11", "end_date": "2027-02-22"},
        "E4": {"start_date": "2027-01-08", "end_date": "2027-10-03"},
        "EF": {"start_date": "2026-09-10", "end_date": "2026-12-31"},
        "ER": {"start_date": "2027-01-08", "end_date": "2027-01-31"},
        "EW": {"start_date": "2027-01-04", "end_date": "2027-04-30"},
        "F2": {"start_date": "2026-09-14", "end_date": "2026-12-11"},
        "F3": {"start_date": "2026-09-07", "end_date": "2026-12-08"},
        "F4": {"start_date": "2026-09-14", "end_date": "2026-09-21"},
        "FP": {"start_date": "2026-09-19", "end_date": "2026-12-13"},
        "M": {"start_date": "2026-11-02", "end_date": "2026-12-12"},
        "N": {"start_date": "2027-03-01", "end_date": "2027-04-12"},
        "W2": {"start_date": "2027-01-11", "end_date": "2027-04-12"},
        "W3": {"start_date": "2027-01-04", "end_date": "2027-04-05"},
        "W4": {"start_date": "2027-01-16", "end_date": "2027-01-23"},
        "WP": {"start_date": "2027-01-16", "end_date": "2027-04-11"},
        "WS": {"start_date": "2027-01-04", "end_date": "2027-08-03"},
        "Z1": {"start_date": "2026-09-14", "end_date": "2027-04-12"},
    },
    "SC": {
        "W4": {"start_date": "2027-03-29", "end_date": "2027-04-16"},
        "WS": {"start_date": "2027-01-04", "end_date": "2027-08-03"},
    },
}

LectureType = typing.Literal[
    "LECT",
    "ONLN",
    "ONCA",
    "BLEN",
    "DIRD",
    "FDEX",
    "SEMR",
    "LGCL",
    "TUTR",
    "THES",
    "LAB",
    "ISTY",
    "INSP",
    "PRAC",
    "HYFX",
    "FIEL",
    "WKSP",
    "STDO",
    "REEV",
    "CLIN",
    "RESP",
    "IDS",
    "COOP",
]


@dataclass(frozen=True)
class CourseKey:
    department: str
    code_letters: str
    code_numbers: str
    num_credits: str
    language: str
    title: str


class Weekday(StrEnum):
    MONDAY = "Monday"
    TUESDAY = "Tuesday"
    WEDNESDAY = "Wednesday"
    THURSDAY = "Thursday"
    FRIDAY = "Friday"
    SATURDAY = "Saturday"
    SUNDAY = "Sunday"


@dataclass
class Meeting:
    weekday: Weekday
    start: int
    end: int
    campus: str
    room: str


# A tutorial or lab attached to an offering.
@dataclass
class Activity:
    number: str
    meetings: list[Meeting]


@dataclass
class Offering:
    lecture_type: LectureType
    term: str
    section: str
    professor: str | None
    meetings: list[Meeting]
    tutorials: list[Activity]
    labs: list[Activity]


@dataclass
class Course:
    key: CourseKey
    offerings: list[Offering]


@dataclass(frozen=True)
class Selection:
    offering: Offering
    tutorial: Activity | None
    lab: Activity | None

    def meetings(self) -> list[Meeting]:
        result = list(self.offering.meetings)
        if self.tutorial:
            result += self.tutorial.meetings
        if self.lab:
            result += self.lab.meetings
        return result


@dataclass(frozen=True)
class Interval:
    start: int
    end: int


def main() -> None:
    with open("data/terms.json", "w", encoding="utf-8") as f:
        json.dump(TERMS, f, ensure_ascii=False, indent=2)

    # FW2026LW not released yet.
    faculties = ["AP", "ED", "EU", "FA", "GL", "GS", "HH", "LE", "SB", "SC"]
    filenames = [f"FW2026{f}.html" for f in faculties]

    download_files(BASE_URL, filenames)

    courses: list[Course] = []
    for filename in filenames:
        print(f"parsing {filename}", file=sys.stderr)
        courses += parse_file(f"downloads/{filename}")

    data = [dataclasses.asdict(course) for course in courses]

    with open("data/courses.json", "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)

    print(f"wrote {len(data)} courses")


def parse_file(filename: str) -> list[Course]:
    with open(filename, "rb") as f:
        # html5lib is used instead of html.parser because the latter fails
        # silently with malformed HTML, whereas with the former, what you see in
        # your browser is what you get.
        soup = BeautifulSoup(f, "html5lib")

    table = soup.find("table")
    assert table is not None, f"{filename}: table HTML tag not found"

    # html5lib follows browser rules and auto-inserts a `<tbody>`.
    tbody = table.find("tbody")
    assert tbody is not None, f"{filename}: tbody HTML tag not found"

    rows = tbody.find_all("tr", recursive=False)

    courses: dict[CourseKey, Course] = {}
    department: str | None = None
    course_term: str | None = None
    code_letters: str | None = None
    course_title: str | None = None
    offering: Offering | None = None

    valid_section = False

    for row in rows:
        cells = row.find_all("td", recursive=False)
        if not cells:
            continue

        kind = classify_row(cells)
        assert kind is not None, "Unrecognized row kind"

        if kind == "course":
            offering = None
            valid_section = False
            assert len(cells) == 4, f"Course row has {len(cells)} cells, expected 4"

            department = cells[0].get_text(strip=True)
            code_letters = cells[1].get_text(strip=True).upper()
            course_term = cells[2].get_text(strip=True)
            course_title = cells[3].get_text(strip=True)
        elif kind == "section":
            # assert course_term is not None, "Section has no course term"
            assert course_term in (
                "F",
                "F1",
                "F2",
                "F3",
                "F4",
                "FA",
                "FW",
                "W",
                "Y",
                "WS",
                "B2",
                "B4",
                "B1",
                "FB",
                "WA",
                "WB",
                "WP",
                "B3",
                "O3",
                "E1",
                "NQ",
                "FS",
                "GH",
                "A1",
                "A2",
                "A3",
                "M3",
                "M4",
                "M5",
                "M6",
                "A4",
                "W4",
                "W1",
                "W3",
                "W2",
                "LF",
                "C",
                "N",
                "A",
                "M",
                "EW",
                "ER",
                "EF",
                "Z1",
                "GY",  # new
                "RW",  # new
            ), f"Invalid course term {course_term}"

            # Some offerings list a section with no scheduling component at all
            # (3-cell row: number/credits/section + language, nothing else).
            valid_section = len(cells) >= 8
            if not valid_section:
                continue

            lecture_type = cells[3].get_text(strip=True).upper()
            assert lecture_type in typing.get_args(LectureType), (
                f"Unrecognized lecture type {lecture_type}"
            )

            lecture_type = typing.cast(LectureType, lecture_type)

            catalogue_num = cells[5].get_text(strip=True)
            catalogue_num = catalogue_num.upper()

            valid_section = catalogue_num != "CANCELLED"
            if not valid_section:
                continue

            cell_list = cells[1].get_text().split()
            code_numbers, num_credits, section = cell_list

            professor: str | None = cells[7].get_text(strip=True)
            if professor == "":
                professor = None

            language = cells[2].get_text(strip=True)

            offering = Offering(
                lecture_type=lecture_type,
                term=course_term,
                section=section,
                professor=professor,
                meetings=[],
                tutorials=[],
                labs=[],
            )

            # The section header is itself a component. cells[3] is its type
            # (LECT, TUTR, LAB, SEMR, ONLN, ...), so route its meetings by type
            # rather than assuming every header is a lecture.
            header_type = cells[3].get_text(strip=True).upper()
            header_number = cells[4].get_text(strip=True).upper()
            meetings_table = cells[6].find("table")

            assert header_type in typing.get_args(LectureType)
            header_type = typing.cast(LectureType, header_type)
            assert meetings_table is not None, "Cannot find meetings table"

            add_component(
                offering, header_type, header_number, get_meetings(meetings_table)
            )

            assert department is not None, "Missing department"
            assert code_letters is not None, "Missing course code letters"
            assert course_title is not None, "Missing course title"

            course_key = CourseKey(
                department=department,
                code_letters=code_letters,
                code_numbers=code_numbers,
                num_credits=num_credits,
                language=language,
                title=course_title,
            )

            if course_key not in courses:
                courses[course_key] = Course(key=course_key, offerings=[])

            courses[course_key].offerings.append(offering)
        elif (
            kind == "tutorial" or kind == "lab" or kind == "lecture"
        ) and valid_section:
            assert offering is not None, "Missing offering"

            catalogue_num = cells[3].get_text(strip=True)
            catalogue_num = catalogue_num.upper()
            if catalogue_num == "CANCELLED":
                continue

            component_type = cells[1].get_text(strip=True).upper()
            number = cells[2].get_text(strip=True)
            meetings_table = cells[4].find("table")
            assert meetings_table is not None, "Cannot find meetings table"

            assert component_type in typing.get_args(LectureType), (
                f"Unrecognized component type {LectureType}"
            )
            component_type = typing.cast(LectureType, component_type)

            add_component(
                offering, component_type, number, get_meetings(meetings_table)
            )

    return list(courses.values())


# Place a component's meetings onto an offering according to its type. TUTR and
# LAB are choose-one activities; every other type (LECT, SEMR, ONLN, ...) is part
# of the offering’s core meetings, which you always attend.
def add_component(
    offering: Offering,
    component_type: LectureType,
    number: str,
    meetings: list[Meeting],
) -> None:
    if component_type == "TUTR":
        offering.tutorials.append(Activity(number=number, meetings=meetings))
    elif component_type == "LAB":
        offering.labs.append(Activity(number=number, meetings=meetings))
    else:
        offering.meetings += meetings


def get_meetings(table: Tag) -> list[Meeting]:
    meetings: list[Meeting] = []

    tbody = table.find("tbody")
    assert tbody is not None, "tbody HTML tag not found"

    rows = tbody.find_all("tr", recursive=False)

    for row in rows:
        tds = row.find_all("td", recursive=False)
        assert len(tds) == 5, f"Meetings table row has {len(tds)} elements, expected 5"

        weekday_letter = tds[0].get_text(strip=True)
        weekday = get_weekday_from_letter(weekday_letter)
        start_time = tds[1].get_text(strip=True)
        duration = tds[2].get_text(strip=True)
        campus = tds[3].get_text(strip=True)
        room = tds[4].get_text(strip=True)

        if weekday is None:
            continue

        start = time_to_minutes(start_time)
        end = start + int(duration)

        meeting = Meeting(
            weekday=weekday, start=start, end=end, campus=campus, room=room
        )
        meetings.append(meeting)

    return meetings


def get_weekday_from_letter(letter: str) -> Weekday | None:
    match letter:
        case "M":
            return Weekday.MONDAY
        case "T":
            return Weekday.TUESDAY
        case "W":
            return Weekday.WEDNESDAY
        case "R":
            return Weekday.THURSDAY
        case "F":
            return Weekday.FRIDAY
        case "S":
            return Weekday.SATURDAY
        case "U":
            return Weekday.SUNDAY
        case "":
            return None
        case _:
            raise AssertionError(f"Unrecognized letter {letter}")


def classify_row(cells: list[Tag]) -> str | None:
    if any(c.get("colspan") == "8" for c in cells):
        return "course"
    if cells[0].get("colspan") == "3":
        return "section"
    if cells[0].get("colspan") == "5":
        cell = cells[1].get_text(strip=True).upper()
        if cell == "TUTR":
            return "tutorial"
        elif cell == "LAB":
            return "lab"
        else:
            return "lecture"
    if cells[0].get_text(strip=True).upper() == "FAC":
        return "header"
    return None


def download_files(base_url: str, filenames: list[str]) -> None:
    if not os.path.exists(COOKIE_FILE):
        sys.exit(
            f"Error: cookie file '{COOKIE_FILE}' not found in the current directory."
        )

    with open(COOKIE_FILE, "r") as f:
        cookie = f.read().strip()

    if not cookie:
        sys.exit(f"Error: cookie file '{COOKIE_FILE}' is empty.")

    for filename in filenames:
        response = requests.get(f"{base_url}/{filename}", headers={"Cookie": cookie})
        response.raise_for_status()

        with open(f"downloads/{filename}", "wb") as f:
            f.write(response.content)


def time_to_minutes(time: str):
    hours, minutes = map(int, time.split(":"))
    assert hours < 24 and minutes < 60, f"Invalid time {time}"
    return hours * 60 + minutes


if __name__ == "__main__":
    main()