font.py

import copy
import glob
import os
import sys
import platform
import subprocess
from collections import OrderedDict
from pathlib import Path
from typing import List, Iterable, Dict, Tuple
from iso639 import Language
from fontTools.ttLib import TTFont, TTLibError
from unicode_ranges import UnicodeRanges
from ttf import Compression, TrueTypeFont, TrueTypeFactory
from font_config import FontConfiguration, FontSize

from unt import read_unique_characters_from_unt_file
from localization import read_localization_config


def read_font_unicode_ranges(font_path: str) -> UnicodeRanges:
    font_unicode_ranges = UnicodeRanges()
    font_number = -1
    if font_path.endswith('.ttc'):
        font_number = 0
    with TTFont(font_path, fontNumber=font_number) as ttf:
        from itertools import chain
        x = chain.from_iterable(x.cmap.keys() for x in ttf['cmap'].tables)
        for ordinal in x:
            font_unicode_ranges.add_ordinal(ordinal)
    print(f'Font "{font_path}" supports {len(list(font_unicode_ranges.iter_ordinals()))} characters')
    return font_unicode_ranges


def get_language_extension(language_code: str):
    part3 = Language.from_part1(language_code).part3
    return iso639_to_extension.get(part3, part3)

class FontPackage:
    def __init__(self, mod: str, package_name: str):
        self.mod = mod
        self.package_name = package_name
        self.font_factories: Dict[str, TrueTypeFactory] = {}


class FontStyleItem:
    def __init__(self, font_index: int, resolution: int):
        self.font_index = font_index
        self.resolution = resolution


def get_font_resolutions_and_sizes(config: FontConfiguration, font_style_size: FontSize) -> Tuple[List[int], List[int]]:
    match font_style_size.method:
        case 'proportional':
            resolution_baseline = config.resolution.baseline
            resolution_groups = config.resolution.groups
            # Get the resolution group we're using.
            font_size_baseline = font_style_size.baseline
            resolution_group = font_style_size.resolution_group if font_style_size.resolution_group else config.defaults.resolution_group

                    # Make sure the resolution group exists.
            if resolution_group not in resolution_groups:
                raise Exception(f'Resolution group "{resolution_group}" does not exist')

            resolutions = resolution_groups[resolution_group]
            sizes = [int(font_size_baseline * resolution / resolution_baseline) for resolution in resolutions]
                    # Round the sizes round up to a multiple of 2 (stops there from being a 1px difference between
                    # sizes, wasting space).
            sizes = [size + 1 if size % 2 == 1 else size for size in sizes]
        case 'fixed':
            sizes = font_style_size.sizes
            resolutions = [0] * len(sizes)
        case _:
            raise Exception(f'Invalid size method: {font_style_size.method}, expected one of {["proportional", "fixed"]}')    
    if type(sizes) == int:
        sizes = [sizes]
    elif type(sizes) != list:
        raise Exception(f'Invalid sizes for font style: {sizes}')
    return resolutions, sizes


def write_font_style_class_file(path: Path, font_style_name: str, class_name: str):
    with open(path, 'w') as f:
        lines = [
            '//==============================================================================',
            '// This file was automatically generated by Unreal-Localization.',
            '// Do not edit this file directly.',
            '// To regenerate this file, run ./tools/localization/generate_fonts.bat',
            '//==============================================================================',
            '',
            f'class {font_style_name} extends GUIFont;',
            '',
            f'event Font GetFont(int ResX)',
            '{',
            f'    return Class\'{class_name}\'.static.Get{font_style_name}ByResolution(Controller.ResX, Controller.ResY);',
            '}',
            '',
            'defaultproperties',
            '{',
            f'    KeyName="{font_style_name}"',
            '}',
        ]
        for line in lines:
            f.write(line + '\n')


font_weight_to_styles = [
    (100, ['Thin', 'Hairline']),
    (200, ['ExtraLight', 'UltraLight']),
    (300, ['Light']),
    (400, ['Normal', 'Regular']),
    (500, ['Medium']),
    (600, ['SemiBold', 'DemiBold']),
    (700, ['Bold']),
    (800, ['ExtraBold', 'UltraBold']),
    (900, ['Black', 'Heavy']),
    (950, ['ExtraBlack', 'UltraBlack'])
]

font_style_to_weight = OrderedDict()
for weight, styles in font_weight_to_styles:
    for style in styles:
        font_style_to_weight[style] = weight


def get_font_styles_from_weight(weight: int) -> List[str]:
    weight_min = 100
    weight_max = 950
    weight = max(weight_min, min(weight, weight_max))
    for key, styles in reversed(font_weight_to_styles):
        if weight <= key:
            return styles
    return ['Normal', 'Regular']


def get_font_style_closest(installed_fonts, styles: Iterable[str], fontname: str) -> str | None:
    installed_font_styles = installed_fonts[fontname].keys()
    font_style_list = []
    for style in styles:
        weight = font_style_to_weight[style]
        for installed_font_style in installed_font_styles:
            if 'Italic' in installed_font_style:
                continue
            font_style_list.append((abs(weight - font_style_to_weight[installed_font_style]), installed_font_style))
    font_style_list.sort(key=lambda x: x[0])
    return installed_fonts[fontname][font_style_list[0][1]]


def get_font_paths() -> List[str]:
    font_paths = set()
    match platform.system():
        case 'Linux':
            import shutil
            if shutil.which('fc-list') is None:
                raise RuntimeError('fc-list is not installed')
            fc_list_output = subprocess.run(['fc-list', '--format=%{file}\n'], capture_output=True, text=True).stdout
            font_paths |= set([x for x in fc_list_output.split('\n') if x])
        case 'Windows':
            from os import walk
            font_directories = [
                Path(r'C:\Windows\Fonts').resolve(),
                Path(fr'{os.getenv("LOCALAPPDATA")}\Microsoft\Windows\Fonts').resolve(),
            ]
            font_extensions = ['.ttf', '.otf', '.ttc', '.ttz', '.woff', '.woff2']
            for font_directory in font_directories:
                for (dirpath, dirnames, filenames) in walk(font_directory):
                    for filename in filenames:
                        if any(filename.endswith(ext) for ext in font_extensions):
                            font_paths.add(dirpath.replace('\\\\', '\\') + '\\' + filename)
        case _:
            raise RuntimeError(f'Unhandled platform: {platform.system()}')
    return list(font_paths)
            

# Get the list of all installed Fonts.
def get_installed_fonts() -> Dict[str, Dict[str, str]]:
    def get_font(font: TTFont, font_path: str):
        x = lambda x: font['name'].getDebugName(x)
        if x(16) is None:
            return x(1), x(2), font_path
        elif x(16) is not None:
            return x(16), x(17), font_path
        else:
            return None

    ttf_fonts = []
    font_paths = get_font_paths()

    for font_path in font_paths:
        # Ensure that we even have permission to read the font file.
        if not os.access(font_path, os.R_OK):
            continue

        if font_path.endswith('.ttc'):
            try:
                # Try to get the sizes of the font from 0 to 100.
                for font_index in range(100):
                    ttf_font = get_font(TTFont(font_path, fontNumber=font_index, lazy=True), font_path)
                    if ttf_font is not None:
                        ttf_fonts.append(ttf_font)
            except:
                pass
        elif font_path.endswith(('.ttf', '.otf', '.ttz', '.woff', '.woff2')):
            try:
                ttf_font = TTFont(font_path, lazy=True)
                ttf_fonts.append(get_font(ttf_font, font_path))
            except TTLibError as e:
                raise RuntimeError(f'Error reading font at "{font_path}": {e}') from e

    installed_fonts: Dict[str, Dict[str, str]] = {}
    for (family, style, path) in ttf_fonts:
        if family not in installed_fonts:
            installed_fonts[family] = {}
        installed_fonts[family][style] = path

    return installed_fonts


def write_fonts_class_file(
        fonts_package_name: str, 
        fonts: Iterable[TTFont], 
        font_style_items, 
        class_name: str,
        unrealscript_fonts_path: Path,
        ):
    lines = []
    lines += [
        '//==============================================================================',
        '// This file was automatically generated by Unreal-Localization.',
        '// Do not edit this file directly.',
        '// To regenerate this file, run ./tools/localization/generate_fonts.bat',
        '//==============================================================================',
        '',
    ]

    lines += [
        f'class {class_name} extends Object',
        '    abstract;',
        '',
        'struct FontStyleItem {',
        '    var int FontIndex;',
        '    var int Resolution;',
        '};',
        '',
        f'var localized string FontNames[{len(fonts)}];',
        f'var Font Fonts[{len(fonts)}];',
    ]

    # Create the string arrays for the font style.
    for font_style_name, items in font_style_items.items():
        lines.append(f'var FontStyleItem {font_style_name}Items[{len(items)}];')

    lines.append('')

    lines += [
        'static function Font GetFontByIndex(int i) {',
        '    if (default.Fonts[i] == none) {',
        '        default.Fonts[i] = Font(DynamicLoadObject(default.FontNames[i], Class\'Font\'));',
        '        if (default.Fonts[i] == none) {',
        '            Warn("Could not dynamically load" @ default.FontNames[i]);',
        '        }',
        '    }',
        '    return default.Fonts[i];',
        '}',
        '',
        'static function int GetEffectiveResolution(int ResX, int ResY)',
        '{',
        '    const BASELINE_ASPECT_RATIO = 1.7777777777777777777777777777778;    // 16:9 aspect ratio',
        '    return ResY * FMax(1.0, ((float(ResX) / float(ResY)) / BASELINE_ASPECT_RATIO));',
        '}',
        '',
    ]

    # Create the function to load the fonts.
    for font_style_name in font_style_items.keys():
        items_array_name = f'{font_style_name}Items'
        lines += [
            f'static function Font Get{font_style_name}ByIndex(int i) {{',
            f'    return GetFontByIndex(default.{items_array_name}[i].FontIndex);',
            f'}}',
            f'',
            f'// Load a font by the nearest target resolution',
            f'static function Font Get{font_style_name}ByResolution(int ResX, int ResY) {{',
            f'    local int i;',
            f'    ResY = GetEffectiveResolution(ResX, ResY);',
            f'    for (i = 0; i < arraycount(default.{items_array_name}); i++) {{',
            f'        if (ResY >= default.{items_array_name}[i].Resolution) {{',
            f'            return Get{font_style_name}ByIndex(i);',
            f'        }}',
            f'    }}',
            f'    return Get{font_style_name}ByIndex(arraycount(default.{items_array_name}) - 1);',
            f'}}',
            '',
        ]

    lines.append('defaultproperties')
    lines.append('{')

    # TODO: make sure we make a distiction between english and other localization's font substitutions!

    for font_index, (font_name, _) in enumerate(fonts.items()):
        lines.append(f'    FontNames({font_index})="{fonts_package_name}.{font_name}"')

    for font_style_name, items in font_style_items.items():
        for item_index, item in enumerate(items):
            lines.append(f'    {font_style_name}Items({item_index})=(FontIndex={item.font_index},Resolution={item.resolution})')

    lines.append('}')

    # Write the lines to the file.
    with open(unrealscript_fonts_path, 'w') as file:
        for line in lines:
            file.write(line + '\n')


def write_font_package_generation_script(path: Path, font_packages: Iterable[FontPackage]):
    lines = []

    for font_package in font_packages:
        for _, font_factory in font_package.font_factories.items():
            font_factory.write_characters_to_disk()
            command_string = font_factory.get_command_string()
            if len(command_string) > 256:
                lines.append('; WARNING: The following line exceeds 256 characters and will not be fully parsed!')
            lines.append(command_string)
        
        lines.append('')
        lines.append(f'OBJ SAVEPACKAGE PACKAGE={font_package.package_name} FILE="..\\{font_package.mod}\\Textures\\{font_package.package_name}.utx"')
        lines.append('')
    
    lines += [
            '',
            '; Execute this with the following command:',
            f'; EXEC "{path.resolve()}"'
    ]

    for line in lines:
        print(line)

    with open(path, 'w') as file:
        file.write('\n'.join(lines))


def generate(args):
    # Load the YAML file
    mod = args.mod

    root_path = Path(args.root_path).absolute()

    # Make sure that the root path exists and is a directory.
    if not root_path.exists() or not root_path.is_dir():
        print(f'Error: root path "{root_path}" does not exist or is not a directory', file=sys.stderr)
        return

    # TODO: mod should be optional!
    mod_path = root_path / mod

    # Make sure the mod path exists and is a directory.
    if not mod_path.exists() or not mod_path.is_dir():
        print(f'Error: mod path "{mod_path}" does not exist or is not a directory', file=sys.stderr)
        return

    font_directory = mod_path / 'Fonts'
    fonts_config_path = font_directory / 'fonts.yml'

    # Make sure the fonts.yml file exists.
    if not fonts_config_path.exists():
        # Print out to stderr so that it can be captured by the caller.
        print(f'Error: fonts.yml not found at {fonts_config_path}', file=sys.stderr)
        return

    # Read the font configuration file.
    config = FontConfiguration.from_file(fonts_config_path)

    fonts: Dict[str, TrueTypeFont] = OrderedDict()

    # Font style items should only be determined for the BASE language.
    # For all other languages, we need to make a mapping then output a localization file.
    font_style_items: Dict[str, List[FontStyleItem]] = {}
    font_packages: Dict[str, FontPackage] = dict()

    # Add the initial font package.
    font_package = FontPackage(mod, config.package_name)
    font_packages[config.package_name] = font_package

    for font_style_name, style in config.font_styles.items():
        # Merge the default font style with the language's font style.
        style.merge_with_default(config.defaults.font_style)

        if font_style_name not in font_style_items:
            font_style_items[font_style_name] = []

        resolutions, sizes = get_font_resolutions_and_sizes(config, style.size)

        # Add fonts for each size.
        for size, resolution in zip(sizes, resolutions):            
            font = TrueTypeFont(
                package=config.package_name,
                fontname=style.font,
                height=size,
                anti_alias=style.anti_alias if style.anti_alias is not None else False,
                drop_shadow_x=style.drop_shadow.x if style.drop_shadow is not None else 0,
                drop_shadow_y=style.drop_shadow.y if style.drop_shadow is not None else 0,
                u_size=style.texture_size.x if style.texture_size is not None else 512,
                v_size=style.texture_size.y if style.texture_size is not None else 512,
                x_pad=style.padding.x if style.padding is not None else 0,
                y_pad=style.padding.y if style.padding is not None else 0,
                extend_box_bottom=style.margin.bottom if style.margin is not None else 0,
                extend_box_top=style.margin.top if style.margin is not None else 0,
                extend_box_left=style.margin.left if style.margin is not None else 0,
                extend_box_right=style.margin.right if style.margin is not None else 0,
                kerning=style.kerning if style.kerning is not None else 0,
                style=style.weight if style.weight is not None else 500,
                italic=style.italic if style.italic is not None else False,
                compression=style.compression if style.compression is not None else Compression.RGBA8
            )
            font_package.font_factories[font.name] = TrueTypeFactory(font, config.defaults.unicode_ranges)

            if font.name not in fonts:
                # Add the font if the font has not yet been encountered.
                fonts[font.name] = font

            # Retrieve the font index from the list of fonts.
            font_index = list(fonts.keys()).index(font.name)

            # Add a font & resolution pair for the font style.
            font_style_items[font_style_name].append(
                FontStyleItem(font_index=font_index, resolution=resolution)
            )

    #======================================
    # LOCALIZATION
    #======================================

    # Iterate over the packages.
    for package_name, package in config.packages.items():
        # Determine the unicode ranges for the language.
        package_unicode_ranges = copy.deepcopy(config.defaults.unicode_ranges)

        if package.unicode_ranges is not None:
            package_unicode_ranges.merge(package.unicode_ranges)

        if package.ensure_all_used_characters:
            # Go through each of the Unreal translation files for all the 
            # languages that the package covers.
            used_characters = set()
            for language_code in package.languages:
                language_extension = get_language_extension(language_code)
                pattern = Path(mod_path) / 'System' / f'*.{language_extension}'
                for filename in glob.glob(str(pattern)):
                    used_characters |= read_unique_characters_from_unt_file(filename)
            # Add all of the used characters to the package's unicode ranges.
            package_unicode_ranges.add_ordinals(used_characters)
        
        language_font_names: List[Tuple[int, str]] = []

        font_package = FontPackage(mod, package_name)

        # Create language font substitution mappings.
        for font_index, (font_name, font) in enumerate(fonts.items()):
            if font.fontname not in package.font_substitutions:
                continue
        
            # Make a deep copy of the font and change the font name and package.
            font_substitute = copy.deepcopy(font)
            font_substitute.fontname = package.font_substitutions[font.fontname]
            font_substitute.package = package_name

            font_package.font_factories[font_substitute.name] = TrueTypeFactory(font_substitute, package_unicode_ranges)

            language_font_names.append((font_index, font_substitute.name))

        # Write a localization file that swaps out the fonts as described in the package's
        # font substitutions data for each language.
        for language_code in package.languages:
            language_extension = get_language_extension(language_code)
            localization_file_path = root_path / mod / 'System' / f'{config.unrealscript.fonts_package_name}.{language_extension}'

            with open(localization_file_path.resolve(), 'wb') as fp:
                contents = ''
                # Go through all of the fonts and make substitutions if necessary.
                contents += f'[{config.unrealscript.fonts_class_name}]\n'
                for font_index, font_name in language_font_names:
                    contents += f'FontNames[{font_index}]="{package_name}.{font_name}"\n'
                fp.write(b'\xff\xfe')  # Byte-order-mark.
                fp.write(contents.encode('utf-16-le'))
        
        font_packages[package_name] = font_package

    #======================================
    # UNREALSCRIPT
    #======================================

    unrealscript_gui_font_path = Path(root_path) / config.unrealscript.gui_fonts_directory

    # Write the font style classes.
    for font_style_name in font_style_items.keys():
        gui_font_path = unrealscript_gui_font_path / f'{font_style_name}.uc'
        write_font_style_class_file(gui_font_path, font_style_name, config.unrealscript.fonts_class_name)

    # Write the fonts class.
    unrealscript_fonts_path = Path(root_path) / config.unrealscript.fonts_package_name / 'Classes' / f'{config.unrealscript.fonts_class_name}.uc'

    write_fonts_class_file(
        config.package_name,
        fonts,
        font_style_items, 
        config.unrealscript.fonts_class_name, 
        unrealscript_fonts_path
        )
    
    if not args.slim:
        #======================================
        # FONT PACKAGE GENERATION
        #======================================

        # Keep a cache of evaluated unicode ranges for each font, since this is a very intensive operation.
        font_unicode_ranges_cache: Dict[str, UnicodeRanges] = dict()
        installed_fonts = get_installed_fonts()

        # Refine the unicode ranges based on what the font actually supports.
        for _, font_package in font_packages.items():
            for _, font_factory in font_package.font_factories.items():
                font = font_factory.font
                styles = get_font_styles_from_weight(font.style)

                # Ensure that the font is installed.
                if font.fontname not in installed_fonts:
                    raise Exception(f'Font family "{font.fontname}" is not installed')

                # Get the path to the font that is closest to matching the requested styles.
                font_path = get_font_style_closest(installed_fonts, styles, font.fontname)
                if font_path is None:
                    raise Exception(f'Could not find {styles} styles for installed font "{font.fontname}"')

                # Load the font and get the supported unicode ranges.
                if font_path not in font_unicode_ranges_cache:
                    font_unicode_ranges_cache[font_path] = read_font_unicode_ranges(font_path)
                
                font_unicode_ranges = font_unicode_ranges_cache[font_path]

                # Intersect the font's unicode ranges with the system's supported unicode ranges.
                font_factory.unicode_ranges = font_unicode_ranges.intersect(font_factory.unicode_ranges)
        
        #======================================
        # PACKAGE GENERATION SCRIPT
        #======================================
        package_generation_script_path = font_directory / 'ImportFonts.exec.txt'
        write_font_package_generation_script(
            package_generation_script_path,
            font_packages.values()
            )


if __name__ == '__main__':
    from argparse import ArgumentParser

    # Create the top-level parser
    argparse = ArgumentParser(prog='fonts', description='Unreal Tournament 2 font generation tool')

    subparsers = argparse.add_subparsers(dest='command', required=True)

    generate_font_scripts_parser = subparsers.add_parser('generate', help='Generate font scripts from a YAML file.')
    generate_font_scripts_parser.add_argument('root_path', help='The path of the game root directory.')
    generate_font_scripts_parser.add_argument('-m', '--mod', help='The name of the mod to generate font scripts for.', required=True)
    generate_font_scripts_parser.add_argument('-l', '--language_code', help='The language to generate font scripts for (ISO 639-1 codes)', required=False)
    generate_font_scripts_parser.add_argument('-s', '--slim', help='Only generate the UnrealScript and localization files')
    generate_font_scripts_parser.set_defaults(func=generate)

    args = argparse.parse_args()
    args.func(args)