diff --git a/README.md b/README.md index b1d1f7e5..01e57f8e 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,17 @@ This mod gives SWAG the ability to start containers on-demand when accessed thro - `SWAG_ONDEMAND_STOP_THRESHOLD` - duration of inactivity in seconds before stopping on-demand containers, defaults to `600` (10 minutes). - `SWAG_ONDEMAND_CONTAINER_QUERY_SLEEP` - sleep time in seconds between querying containers, defaults to `5.0`. - `SWAG_ONDEMAND_LOG_READER_SLEEP` - sleep time in seconds between log reads, defaults to `1.0`. - - `SWAG_ONDEMAND_REMOTE1...20` - the remote API of other hosts for ondemand to manage, such as: tcp://otherhost:2375. can add up to 20. + - `SWAG_ONDEMAND_DOCKER_API_TIMEOUT` - the timeout for docker's API. Defaults to `5`. + - `SWAG_ONDEMAND_REMOTE1` - the remote API of other hosts for ondemand to manage. For example: `tcp://otherhost:2375`. + - `SWAG_ONDEMAND_REMOTE1_WOL_MAC` - Required for WoL, specifies which MAC address to send the WoL packet to. For example: `00:00:0A:BB:28:FC`. + - `SWAG_ONDEMAND_REMOTE1_WOL_URLS` - Required for WoL, specifies which URL prefixes would trigger WoL. Same syntax as `swag_ondemand_urls` below. For example: `https://somecontainer.`. + - `SWAG_ONDEMAND_REMOTE1_WOL_BROADCAST` - Optional, override which broadcast to send the WoL packet to. Defaults to `255.255.255.255`. + - `SWAG_ONDEMAND_REMOTE1_WOL_PORT` - Optional, override which port to send the WoL packet to. Defaults to `9`. + - `SWAG_ONDEMAND_REMOTE1_WOL_INTERFACE` - Optional, override which interface to use for sending the WoL packet. Defaults to the first interface. + +**You can increment the number for up to 20 remote hosts. For example: `SWAG_ONDEMAND_REMOTE2`, `SWAG_ONDEMAND_REMOTE3`, etc.** + +**For WoL to work in a container, you need to either set `network_mode: host` or broadcast to the IP of the remote host and set a static ARP on the router. For example: in opnsense add an entry under Interfaces > Neighbors > Static Assignments.** ### Loading Page: diff --git a/root/app/ondemand/container_thread.py b/root/app/ondemand/container_thread.py index 7bdd10bf..a9823ac5 100644 --- a/root/app/ondemand/container_thread.py +++ b/root/app/ondemand/container_thread.py @@ -1,5 +1,4 @@ from data_classes import DockerHost, OnDemandContainer -import helper from shared_state import last_accessed_urls, last_accessed_urls_lock from datetime import datetime @@ -7,6 +6,7 @@ import os import threading import time +import wakeonlan CONTAINER_QUERY_SLEEP = float(os.environ.get("SWAG_ONDEMAND_CONTAINER_QUERY_SLEEP", "5.0")) STOP_THRESHOLD = int(os.environ.get("SWAG_ONDEMAND_STOP_THRESHOLD", "600")) @@ -22,43 +22,41 @@ def __init__(self): def init_docker_hosts(self): docker_host_url = os.environ.get("DOCKER_HOST", None) - client, url = helper.get_docker_client(docker_host_url, True) - if client: - self.docker_hosts.append(DockerHost(client=client, url=url)) + if docker_host_url and not docker_host_url.startswith("tcp://"): + docker_host_url = f"tcp://{docker_host_url}:2375" + self.docker_hosts.append(DockerHost(url=docker_host_url)) - remote_hosts_env_vars = { key: value for key, value in os.environ.items() if key.startswith(REMOTE_HOSTS_PREFIX) } + remote_hosts_env_vars = {key: value for key, value in os.environ.items() if key.startswith(REMOTE_HOSTS_PREFIX)} for i in range(1, 21): if f"{REMOTE_HOSTS_PREFIX}{i}" not in remote_hosts_env_vars: break docker_host_url = remote_hosts_env_vars[f"{REMOTE_HOSTS_PREFIX}{i}"] - client, url = helper.get_docker_client(docker_host_url) - - if client: - self.docker_hosts.append(DockerHost(client=client, url=url)) + if docker_host_url and not docker_host_url.startswith("tcp://"): + docker_host_url = f"tcp://{docker_host_url}:2375" + remote_host = DockerHost(url=docker_host_url) + remote_host.wol_mac = remote_hosts_env_vars.get(f"{REMOTE_HOSTS_PREFIX}{i}_WOL_MAC", None) + remote_host.wol_broadcast = remote_hosts_env_vars.get(f"{REMOTE_HOSTS_PREFIX}{i}_WOL_BROADCAST", "255.255.255.255") + remote_host.wol_urls = remote_hosts_env_vars.get(f"{REMOTE_HOSTS_PREFIX}{i}_WOL_URLS", None) + remote_host.wol_port = int(remote_hosts_env_vars.get(f"{REMOTE_HOSTS_PREFIX}{i}_WOL_PORT", "9")) + remote_host.wol_interface = remote_hosts_env_vars.get(f"{REMOTE_HOSTS_PREFIX}{i}_WOL_INTERFACE", None) + self.docker_hosts.append(remote_host) - if not self.docker_hosts: - logging.error("Failed to connect to any docker host") - def process_containers(self): for docker_host in self.docker_hosts: - if not helper.is_docker_connected(docker_host.client): - if docker_host.is_connected: - logging.warning(f"Lost connection to {docker_host.url}") - docker_host.is_connected = False + if not docker_host.is_connected: continue - if not docker_host.is_connected: - logging.info(f"Connection to {docker_host.url} has been restored") - docker_host.is_connected = True + containers = docker_host.get_containers() + if not containers: + continue - containers = docker_host.client.containers.list(all=True, filters={ "label": ["swag_ondemand=enable"] }) container_names = {container.name for container in containers} for container_name in list(docker_host.ondemand_containers.keys()): if container_name not in container_names: docker_host.ondemand_containers.pop(container_name) - logging.info(f"Stopped monitoring {container_name}") + logging.info(f"Stopped monitoring {container_name} on {docker_host.url}") for container in containers: default_url = container.labels.get("swag_url", f"{container.name}.").rstrip("*") @@ -66,12 +64,12 @@ def process_containers(self): if container.name not in docker_host.ondemand_containers: last_accessed = datetime.now() - logging.info(f"Started monitoring {container.name} for urls: {container_urls}") + logging.info(f"Started monitoring {container.name} on {docker_host.url} for urls: {container_urls}") else: existing_container = docker_host.ondemand_containers[container.name] last_accessed = existing_container.last_accessed if container_urls != existing_container.urls: - logging.info(f"Updated urls for {container.name} to: {container_urls}") + logging.info(f"Updated urls for {container.name} on {docker_host.url} to: {container_urls}") docker_host.ondemand_containers[container.name] = OnDemandContainer( status=container.status, @@ -81,52 +79,79 @@ def process_containers(self): def stop_containers(self): for docker_host in self.docker_hosts: - for container_name, container in docker_host.ondemand_containers.items(): - if container.status != "running": + if not docker_host.is_connected: + continue + for container_name, ondemand_container in docker_host.ondemand_containers.items(): + if ondemand_container.status != "running": continue - inactive_seconds = (datetime.now() - container.last_accessed).total_seconds() + inactive_seconds = (datetime.now() - ondemand_container.last_accessed).total_seconds() if inactive_seconds < STOP_THRESHOLD: continue - if not helper.is_docker_connected(docker_host.client): - logging.warning(f"Failed to stop {container_name}, docker host {docker_host.url} is unavailable") + container = docker_host.get_container(container_name) + if not container: continue - docker_host.client.containers.get(container_name).stop() - logging.info(f"Stopped {container_name} after {STOP_THRESHOLD}s of inactivity") - - def start_containers(self): - with last_accessed_urls_lock: - last_accessed_urls_combined = ",".join(last_accessed_urls) - last_accessed_urls.clear() + container.stop() + ondemand_container.status = "exited" + logging.info(f"Stopped {container_name} on {docker_host.url} after {STOP_THRESHOLD}s of inactivity") + def start_containers(self, last_accessed_urls_combined: str): + if not last_accessed_urls_combined: + return + for docker_host in self.docker_hosts: - for container_name, container in docker_host.ondemand_containers.items(): + if not docker_host.is_connected: + continue + for container_name, ondemand_container in docker_host.ondemand_containers.items(): accessed = False - for ondemand_url in container.urls.split(","): + for ondemand_url in ondemand_container.urls.split(","): if ondemand_url in last_accessed_urls_combined: - container.last_accessed = datetime.now() + ondemand_container.last_accessed = datetime.now() accessed = True break - if not accessed or container.status == "running": + if not accessed or ondemand_container.status == "running": continue - if not helper.is_docker_connected(docker_host.client): - logging.warning(f"Failed to start {container_name}, docker host {docker_host.url} is unavailable") + container = docker_host.get_container(container_name) + if not container: continue - - docker_host.client.containers.get(container_name).start() - logging.info(f"Started {container_name}") - container.status = "running" + + container.start() + ondemand_container.status = "running" + logging.info(f"Started {container_name} on {docker_host.url}") + + def send_wol(self, last_accessed_urls_combined: str): + if not last_accessed_urls_combined: + return + + for docker_host in self.docker_hosts: + if not docker_host.wol_mac or not docker_host.wol_urls or docker_host.is_connected: + continue + for wol_url in docker_host.wol_urls.split(","): + if wol_url in last_accessed_urls_combined: + wakeonlan.send_magic_packet( + docker_host.wol_mac, + ip_address=docker_host.wol_broadcast, + port=docker_host.wol_port, + interface=docker_host.wol_interface + ) + logging.info(f"Sent a WoL packet to mac {docker_host.wol_mac} via broadcast {docker_host.wol_broadcast} on port {docker_host.wol_port} on interface {docker_host.wol_interface or 'default'} activated by {wol_url}") + break def run(self): while True: try: + with last_accessed_urls_lock: + last_accessed_urls_combined = ",".join(last_accessed_urls) + last_accessed_urls.clear() + + self.send_wol(last_accessed_urls_combined) self.process_containers() - self.start_containers() + self.start_containers(last_accessed_urls_combined) self.stop_containers() - time.sleep(CONTAINER_QUERY_SLEEP) except Exception as e: logging.exception(e) + time.sleep(CONTAINER_QUERY_SLEEP) diff --git a/root/app/ondemand/data_classes.py b/root/app/ondemand/data_classes.py index ccc4384b..4bc1cf02 100644 --- a/root/app/ondemand/data_classes.py +++ b/root/app/ondemand/data_classes.py @@ -1,6 +1,9 @@ from dataclasses import dataclass, field from datetime import datetime import docker +import logging +import requests +from typing import Optional @dataclass class OnDemandContainer: @@ -10,7 +13,60 @@ class OnDemandContainer: @dataclass class DockerHost: - client: docker.DockerClient url: str + client: Optional[docker.DockerClient] = None + wol_mac: Optional[str] = None + wol_broadcast: str = "255.255.255.255" + wol_port: int = 9 + wol_interface: Optional[str] = None + wol_urls: Optional[str] = None is_connected: bool = False + was_connected: bool = False ondemand_containers: dict[str, OnDemandContainer] = field(default_factory=dict) + + def check_connection(self, timeout: int): + try: + self.was_connected = self.is_connected + if self.client and self.client.ping(): + self.is_connected = True + return + + if self.url: + self.client = docker.DockerClient(base_url=self.url, timeout=timeout) + else: + self.client = docker.from_env(timeout=timeout) + self.url = "unix:///var/run/docker.sock" + + self.is_connected = True + if not self.was_connected: + logging.info(f"Connection to {self.url} has been restored") + except (docker.errors.DockerException, requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): + self.client = None + self.is_connected = False + if self.was_connected: + logging.warning(f"Lost connection to {self.url} during health check") + + def handle_disconnect(self): + self.client = None + self.is_connected = False + logging.warning(f"Lost connection to {self.url} during runtime operation") + + def get_container(self, container_name: str): + try: + client = self.client + if not client or not self.is_connected: + return None + return client.containers.get(container_name) + except (docker.errors.DockerException, requests.exceptions.ConnectionError): + self.handle_disconnect() + return None + + def get_containers(self): + try: + client = self.client + if not client or not self.is_connected: + return None + return client.containers.list(all=True, filters={"label": ["swag_ondemand=enable"]}) + except (docker.errors.DockerException, requests.exceptions.ConnectionError): + self.handle_disconnect() + return None diff --git a/root/app/ondemand/healthcheck_thread.py b/root/app/ondemand/healthcheck_thread.py new file mode 100644 index 00000000..779e66d8 --- /dev/null +++ b/root/app/ondemand/healthcheck_thread.py @@ -0,0 +1,35 @@ +from data_classes import DockerHost + +from concurrent.futures import ThreadPoolExecutor +import logging +import os +import threading +import time + +DOCKER_API_TIMEOUT = int(os.environ.get("SWAG_ONDEMAND_DOCKER_API_TIMEOUT", "5")) + + +class HealthcheckThread(threading.Thread): + def __init__(self, docker_hosts: list[DockerHost]): + super().__init__(name="HealthcheckThread") + self.daemon = True + self.docker_hosts = docker_hosts + + def run(self): + max_workers = max(1, len(self.docker_hosts)) + logging.info(f"Starting HealthcheckThread with a pool of {max_workers} workers.") + + with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="HealthcheckWorker") as executor: + while True: + futures = [ + executor.submit(docker_host.check_connection, DOCKER_API_TIMEOUT) + for docker_host in self.docker_hosts + ] + + for future in futures: + try: + future.result() + except Exception as e: + logging.exception(e) + + time.sleep(1) diff --git a/root/app/ondemand/helper.py b/root/app/ondemand/helper.py deleted file mode 100644 index 4af9d35c..00000000 --- a/root/app/ondemand/helper.py +++ /dev/null @@ -1,24 +0,0 @@ -import docker -import requests -from typing import Optional - - -def get_docker_client(docker_host_url: str, from_env: bool = False) -> tuple[Optional[docker.DockerClient], str]: - try: - if docker_host_url: - if not docker_host_url.startswith("tcp://"): - docker_host_url = f"tcp://{docker_host_url}:2375" - return docker.DockerClient(base_url=docker_host_url), docker_host_url - elif from_env: - return docker.from_env(), "unix://var/run/docker.sock" - else: - return None, "" - except (docker.errors.DockerException, requests.exceptions.ConnectionError): - return None, "" - -def is_docker_connected(client: docker.DockerClient) -> bool: - try: - return client.ping() - except (docker.errors.DockerException, requests.exceptions.ConnectionError): - return False - \ No newline at end of file diff --git a/root/app/ondemand/main.py b/root/app/ondemand/main.py index a0278f53..11d5ee18 100644 --- a/root/app/ondemand/main.py +++ b/root/app/ondemand/main.py @@ -1,4 +1,5 @@ from container_thread import ContainerThread +from healthcheck_thread import HealthcheckThread from log_reader_thread import LogReaderThread import logging @@ -16,9 +17,14 @@ datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO) logging.info("Starting swag-ondemand...") + + container_thread = ContainerThread() + healthcheck_thread = HealthcheckThread(container_thread.docker_hosts) + log_reader_thread = LogReaderThread() - ContainerThread().start() - LogReaderThread().start() + healthcheck_thread.start() + container_thread.start() + log_reader_thread.start() while True: time.sleep(1) diff --git a/root/app/ondemand/requirements.txt b/root/app/ondemand/requirements.txt new file mode 100644 index 00000000..5d0b057b --- /dev/null +++ b/root/app/ondemand/requirements.txt @@ -0,0 +1,2 @@ +docker +wakeonlan \ No newline at end of file diff --git a/root/etc/s6-overlay/s6-rc.d/init-mod-swag-ondemand-setup/run b/root/etc/s6-overlay/s6-rc.d/init-mod-swag-ondemand-setup/run index 57536391..bbb073b8 100755 --- a/root/etc/s6-overlay/s6-rc.d/init-mod-swag-ondemand-setup/run +++ b/root/etc/s6-overlay/s6-rc.d/init-mod-swag-ondemand-setup/run @@ -7,18 +7,20 @@ if [ ! -S /var/run/docker.sock ] && [ -z "$DOCKER_HOST" ]; then exit 0 fi -echo '**** Checking if docker-py is already installed ****' -if ! pip list 2>&1 | grep -q "docker"; then - echo "**** Adding docker-py to package install lists ****" - echo "\ - docker" >> /mod-pip-packages-to-install.list -else - echo "**** docker-py already installed, skipping ****" -fi +for pkg in $(cat /app/ondemand/requirements.txt); do + echo "**** Checking if $pkg are already installed ****" + if ! pip list 2>&1 | grep -iq "$pkg"; then + echo "**** Adding $pkg to package install lists ****" + echo "\ + $pkg" >> /mod-pip-packages-to-install.list + else + echo "**** $pkg already installed, skipping ****" + fi +done if [ ! -f /config/nginx/ondemand.conf ]; then cp /defaults/ondemand.conf /config/nginx/ondemand.conf lsiown -R abc:abc /config/nginx/ondemand.conf fi -echo "Applied the swag-ondemand mod" \ No newline at end of file +echo "Applied the swag-ondemand mod"