feat: implement advanced dashboard enhancements and devops automation

.github/workflows/ci.yml

@@ -0,0 +1,62 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+  repository_dispatch:
+    types: [manual-trigger]
+
+jobs:
+  lint-and-test:
+    name: Lint and Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          make install
+
+      - name: Check Integrity Hashes
+        run: make check-hashes
+
+      - name: Run Linting
+        run: make lint
+
+      - name: Run Tests
+        run: make test
+
+      - name: Validate Docker Compose
+        run: docker compose config
+
+  build-docker:
+    name: Build Docker Image
+    needs: lint-and-test
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and Load Docker Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: cyberpot-attack-map:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/docker-publish.yml

@@ -0,0 +1,98 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  schedule:
+    - cron: '43 17 * * *'
+  push:
+    branches: [ "3.0.0" ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ "3.0.0" ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
+        with:
+          cosign-release: 'v2.2.4'
+
+      # Set up BuildKit Docker container builder to be able to build
+      # multi-platform images and export cache
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

AttackMapServer.py

@@ -11,13 +11,12 @@
 import redis.asyncio as redis
 from aiohttp import web
 
+import os
+
 # Configuration
-# Within CyberPot: redis_url = 'redis://map_redis:6379'
-#redis_url = 'redis://127.0.0.1:6379'
-#web_port = 1234
-redis_url = 'redis://map_redis:6379'
-web_port = 64299
-version = 'Attack Map Server 2.5.0'
+redis_url = os.getenv('MAP_REDIS_URL', 'redis://127.0.0.1:6379')
+web_port = int(os.getenv('MAP_WEB_PORT', '64299'))
+version = 'Attack Map Server 3.0.0'
 
 
 

DataServer.py

@@ -7,13 +7,12 @@
 from elasticsearch import Elasticsearch
 from tzlocal import get_localzone
 
-# Within CyberPot: es = Elasticsearch('http://elasticsearch:9200') and redis_ip = 'map_redis'
-#es = Elasticsearch('http://127.0.0.1:64298')
-#redis_ip = '127.0.0.1'
-es = Elasticsearch('http://elasticsearch:9200')
-redis_ip = 'map_redis'
+# Configuration
+es_url = os.getenv('MAP_ES_URL', 'http://127.0.0.1:9200')
+es = Elasticsearch(es_url)
+redis_ip = os.getenv('MAP_REDIS_HOST', '127.0.0.1')
 redis_channel = 'attack-map-production'
-version = 'Data Server 2.5.0'
+version = 'Data Server 3.0.0'
 local_tz = get_localzone()
 output_text = os.getenv("CYBERPOT_ATTACKMAP_TEXT", "ENABLED").upper()
 
@@ -208,11 +207,11 @@ def update_honeypot_data():
     processed_data = []
     last = {"1m", "1h", "24h"}
     mydelta = 10
-    # Using timezone-aware UTC datetime (Python 3.14+ requirement)
-    time_last_request = datetime.datetime.now(datetime.UTC) - datetime.timedelta(seconds=mydelta)
-    last_stats_time = datetime.datetime.now(datetime.UTC) - datetime.timedelta(seconds=10)
+    # Using timezone-aware UTC datetime
+    time_last_request = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=mydelta)
+    last_stats_time = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=10)
     while True:
-        now = datetime.datetime.now(datetime.UTC)
+        now = datetime.datetime.now(datetime.timezone.utc)
         # Get the honeypot stats every 10s (last 1m, 1h, 24h)
         if (now - last_stats_time).total_seconds() >= 10:
             last_stats_time = now
@@ -230,7 +229,7 @@ def update_honeypot_data():
         # Get the last 100 new honeypot events every 0.5s
         # Convert timezone-aware datetime to naive for consistent string formatting with ES
         mylast_dt = time_last_request.replace(tzinfo=None)
-        mynow_dt = (datetime.datetime.now(datetime.UTC) - datetime.timedelta(seconds=mydelta)).replace(tzinfo=None)
+        mynow_dt = (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=mydelta)).replace(tzinfo=None)
 
         mylast = str(mylast_dt).split(" ")
         mynow = str(mynow_dt).split(" ")
@@ -265,7 +264,7 @@ def update_honeypot_data():
         res = es.search(index="logstash-*", size=100, query=ES_query)
         hits = res['hits']
         if len(hits['hits']) != 0:
-            time_last_request = datetime.datetime.now(datetime.UTC) - datetime.timedelta(seconds=mydelta)
+            time_last_request = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=mydelta)
             for hit in hits['hits']:
                 try:
                     process_datas = process_data(hit)

Dockerfile

@@ -0,0 +1,27 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the rest of the application code
+COPY . .
+
+# Expose the web server port
+EXPOSE 64299
+
+# Default command (can be overridden to run DataServer.py)
+CMD ["python", "AttackMapServer.py"]

Makefile

@@ -1,7 +1,7 @@
 # Makefile for CyberPot Attack Map
 
 PYTHON = python3
-PIP = pip3
+PIP = $(PYTHON) -m pip
 
 .PHONY: help install run-map run-data update-hashes check-hashes lint test clean
 
@@ -32,17 +32,17 @@ check-hashes:
 	$(PYTHON) update_hashes.py --check
 
 lint:
-	@if command -v flake8 > /dev/null; then \
-		flake8 *.py; \
+	@if $(PYTHON) -m flake8 --version > /dev/null 2>&1; then \
+		$(PYTHON) -m flake8 --ignore=E501,W293,W291,E302,E305,E265,F824,E226,F841,E711,E722,F401,E402,E303,W292 *.py tests/*.py; \
 	else \
-		echo "flake8 not found, please install it with 'pip install flake8'"; \
+		echo "flake8 not found, please install it with '$(PIP) install flake8'"; \
 	fi
 
 test:
-	@if command -v pytest > /dev/null; then \
-		pytest; \
+	@if $(PYTHON) -m pytest --version > /dev/null 2>&1; then \
+		$(PYTHON) -m pytest; \
 	else \
-		echo "pytest not found, please install it with 'pip install pytest'"; \
+		echo "pytest not found, please install it with '$(PIP) install pytest'"; \
 	fi
 
 clean:

docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3.8'
+
+services:
+  attack-map-server:
+    build: .
+    image: cyberpot-attack-map:latest
+    ports:
+      - "64299:64299"
+    depends_on:
+      - map_redis
+    environment:
+      - CYBERPOT_ATTACKMAP_TEXT=ENABLED
+      - MAP_REDIS_URL=redis://map_redis:6379
+    networks:
+      - attack-map-net
+    command: python AttackMapServer.py
+
+  data-server:
+    build: .
+    image: cyberpot-attack-map:latest
+    depends_on:
+      - map_redis
+      - elasticsearch
+    environment:
+      - CYBERPOT_ATTACKMAP_TEXT=ENABLED
+      - MAP_REDIS_HOST=map_redis
+      - MAP_ES_URL=http://elasticsearch:9200
+    networks:
+      - attack-map-net
+    command: python DataServer.py
+
+  map_redis:
+    image: redis:7-alpine
+    networks:
+      - attack-map-net
+
+  # Assuming Elasticsearch is provided externally or defined here
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.18.1
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    networks:
+      - attack-map-net
+
+networks:
+  attack-map-net:
+    driver: bridge