Prepare 3.1.1 security hotfix

.github/workflows/python-package.yml

@@ -68,9 +68,20 @@ jobs:
           cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
           touch ~/.msticpy/GeoLite2-City.mmdb
           cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor
-          touch ~/.msticpy/mordor/mitre_tact_cache.pkl
-          touch ~/.msticpy/mordor/mitre_tech_cache.pkl
-          touch ~/.msticpy/mordor/mordor_cache.pkl
+          python - <<'PY'
+          import json
+          from datetime import datetime, timezone
+          from pathlib import Path
+
+          cache_folder = Path.home().joinpath(".msticpy", "mordor")
+          for filename in ("mitre_tact_cache.json", "mitre_tech_cache.json"):
+              cache_folder.joinpath(filename).touch()
+          metadata_path = cache_folder.joinpath("mordor_cache.json")
+          metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+          for item in metadata.values():
+              item["mp_last_updated"] = datetime.now(timezone.utc).isoformat()
+          metadata_path.write_text(json.dumps(metadata, indent=4), encoding="utf-8")
+          PY
       - name: Pytest
         env:
           MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }}

.github/workflows/python-publish.yml

@@ -1,4 +1,7 @@
-# This workflow will upload a Python Package using Twine when a release is created
+# Publishing is intentionally disabled. Remove the false job guards only after
+# configuring the protected "pypi" environment and a PyPI trusted publisher for
+# microsoft/msticpy, python-publish.yml, and the "pypi" environment.
+# The environment must require approval and restrict deployments to release tags.
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
 
 # This workflow uses actions that are not certified by GitHub.
@@ -11,18 +14,22 @@ name: Upload Python Package to PyPI Prod
 on:
   release:
     types: [published]
-  workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
-  deploy:
-
+  build:
+    if: >-
+      ${{ false && github.event_name == 'release' &&
+          !github.event.release.prerelease && startsWith(github.ref, 'refs/tags/v') }}
     runs-on: ubuntu-latest
-    permissions: read-all
+    permissions:
+      contents: read
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
     - name: Set up Python
       uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
       with:
@@ -38,8 +45,31 @@ jobs:
         --sdist
         --wheel
         --outdir dist/
+    - name: Upload distributions
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: python-distributions
+        path: dist/
+        if-no-files-found: error
+        retention-days: 1
+
+  publish:
+    needs: build
+    if: >-
+      ${{ false && needs.build.result == 'success' &&
+          github.event_name == 'release' && !github.event.release.prerelease &&
+          startsWith(github.ref, 'refs/tags/v') }}
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/msticpy
+    permissions:
+      id-token: write
+    steps:
+    - name: Download distributions
+      uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+      with:
+        name: python-distributions
+        path: dist/
     - name: Publish package
       uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
   - repo: https://github.com/PyCQA/pylint
-    rev: v4.0.2
+    rev: v4.0.9
     hooks:
       - id: pylint
         args:

azure-pipelines.yml

@@ -76,9 +76,6 @@ stages:
         cp $(Build.SourcesDirectory)/tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
         touch ~/.msticpy/GeoLite2-City.mmdb
         cp -r $(Build.SourcesDirectory)/tests/testdata/mordor/* ~/.msticpy/mordor
-        touch ~/.msticpy/mordor/mitre_tact_cache.pkl
-        touch ~/.msticpy/mordor/mitre_tech_cache.pkl
-        touch ~/.msticpy/mordor/mordor_cache.pkl
       condition: ne(variables.imageName, 'windows-latest')
       displayName: 'Prepare test dummy data - Linux'
     - script: |
@@ -88,6 +85,23 @@ stages:
         copy /Y $(Build.SourcesDirectory)\tests\testdata\mordor\* %USERPROFILE%\.msticpy\mordor
       condition: eq(variables.imageName, 'windows-latest')
       displayName: 'Prepare test dummy data - Windows'
+    - task: PythonScript@0
+      displayName: 'Refresh Mordor JSON test caches'
+      inputs:
+        scriptSource: inline
+        script: |
+          import json
+          from datetime import datetime, timezone
+          from pathlib import Path
+
+          cache_folder = Path.home().joinpath(".msticpy", "mordor")
+          for filename in ("mitre_tact_cache.json", "mitre_tech_cache.json"):
+              cache_folder.joinpath(filename).touch()
+          metadata_path = cache_folder.joinpath("mordor_cache.json")
+          metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+          for item in metadata.values():
+              item["mp_last_updated"] = datetime.now(timezone.utc).isoformat()
+          metadata_path.write_text(json.dumps(metadata, indent=4), encoding="utf-8")
     - script: |
         echo MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/msticpyconfig-test.yaml
         echo Env $MSTICPYCONFIG or %MSTICPYCONFIG%

docs/source/data_acquisition/MordorData.rst

@@ -174,6 +174,13 @@ will apply to all queries.
    metadata will be cached so you will not have to download this again
    in this session.
 
+.. note:: MITRE techniques and tactics are cached as
+   ``mitre_tech_cache.json`` and ``mitre_tact_cache.json`` in the save folder.
+   Legacy ``.pkl`` caches are never loaded or migrated; the first connection
+   after upgrading downloads fresh MITRE data. Old pickle cache files can
+   be deleted. JSON cache files expire after 30 days, and invalid MITRE
+   caches are logged and refreshed.
+
 Using these parameters in the query will override the provider settings
 and defaults for that query.
 
@@ -423,4 +430,3 @@ for more details.
 .. figure:: _static/MordorBrowser-std-query-browser.png
    :alt: Using the standard query browser to view Mordor queries.
    :width: 5in
-

docs/source/data_acquisition/TIProviders.rst

@@ -916,6 +916,10 @@ Address or URL) for all providers.
 For each provider that returns a result for an IoC, the summarized details
 will be shown in a table below the browse list.
 
+Provider-supplied values, including raw results, are displayed as text rather
+than interpreted as HTML. Only absolute HTTP and HTTPS reference URLs are
+clickable; other references are displayed as plain text.
+
 .. figure:: _static/ti_browser_results1.png
    :alt: Threat Intel results browser
    :width: 5.23000in

msticpy/_version.py

@@ -1,3 +1,3 @@
 """Version file."""
 
-VERSION = "3.1.0"
+VERSION = "3.1.1"

msticpy/data/drivers/mordor_driver.py

@@ -8,7 +8,7 @@
 from __future__ import annotations
 
 import json
-import pickle  # nosec
+import logging
 import zipfile
 from collections import defaultdict
 from collections.abc import Generator, Iterable
@@ -34,6 +34,8 @@
 __version__ = VERSION
 __author__ = "Ian Hellen"
 
+logger = logging.getLogger(__name__)
+
 
 _MORDOR_TREE_URI = (
     "https://api.github.com/repos/OTRF/Security-Datasets/git/trees/master?recursive=1"
@@ -50,8 +52,8 @@
 MITRE_TECHNIQUES: pd.DataFrame | None = None
 MITRE_TACTICS: pd.DataFrame | None = None
 
-_MITRE_TECH_CACHE = "mitre_tech_cache.pkl"
-_MITRE_TACTICS_CACHE = "mitre_tact_cache.pkl"
+_MITRE_TECH_CACHE = "mitre_tech_cache.json"
+_MITRE_TACTICS_CACHE = "mitre_tact_cache.json"
 _MORDOR_CACHE = "mordor_cache.json"
 
 _HTTP_TIMEOUT = DriverBase.get_http_timeout()
@@ -645,6 +647,8 @@ def _fetch_mdr_metadata(cache_folder: str | None = None) -> dict[str, MordorEntr
 def _read_mordor_cache(cache_folder) -> dict[str, Any]:
     """Return dictionary of cached metadata if cached_folder is a valid path."""
     md_cached_metadata: dict[str, Any] = {}
+    if cache_folder is None:
+        return md_cached_metadata
     mordor_cache = Path(cache_folder).joinpath(_MORDOR_CACHE)
     if _valid_cache(mordor_cache):
         try:
@@ -657,6 +661,8 @@ def _read_mordor_cache(cache_folder) -> dict[str, Any]:
 
 def _write_mordor_cache(md_cached_metadata, cache_folder):
     """Write dictionary of cached metadata if cached_folder is a valid path."""
+    if cache_folder is None:
+        return
     mordor_cache = Path(cache_folder).joinpath(_MORDOR_CACHE)
     json_text = json.dumps(md_cached_metadata, indent=4)
     Path(mordor_cache).write_text(json_text, encoding="utf-8")
@@ -900,12 +906,18 @@ def _get_mitre_categories(
         tactics_cache = Path(cache_folder).joinpath(_MITRE_TACTICS_CACHE)
         if _valid_cache(tech_cache) and _valid_cache(tactics_cache):
             try:
-                # caching location only used if sepcified.
-                tech_df = pd.read_pickle(tech_cache)  # nosec
-                tactics_df = pd.read_pickle(tactics_cache)  # nosec
+                tech_df = _read_mitre_cache(tech_cache)
+                tactics_df = _read_mitre_cache(tactics_cache)
+                for cached_df in (tech_df, tactics_df):
+                    if cached_df.index.name != "ID" or list(cached_df.columns) != [
+                        "Name",
+                        "Description",
+                        "url",
+                    ]:
+                        raise ValueError("Invalid MITRE cache schema")
                 return tech_df, tactics_df
-            except pickle.PickleError:
-                pass
+            except (ValueError, KeyError, TypeError) as cache_err:
+                logger.warning("Invalid MITRE cache in %s: %s", cache_folder, cache_err)
     resp = httpx.get(_MITRE_JSON_URL, timeout=_HTTP_TIMEOUT, headers=mp_ua_header())
     mitre = pd.json_normalize(resp.json()["objects"])
 
@@ -916,12 +928,18 @@ def _get_mitre_categories(
     tech_df = _reshape_mitre_df(mitre[mitre.type == "attack-pattern"])
     tactics_df = _reshape_mitre_df(mitre[mitre.type == "x-mitre-tactic"])
     if cache_folder:
-        tech_df.to_pickle(tech_cache)
-        tactics_df.to_pickle(tactics_cache)
+        tech_df.to_json(tech_cache, orient="table", indent=2)
+        tactics_df.to_json(tactics_cache, orient="table", indent=2)
 
     return tech_df, tactics_df
 
 
+def _read_mitre_cache(cache_path: Path) -> pd.DataFrame:
+    """Read JSON while preserving text-column dtypes and null values."""
+    cached_df = pd.read_json(cache_path, orient="table").astype(object)
+    return cached_df.where(cached_df.notna(), None)
+
+
 def _valid_cache(path: Path, expired_days=30):
     """Return True if the file exists and is younger than `expired_days`."""
     if not path.is_file():

msticpy/vis/ti_browser.py

@@ -8,6 +8,9 @@
 from __future__ import annotations
 
 import pprint
+from html import escape
+from typing import Any
+from urllib.parse import urlsplit
 
 import pandas as pd
 from IPython.display import HTML
@@ -110,24 +113,24 @@ def get_ti_details(ioc_prov):
         ioc, provs = ioc_prov
         h2_style = "border: 1px solid;background-color: DarkGray; padding: 6px"
         h3_style = "background-color: SteelBlue; padding: 6px"
-        results = [f"<h2 style='{h2_style}'>{ioc}</h2>"]
+        results = [f"<h2 style='{h2_style}'>{escape(str(ioc))}</h2>"]
         for prov in provs:
             ioc_match = ti_data[(ti_data["Ioc"] == ioc) & (ti_data["Provider"] == prov)].iloc[
                 0
             ]
             results.extend(
                 (
-                    f"<h3 style='{h3_style}'>Type: '{ioc_match.IocType}', Provider: {prov}, "
-                    + f"severity: {ioc_match.Severity}</h3>",
+                    f"<h3 style='{h3_style}'>Type: '{escape(str(ioc_match.IocType))}', "
+                    + f"Provider: {escape(str(prov))}, "
+                    + f"severity: {escape(str(ioc_match.Severity))}</h3>",
                     "<h4>Details</h4>",
                 )
             )
 
             results.append(_ti_detail_table(ioc_match.Details))
             results.extend(
                 (
-                    f"<h4>Reference: </h4><a href='{ioc_match.Reference}'>"
-                    + f"{ioc_match.Reference}</a><br>",
+                    "<h4>Reference: </h4>" + _reference_html(ioc_match.Reference) + "<br>",
                     "<hr>",
                     "<h4>Raw Results</h4>",
                 )
@@ -141,7 +144,9 @@ def get_ti_details(ioc_prov):
 
 def raw_results(raw_result: str) -> str:
     """Create pre-formatted details for raw results."""
-    fmt_details = pprint.pformat(raw_result).replace("\n", "<br>").replace(" ", "&nbsp;")
+    fmt_details = (
+        escape(pprint.pformat(raw_result)).replace("\n", "<br>").replace(" ", "&nbsp;")
+    )
     return f"""
         <details>
         <summary> <u>Raw results from provider...</u></summary>
@@ -150,6 +155,19 @@ def raw_results(raw_result: str) -> str:
         """
 
 
+def _reference_html(reference: object) -> str:
+    """Render HTTP/HTTPS references as links and other references as text."""
+    reference_text = str(reference)
+    escaped_reference = escape(reference_text)
+    try:
+        parsed = urlsplit(reference_text)
+    except ValueError:
+        return escaped_reference
+    if parsed.scheme.lower() in {"http", "https"} and parsed.hostname:
+        return f"<a href='{escaped_reference}'>{escaped_reference}</a>"
+    return escaped_reference
+
+
 _TI_TABLE_STYLE = """
 <style>
     .tb_ti_res {border-collapse: collapse; width: 60%; border: 1px solid #ddd !important;}
@@ -173,14 +191,17 @@ def _ti_detail_table(detail_dict: dict) -> str:
     )
 
 
-def _dict_to_html(detail_dict):
+def _dict_to_html(detail_dict: Any) -> list[str]:
+    """Render nested details with escaped keys and values."""
     html_txt = []
     if not isinstance(detail_dict, dict):
-        return detail_dict
+        return [escape(str(detail_dict))]
     for key, val in detail_dict.items():
-        html_txt.append(f"<tr class='cell_ti'><td class='cell_ti_first'>{key}</td>")
+        html_txt.append(
+            f"<tr class='cell_ti'><td class='cell_ti_first'>{escape(str(key))}</td>"
+        )
         if not isinstance(val, dict):
-            html_txt.append(f"<td class='cell_ti'>{val}</td></tr>")
+            html_txt.append(f"<td class='cell_ti'>{escape(str(val))}</td></tr>")
         else:
             html_txt.extend(_dict_to_html(val))
             html_txt.append("</tr>")

requirements-dev.txt

@@ -15,7 +15,8 @@ pipreqs>=0.4.9
 pre-commit>=2.7.1
 pydocstyle>=6.3.0
 pygeohash>=1.2.0
-pylint>=2.5.3
+# Match pre-commit; review new checks explicitly when upgrading.
+pylint==4.0.9
 pyroma>=5.0.1
 pytest-check>=1.0.1
 pytest-cov>=7.1.0