diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 289af4dc..44a32cb3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -68,9 +68,20 @@ jobs: cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy touch ~/.msticpy/GeoLite2-City.mmdb cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor - touch ~/.msticpy/mordor/mitre_tact_cache.pkl - touch ~/.msticpy/mordor/mitre_tech_cache.pkl - touch ~/.msticpy/mordor/mordor_cache.pkl + python - <<'PY' + import json + from datetime import datetime, timezone + from pathlib import Path + + cache_folder = Path.home().joinpath(".msticpy", "mordor") + for filename in ("mitre_tact_cache.json", "mitre_tech_cache.json"): + cache_folder.joinpath(filename).touch() + metadata_path = cache_folder.joinpath("mordor_cache.json") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + for item in metadata.values(): + item["mp_last_updated"] = datetime.now(timezone.utc).isoformat() + metadata_path.write_text(json.dumps(metadata, indent=4), encoding="utf-8") + PY - name: Pytest env: MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 87d3d46d..709925d5 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,4 +1,7 @@ -# This workflow will upload a Python Package using Twine when a release is created +# Publishing is intentionally disabled. Remove the false job guards only after +# configuring the protected "pypi" environment and a PyPI trusted publisher for +# microsoft/msticpy, python-publish.yml, and the "pypi" environment. +# The environment must require approval and restrict deployments to release tags. # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries # This workflow uses actions that are not certified by GitHub. @@ -11,18 +14,22 @@ name: Upload Python Package to PyPI Prod on: release: types: [published] - workflow_dispatch: permissions: contents: read jobs: - deploy: - + build: + if: >- + ${{ false && github.event_name == 'release' && + !github.event.release.prerelease && startsWith(github.ref, 'refs/tags/v') }} runs-on: ubuntu-latest - permissions: read-all + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -38,8 +45,31 @@ jobs: --sdist --wheel --outdir dist/ + - name: Upload distributions + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: python-distributions + path: dist/ + if-no-files-found: error + retention-days: 1 + + publish: + needs: build + if: >- + ${{ false && needs.build.result == 'success' && + github.event_name == 'release' && !github.event.release.prerelease && + startsWith(github.ref, 'refs/tags/v') }} + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/msticpy + permissions: + id-token: write + steps: + - name: Download distributions + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: python-distributions + path: dist/ - name: Publish package uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c185ff82..c64a650d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/PyCQA/pylint - rev: v4.0.2 + rev: v4.0.9 hooks: - id: pylint args: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b4971f71..38547f6f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,9 +76,6 @@ stages: cp $(Build.SourcesDirectory)/tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy touch ~/.msticpy/GeoLite2-City.mmdb cp -r $(Build.SourcesDirectory)/tests/testdata/mordor/* ~/.msticpy/mordor - touch ~/.msticpy/mordor/mitre_tact_cache.pkl - touch ~/.msticpy/mordor/mitre_tech_cache.pkl - touch ~/.msticpy/mordor/mordor_cache.pkl condition: ne(variables.imageName, 'windows-latest') displayName: 'Prepare test dummy data - Linux' - script: | @@ -88,6 +85,23 @@ stages: copy /Y $(Build.SourcesDirectory)\tests\testdata\mordor\* %USERPROFILE%\.msticpy\mordor condition: eq(variables.imageName, 'windows-latest') displayName: 'Prepare test dummy data - Windows' + - task: PythonScript@0 + displayName: 'Refresh Mordor JSON test caches' + inputs: + scriptSource: inline + script: | + import json + from datetime import datetime, timezone + from pathlib import Path + + cache_folder = Path.home().joinpath(".msticpy", "mordor") + for filename in ("mitre_tact_cache.json", "mitre_tech_cache.json"): + cache_folder.joinpath(filename).touch() + metadata_path = cache_folder.joinpath("mordor_cache.json") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + for item in metadata.values(): + item["mp_last_updated"] = datetime.now(timezone.utc).isoformat() + metadata_path.write_text(json.dumps(metadata, indent=4), encoding="utf-8") - script: | echo MSTICPYCONFIG: $(Build.SourcesDirectory)/tests/msticpyconfig-test.yaml echo Env $MSTICPYCONFIG or %MSTICPYCONFIG% diff --git a/docs/source/data_acquisition/MordorData.rst b/docs/source/data_acquisition/MordorData.rst index 43d41735..e1075b55 100644 --- a/docs/source/data_acquisition/MordorData.rst +++ b/docs/source/data_acquisition/MordorData.rst @@ -174,6 +174,13 @@ will apply to all queries. metadata will be cached so you will not have to download this again in this session. +.. note:: MITRE techniques and tactics are cached as + ``mitre_tech_cache.json`` and ``mitre_tact_cache.json`` in the save folder. + Legacy ``.pkl`` caches are never loaded or migrated; the first connection + after upgrading downloads fresh MITRE data. Old pickle cache files can + be deleted. JSON cache files expire after 30 days, and invalid MITRE + caches are logged and refreshed. + Using these parameters in the query will override the provider settings and defaults for that query. @@ -423,4 +430,3 @@ for more details. .. figure:: _static/MordorBrowser-std-query-browser.png :alt: Using the standard query browser to view Mordor queries. :width: 5in - diff --git a/docs/source/data_acquisition/TIProviders.rst b/docs/source/data_acquisition/TIProviders.rst index 328446bb..1a6184c3 100644 --- a/docs/source/data_acquisition/TIProviders.rst +++ b/docs/source/data_acquisition/TIProviders.rst @@ -916,6 +916,10 @@ Address or URL) for all providers. For each provider that returns a result for an IoC, the summarized details will be shown in a table below the browse list. +Provider-supplied values, including raw results, are displayed as text rather +than interpreted as HTML. Only absolute HTTP and HTTPS reference URLs are +clickable; other references are displayed as plain text. + .. figure:: _static/ti_browser_results1.png :alt: Threat Intel results browser :width: 5.23000in diff --git a/msticpy/_version.py b/msticpy/_version.py index 45c5556d..7b45ec3e 100644 --- a/msticpy/_version.py +++ b/msticpy/_version.py @@ -1,3 +1,3 @@ """Version file.""" -VERSION = "3.1.0" +VERSION = "3.1.1" diff --git a/msticpy/data/drivers/mordor_driver.py b/msticpy/data/drivers/mordor_driver.py index 545c89fb..7cb3753d 100644 --- a/msticpy/data/drivers/mordor_driver.py +++ b/msticpy/data/drivers/mordor_driver.py @@ -8,7 +8,7 @@ from __future__ import annotations import json -import pickle # nosec +import logging import zipfile from collections import defaultdict from collections.abc import Generator, Iterable @@ -34,6 +34,8 @@ __version__ = VERSION __author__ = "Ian Hellen" +logger = logging.getLogger(__name__) + _MORDOR_TREE_URI = ( "https://api.github.com/repos/OTRF/Security-Datasets/git/trees/master?recursive=1" @@ -50,8 +52,8 @@ MITRE_TECHNIQUES: pd.DataFrame | None = None MITRE_TACTICS: pd.DataFrame | None = None -_MITRE_TECH_CACHE = "mitre_tech_cache.pkl" -_MITRE_TACTICS_CACHE = "mitre_tact_cache.pkl" +_MITRE_TECH_CACHE = "mitre_tech_cache.json" +_MITRE_TACTICS_CACHE = "mitre_tact_cache.json" _MORDOR_CACHE = "mordor_cache.json" _HTTP_TIMEOUT = DriverBase.get_http_timeout() @@ -645,6 +647,8 @@ def _fetch_mdr_metadata(cache_folder: str | None = None) -> dict[str, MordorEntr def _read_mordor_cache(cache_folder) -> dict[str, Any]: """Return dictionary of cached metadata if cached_folder is a valid path.""" md_cached_metadata: dict[str, Any] = {} + if cache_folder is None: + return md_cached_metadata mordor_cache = Path(cache_folder).joinpath(_MORDOR_CACHE) if _valid_cache(mordor_cache): try: @@ -657,6 +661,8 @@ def _read_mordor_cache(cache_folder) -> dict[str, Any]: def _write_mordor_cache(md_cached_metadata, cache_folder): """Write dictionary of cached metadata if cached_folder is a valid path.""" + if cache_folder is None: + return mordor_cache = Path(cache_folder).joinpath(_MORDOR_CACHE) json_text = json.dumps(md_cached_metadata, indent=4) Path(mordor_cache).write_text(json_text, encoding="utf-8") @@ -900,12 +906,18 @@ def _get_mitre_categories( tactics_cache = Path(cache_folder).joinpath(_MITRE_TACTICS_CACHE) if _valid_cache(tech_cache) and _valid_cache(tactics_cache): try: - # caching location only used if sepcified. - tech_df = pd.read_pickle(tech_cache) # nosec - tactics_df = pd.read_pickle(tactics_cache) # nosec + tech_df = _read_mitre_cache(tech_cache) + tactics_df = _read_mitre_cache(tactics_cache) + for cached_df in (tech_df, tactics_df): + if cached_df.index.name != "ID" or list(cached_df.columns) != [ + "Name", + "Description", + "url", + ]: + raise ValueError("Invalid MITRE cache schema") return tech_df, tactics_df - except pickle.PickleError: - pass + except (ValueError, KeyError, TypeError) as cache_err: + logger.warning("Invalid MITRE cache in %s: %s", cache_folder, cache_err) resp = httpx.get(_MITRE_JSON_URL, timeout=_HTTP_TIMEOUT, headers=mp_ua_header()) mitre = pd.json_normalize(resp.json()["objects"]) @@ -916,12 +928,18 @@ def _get_mitre_categories( tech_df = _reshape_mitre_df(mitre[mitre.type == "attack-pattern"]) tactics_df = _reshape_mitre_df(mitre[mitre.type == "x-mitre-tactic"]) if cache_folder: - tech_df.to_pickle(tech_cache) - tactics_df.to_pickle(tactics_cache) + tech_df.to_json(tech_cache, orient="table", indent=2) + tactics_df.to_json(tactics_cache, orient="table", indent=2) return tech_df, tactics_df +def _read_mitre_cache(cache_path: Path) -> pd.DataFrame: + """Read JSON while preserving text-column dtypes and null values.""" + cached_df = pd.read_json(cache_path, orient="table").astype(object) + return cached_df.where(cached_df.notna(), None) + + def _valid_cache(path: Path, expired_days=30): """Return True if the file exists and is younger than `expired_days`.""" if not path.is_file(): diff --git a/msticpy/vis/ti_browser.py b/msticpy/vis/ti_browser.py index 6dabb61c..08c650d6 100644 --- a/msticpy/vis/ti_browser.py +++ b/msticpy/vis/ti_browser.py @@ -8,6 +8,9 @@ from __future__ import annotations import pprint +from html import escape +from typing import Any +from urllib.parse import urlsplit import pandas as pd from IPython.display import HTML @@ -110,15 +113,16 @@ def get_ti_details(ioc_prov): ioc, provs = ioc_prov h2_style = "border: 1px solid;background-color: DarkGray; padding: 6px" h3_style = "background-color: SteelBlue; padding: 6px" - results = [f"

{ioc}

"] + results = [f"

{escape(str(ioc))}

"] for prov in provs: ioc_match = ti_data[(ti_data["Ioc"] == ioc) & (ti_data["Provider"] == prov)].iloc[ 0 ] results.extend( ( - f"

Type: '{ioc_match.IocType}', Provider: {prov}, " - + f"severity: {ioc_match.Severity}

", + f"

Type: '{escape(str(ioc_match.IocType))}', " + + f"Provider: {escape(str(prov))}, " + + f"severity: {escape(str(ioc_match.Severity))}

", "

Details

", ) ) @@ -126,8 +130,7 @@ def get_ti_details(ioc_prov): results.append(_ti_detail_table(ioc_match.Details)) results.extend( ( - f"

Reference:

" - + f"{ioc_match.Reference}
", + "

Reference:

" + _reference_html(ioc_match.Reference) + "
", "
", "

Raw Results

", ) @@ -141,7 +144,9 @@ def get_ti_details(ioc_prov): def raw_results(raw_result: str) -> str: """Create pre-formatted details for raw results.""" - fmt_details = pprint.pformat(raw_result).replace("\n", "
").replace(" ", " ") + fmt_details = ( + escape(pprint.pformat(raw_result)).replace("\n", "
").replace(" ", " ") + ) return f"""
Raw results from provider... @@ -150,6 +155,19 @@ def raw_results(raw_result: str) -> str: """ +def _reference_html(reference: object) -> str: + """Render HTTP/HTTPS references as links and other references as text.""" + reference_text = str(reference) + escaped_reference = escape(reference_text) + try: + parsed = urlsplit(reference_text) + except ValueError: + return escaped_reference + if parsed.scheme.lower() in {"http", "https"} and parsed.hostname: + return f"{escaped_reference}" + return escaped_reference + + _TI_TABLE_STYLE = """