From 0bc1769d32256ce53bb3a35ecbaa88e79bedb2ea Mon Sep 17 00:00:00 2001 From: Quercus Hamlin Date: Wed, 30 Sep 2026 16:42:37 -0400 Subject: [PATCH 1/3] chore: trying /tmp/ with not /var/tmp for deploy --- docker/compose.local.yaml | 8 ++++---- docker/deploy_local.sh | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docker/compose.local.yaml b/docker/compose.local.yaml index c2b3e91..5554cc2 100644 --- a/docker/compose.local.yaml +++ b/docker/compose.local.yaml @@ -22,8 +22,8 @@ services: - ../.env volumes: - ../.pyiceberg.yaml:/app/.pyiceberg.yaml:ro - - /var/tmp/icefabric_local_catalog:/tmp/icefabric_local_catalog:ro - - /var/tmp/icefabric_streamflow_obs:/var/tmp/icefabric_streamflow_obs:ro + - /tmp/icefabric_local_catalog:/tmp/icefabric_local_catalog:ro + - /tmp/icefabric_streamflow_obs:/tmp/icefabric_streamflow_obs:ro command: ["python", "-m", "app.main", "--catalog", "sql"] restart: always healthcheck: @@ -44,6 +44,6 @@ services: - ../.env volumes: - ../.pyiceberg.yaml:/app/.pyiceberg.yaml:ro - - /var/tmp/icefabric_local_catalog:/tmp/icefabric_local_catalog:ro - - /var/tmp/icefabric_streamflow_obs:/var/tmp/icefabric_streamflow_obs:ro + - /tmp/icefabric_local_catalog:/tmp/icefabric_local_catalog:ro + - /tmp/icefabric_streamflow_obs:/tmp/icefabric_streamflow_obs:ro restart: always diff --git a/docker/deploy_local.sh b/docker/deploy_local.sh index ab39c52..f1a6c4e 100755 --- a/docker/deploy_local.sh +++ b/docker/deploy_local.sh @@ -94,21 +94,21 @@ echo "[INFO] Branch: $(git branch --show-current 2>/dev/null || echo 'not a git echo "[INFO] Commit: $(git rev-parse --short HEAD 2>/dev/null || echo 'N/A')" # --- Extract archive --- -ARCHIVE_DIR="/var/tmp/icefabric_local_catalog" +ARCHIVE_DIR="/tmp/icefabric_local_catalog" WAREHOUSE_DIR="$ARCHIVE_DIR/warehouse" PYICEBERG_DB="$ARCHIVE_DIR/pyiceberg_catalog.db" -LOCAL_ICECHUNK="/var/tmp/icefabric_streamflow_obs" +LOCAL_ICECHUNK="/tmp/icefabric_streamflow_obs" if [[ -f "$PYICEBERG_DB" && -d "$LOCAL_ICECHUNK" ]]; then echo "[INFO] Archive already extracted, skipping download" else echo "[INFO] Downloading and extracting archive from: $S3_ARCHIVE_PATH" ARCHIVE_FILENAME=$(basename "$S3_ARCHIVE_PATH") - $AWS_CMD s3 cp "$S3_ARCHIVE_PATH" "/var/tmp/$ARCHIVE_FILENAME" + $AWS_CMD s3 cp "$S3_ARCHIVE_PATH" "/tmp/$ARCHIVE_FILENAME" echo "[INFO] Extracting archive..." - tar -xf "/var/tmp/$ARCHIVE_FILENAME" -C /var/tmp/ - rm -f "/var/tmp/$ARCHIVE_FILENAME" + tar -xf "/tmp/$ARCHIVE_FILENAME" -C /tmp/ + rm -f "/tmp/$ARCHIVE_FILENAME" fi # --- Verify extracted files --- From bacb34c2aed21303eb76b75ddbffaa4038b82fda Mon Sep 17 00:00:00 2001 From: Quercus Hamlin Date: Fri, 2 Oct 2026 11:23:44 -0400 Subject: [PATCH 2/3] docs: update from /var/tmp to /tmp in docs --- README.md | 3 +-- docs/dashboard-docs/running.md | 2 +- docs/user_guide/deployment.md | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c18ceca..1886437 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,7 @@ docker/deploy_local.sh s3://ngwpc-data/icefabric_catalog_archive.tar {aws_profil This process will take a while (10-30 minutes) because we need to download ~40 GB of data, extract a large archive, build 3 docker images, and then wait for the api to spin up. -The files will be saved to your `/var/tmp/`. If the both directories are present, the shell script will not re-download the archive. Delete `icefabric_local_catalog` and `icefabric_streamflow_obs` directories to force download. - +The files will be saved to your `/tmp/`. If the both directories are present, the shell script will not re-download the archive. Delete `icefabric_local_catalog` and `icefabric_streamflow_obs` directories to force download. Note that your `tmp` file system must be mounted to root and not a `tmpfs` file system tied to memory. You can check this by running: `df -hT /tmp`. The shell script will update your `.env` file to have the appropriate file paths. ### Documentation diff --git a/docs/dashboard-docs/running.md b/docs/dashboard-docs/running.md index 9c88db9..594a838 100644 --- a/docs/dashboard-docs/running.md +++ b/docs/dashboard-docs/running.md @@ -49,7 +49,7 @@ To run the dashboard locally against a local Iceberg catalog and local Icechunk For further information or troubleshooting, please reference [the script](https://github.com/NGWPC/icefabric/blob/main/docker/deploy_local.sh). !!! warning "Important" - The archive pulled from S3 is quite large. You will need ~100GB of free diskspace in the location where you are writing the archive. The script defaults to the root var/temp directory (`/var/tmp`) + The archive pulled from S3 is quite large. You will need ~100GB of free diskspace in the location where you are writing the archive. The script defaults to the root temp directory (`/tmp`) ## Building/deploying the Dashboard through Docker diff --git a/docs/user_guide/deployment.md b/docs/user_guide/deployment.md index b574813..0b42e15 100644 --- a/docs/user_guide/deployment.md +++ b/docs/user_guide/deployment.md @@ -30,7 +30,7 @@ You need AWS CLI 2.x. At time of writing this is not the version stored in the a docker/deploy_local.sh s3://ngwpc-data/icefabric_catalog_archive.tar {aws_profile} ``` -This process will take awhile (10-30 minutes) because we need to download ~40 GB of data, extract a large archive, build 3 docker images, and then wait for the api to spin up. The files will be saved to your `/var/tmp/`. If the both directories are present, the shell script will not re-download the archive. Delete `icefabric_local_catalog` and `icefabric_streamflow_obs` directories to force download. The shell script will update your `.env` file to have the appropriate file paths. +This process will take awhile (10-30 minutes) because we need to download ~40 GB of data, extract a large archive, build 3 docker images, and then wait for the api to spin up. The files will be saved to your `/tmp/`. If the both directories are present, the shell script will not re-download the archive. Delete `icefabric_local_catalog` and `icefabric_streamflow_obs` directories to force download. The shell script will update your `.env` file to have the appropriate file paths. Note that your `tmp` file system must be mounted to root and not a `tmpfs` file system tied to memory. You can check this by running: `df -hT /tmp`. The services will be routed behind an nginx reverse-proxy, with the underlying services only directly accessible from the localhost. From c52e855aafa35cdebcbe6616857044e4fad89b3f Mon Sep 17 00:00:00 2001 From: Quercus Hamlin Date: Fri, 2 Oct 2026 11:49:50 -0400 Subject: [PATCH 3/3] tests: change rise from commented to skip --- tests/app/test_rise_router.py | 181 +++++++++++++++++----------------- 1 file changed, 93 insertions(+), 88 deletions(-) diff --git a/tests/app/test_rise_router.py b/tests/app/test_rise_router.py index 18d9d1c..8f9fc4c 100644 --- a/tests/app/test_rise_router.py +++ b/tests/app/test_rise_router.py @@ -1,90 +1,95 @@ """RISE API NOT IN USE: LEAVING TESTS FOR ARCHIVE""" -# import json - -# import httpx -# import pytest -# from starlette.testclient import TestClient - -# from app.routers.rise_wrappers.router import EXT_RISE_BASE_URL, make_get_req_to_rise - -# resources = ["catalog-item", "catalog-record", "location", "result"] - -# good_ids = ["10835", "4462", "3672", "3672"] - -# bad_ids = ["0", "0", "0", "0"] - -# RISE_TIMEOUT_CODE = 504 -# RISE_UNAVAILABLE_CODE = 503 - - -# def _skip_if_rise_unavailable(response: httpx.Response) -> None: -# """Skip the test if RISE API is down or timing out.""" -# if response.status_code == RISE_TIMEOUT_CODE: -# pytest.skip("RISE API is unavailable (timeout)") -# if response.status_code == RISE_UNAVAILABLE_CODE: -# pytest.skip("RISE API is unavailable (service unavailable)") - - -# @pytest.mark.integration -# @pytest.mark.asyncio -# @pytest.mark.parametrize("resource_type,id", list(zip(resources, good_ids, strict=False))) -# async def test_get_item_by_id_good(client: TestClient, resource_type: str, id: str) -> None: -# """Test all per-ID endpoints for IDs that exist""" -# response = client.get(f"/v1/rise/{resource_type}/{id}") -# _skip_if_rise_unavailable(response) -# assert response.status_code == 200 -# rise_direct_response = await make_get_req_to_rise(f"{EXT_RISE_BASE_URL}/{resource_type}/{id}") -# if rise_direct_response["status_code"] in (RISE_TIMEOUT_CODE, RISE_UNAVAILABLE_CODE): -# pytest.skip(f"RISE API direct request unavailable (status {rise_direct_response['status_code']})") -# assert json.loads(response.text) == rise_direct_response["detail"] - - -# @pytest.mark.integration -# @pytest.mark.asyncio -# @pytest.mark.parametrize("resource_type,id", list(zip(resources, bad_ids, strict=False))) -# async def test_get_item_by_id_bad(client: TestClient, resource_type: str, id: str) -> None: -# """Test all per-ID endpoints for IDs that do not exist""" -# response = client.get(f"/v1/rise/{resource_type}/{id}") -# _skip_if_rise_unavailable(response) -# assert response.status_code == 404 -# rise_direct_response = await make_get_req_to_rise(f"{EXT_RISE_BASE_URL}/{resource_type}/{id}") -# if rise_direct_response["status_code"] in (RISE_TIMEOUT_CODE, RISE_UNAVAILABLE_CODE): -# pytest.skip(f"RISE API direct request unavailable (status {rise_direct_response['status_code']})") -# assert json.loads(response.text)["detail"] == rise_direct_response["detail"] - - -# @pytest.mark.integration -# @pytest.mark.asyncio -# @pytest.mark.parametrize("resource_type", resources) -# async def test_get_collection(client: TestClient, resource_type: str) -> None: -# """Test every resource collection endpoint - all default parameters""" -# # TODO - remove skip once the RISE api/result endpoint is no longer timing out -# if resource_type == "result": -# pytest.skip(f"Skipping {resource_type} endpoint test until RISE API endpoint stops timing out.") -# response = client.get(f"/v1/rise/{resource_type}") -# _skip_if_rise_unavailable(response) -# assert response.status_code == 200 -# rise_direct_response = await make_get_req_to_rise(f"{EXT_RISE_BASE_URL}/{resource_type}") -# if rise_direct_response["status_code"] != 200: -# pytest.skip(f"RISE API direct request failed with status {rise_direct_response['status_code']}") -# assert json.loads(response.text)["data"] == rise_direct_response["detail"]["data"] - - -# @pytest.mark.integration -# @pytest.mark.asyncio -# @pytest.mark.parametrize("resource_type", resources) -# async def test_get_collection_set_of_ids(client: TestClient, resource_type: str) -> None: -# """Test every resource collection endpoint with a set of three IDs""" -# # TODO - remove skip once the RISE api/result endpoint is no longer timing out -# if resource_type == "result": -# pytest.skip(f"Skipping {resource_type} endpoint test until RISE API endpoint stops timing out.") -# response = client.get(f"/v1/rise/{resource_type}?id=3672,4462,10835") -# _skip_if_rise_unavailable(response) -# assert response.status_code == 200 -# rise_direct_response = await make_get_req_to_rise( -# f"{EXT_RISE_BASE_URL}/{resource_type}?id=3672,4462,10835" -# ) -# if rise_direct_response["status_code"] != 200: -# pytest.skip(f"RISE API direct request failed with status {rise_direct_response['status_code']}") -# assert json.loads(response.text)["data"] == rise_direct_response["detail"]["data"] +import json + +import httpx +import pytest +from starlette.testclient import TestClient + +from app.routers.rise_wrappers.router import EXT_RISE_BASE_URL, make_get_req_to_rise + +resources = ["catalog-item", "catalog-record", "location", "result"] + +good_ids = ["10835", "4462", "3672", "3672"] + +bad_ids = ["0", "0", "0", "0"] + +RISE_TIMEOUT_CODE = 504 +RISE_UNAVAILABLE_CODE = 503 + + +@pytest.mark.skip(reason="RISE not used at this time") +def _skip_if_rise_unavailable(response: httpx.Response) -> None: + """Skip the test if RISE API is down or timing out.""" + if response.status_code == RISE_TIMEOUT_CODE: + pytest.skip("RISE API is unavailable (timeout)") + if response.status_code == RISE_UNAVAILABLE_CODE: + pytest.skip("RISE API is unavailable (service unavailable)") + + +@pytest.mark.skip(reason="RISE not used at this time") +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize("resource_type,id", list(zip(resources, good_ids, strict=False))) +async def test_get_item_by_id_good(client: TestClient, resource_type: str, id: str) -> None: + """Test all per-ID endpoints for IDs that exist""" + response = client.get(f"/v1/rise/{resource_type}/{id}") + _skip_if_rise_unavailable(response) + assert response.status_code == 200 + rise_direct_response = await make_get_req_to_rise(f"{EXT_RISE_BASE_URL}/{resource_type}/{id}") + if rise_direct_response["status_code"] in (RISE_TIMEOUT_CODE, RISE_UNAVAILABLE_CODE): + pytest.skip(f"RISE API direct request unavailable (status {rise_direct_response['status_code']})") + assert json.loads(response.text) == rise_direct_response["detail"] + + +@pytest.mark.skip(reason="RISE not used at this time") +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize("resource_type,id", list(zip(resources, bad_ids, strict=False))) +async def test_get_item_by_id_bad(client: TestClient, resource_type: str, id: str) -> None: + """Test all per-ID endpoints for IDs that do not exist""" + response = client.get(f"/v1/rise/{resource_type}/{id}") + _skip_if_rise_unavailable(response) + assert response.status_code == 404 + rise_direct_response = await make_get_req_to_rise(f"{EXT_RISE_BASE_URL}/{resource_type}/{id}") + if rise_direct_response["status_code"] in (RISE_TIMEOUT_CODE, RISE_UNAVAILABLE_CODE): + pytest.skip(f"RISE API direct request unavailable (status {rise_direct_response['status_code']})") + assert json.loads(response.text)["detail"] == rise_direct_response["detail"] + + +@pytest.mark.skip(reason="RISE not used at this time") +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize("resource_type", resources) +async def test_get_collection(client: TestClient, resource_type: str) -> None: + """Test every resource collection endpoint - all default parameters""" + # TODO - remove skip once the RISE api/result endpoint is no longer timing out + if resource_type == "result": + pytest.skip(f"Skipping {resource_type} endpoint test until RISE API endpoint stops timing out.") + response = client.get(f"/v1/rise/{resource_type}") + _skip_if_rise_unavailable(response) + assert response.status_code == 200 + rise_direct_response = await make_get_req_to_rise(f"{EXT_RISE_BASE_URL}/{resource_type}") + if rise_direct_response["status_code"] != 200: + pytest.skip(f"RISE API direct request failed with status {rise_direct_response['status_code']}") + assert json.loads(response.text)["data"] == rise_direct_response["detail"]["data"] + + +@pytest.mark.skip(reason="RISE not used at this time") +@pytest.mark.integration +@pytest.mark.asyncio +@pytest.mark.parametrize("resource_type", resources) +async def test_get_collection_set_of_ids(client: TestClient, resource_type: str) -> None: + """Test every resource collection endpoint with a set of three IDs""" + # TODO - remove skip once the RISE api/result endpoint is no longer timing out + if resource_type == "result": + pytest.skip(f"Skipping {resource_type} endpoint test until RISE API endpoint stops timing out.") + response = client.get(f"/v1/rise/{resource_type}?id=3672,4462,10835") + _skip_if_rise_unavailable(response) + assert response.status_code == 200 + rise_direct_response = await make_get_req_to_rise( + f"{EXT_RISE_BASE_URL}/{resource_type}?id=3672,4462,10835" + ) + if rise_direct_response["status_code"] != 200: + pytest.skip(f"RISE API direct request failed with status {rise_direct_response['status_code']}") + assert json.loads(response.text)["data"] == rise_direct_response["detail"]["data"]