From 36a6c3d2427b1482695d4472066c06e0986d3d0d Mon Sep 17 00:00:00 2001 From: lijunliang Date: Wed, 5 Jun 2024 01:21:44 +0000 Subject: [PATCH 01/12] add refiner --- tests/sd-webui/utils.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index ec3cf32a4..43100e7b2 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -59,11 +59,13 @@ def get_extra_args() -> List[Dict[str, Any]]: {"init_images": [get_init_image()]}, ] - return [ - quant_args, - txt2img_args, + refiner_args = [ + {"Refiner": {"args": [x, "v1-5-pruned-emaonly.safetensors [6ce0161689]", 0.5]}} + for x in [True, False] ] + return [quant_args, txt2img_args, refiner_args] + def get_all_args() -> Iterable[Dict[str, Any]]: for extra_args in product(*get_extra_args()): @@ -77,6 +79,10 @@ def is_txt2img(data: Dict[str, Any]) -> bool: return "init_images" not in data +def is_refiner(data: Dict[str, Any]) -> bool: + return data["Refiner"]["args"][0] not in data + + def is_quant(data: Dict[str, Any]) -> bool: return data["script_args"][0] @@ -121,13 +127,15 @@ def generate_image(filename: str, data: Dict[str, Any]): def get_target_image_filename(data: Dict[str, Any]) -> str: - parent_path = TXT2IMG_TARGET_FOLDER if is_txt2img else IMG2IMG_TARGET_FOLDER + parent_path = TXT2IMG_TARGET_FOLDER if is_txt2img(data) else IMG2IMG_TARGET_FOLDER if not Path(parent_path).exists(): Path(parent_path).mkdir(mode=777, parents=True) txt2img_str = "txt2img" if is_txt2img(data) else "img2img" quant_str = "-quant" if is_quant(data) else "" - return f"{parent_path}/onediff{quant_str}-{txt2img_str}-w{WIDTH}-h{HEIGHT}-seed-{SEED}-numstep-{NUM_STEPS}.png" + refiner_str = "-refiner" if is_refiner(data) else "" + + return f"{parent_path}/onediff{quant_str}{refiner_str}-{txt2img_str}-w{WIDTH}-h{HEIGHT}-seed-{SEED}-numstep-{NUM_STEPS}.png" def check_and_generate_images(): @@ -136,12 +144,13 @@ def check_and_generate_images(): if not Path(image_path).exists(): print(f"Generating image for {get_data_summary(data)}...") generate_image(image_path, data) - print(f"Image for {get_data_summary(data)} exists, skip generating...") - generate_image(image_path, data) + else: + print(f"Image for {get_data_summary(data)} exists, skip generating...") def get_data_summary(data: Dict[str, Any]) -> Dict[str, bool]: return { "is_txt2img": is_txt2img(data), "is_quant": is_quant(data), + "is_refiner": is_refiner(data), } From d139de57435ad0ad85919789343e719b8eba1045 Mon Sep 17 00:00:00 2001 From: lijunliang Date: Wed, 5 Jun 2024 12:51:21 +0000 Subject: [PATCH 02/12] use model checkpoint --- .github/workflows/examples.yml | 3 +++ tests/sd-webui/utils.py | 2 +- tests/webui-docker-compose.yml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index a9eedcdae..79a521a7d 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -18,6 +18,7 @@ env: WEBUI_SRC_DIR: stable-diffusion-webui WEBUI_DEPENDENCIES_SUBDIR: repos SDXL_BASE: /share_nfs/hf_models/sd_xl_base_1.0.safetensors + REFINER_BASE: /share_nfs/onediff_ci/sd-webui/safetensors/sd_xl_refiner_1.0.safetensors UNET_INT8: /share_nfs/hf_models/unet_int8 CONTROL_LORA_OPENPOSEXL2_RANK256: /share_nfs/hf_models/controlnet/control-lora-openposeXL2-rank256.safetensors SELENIUM_CONTAINER_NAME: selenium-test @@ -379,6 +380,8 @@ jobs: - if: matrix.test-suite == 'webui' run: docker exec ${{ env.CONTAINER_NAME }} python3 -m pytest -v -s tests/sd-webui/test_api.py + env: + REFINER_BASE: ${{ env.REFINER_BASE }} - name: Show WebUI Log if: matrix.test-suite == 'webui' diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index 43100e7b2..0310b6047 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -60,7 +60,7 @@ def get_extra_args() -> List[Dict[str, Any]]: ] refiner_args = [ - {"Refiner": {"args": [x, "v1-5-pruned-emaonly.safetensors [6ce0161689]", 0.5]}} + {"Refiner": {"args": [x, "sd_xl_refiner_1.0.safetensors", 0.5]}} for x in [True, False] ] diff --git a/tests/webui-docker-compose.yml b/tests/webui-docker-compose.yml index 6f70c713a..a33860238 100644 --- a/tests/webui-docker-compose.yml +++ b/tests/webui-docker-compose.yml @@ -50,6 +50,7 @@ services: - $HOME/test-container-cache-${CONTAINER_NAME}/dot-cache:/root/.cache - /share_nfs:/share_nfs:ro - /share_nfs/onediff_ci/sd-webui/images:/share_nfs/onediff_ci/sd-webui/images:rw + - ${REFINER_BASE}:/app/${WEBUI_SRC_DIR}/models/Stable-diffusion:ro - ${PWD}/${WEBUI_SRC_DIR}:/app/${WEBUI_SRC_DIR} - ${PWD}/onediff_sd_webui_extensions:/app/${WEBUI_SRC_DIR}/extensions/onediff_sd_webui_extensions - $PWD:/src/onediff From 08f7a0f5ff00d027ef3af855657420fc76158d21 Mon Sep 17 00:00:00 2001 From: lijunliang Date: Wed, 5 Jun 2024 14:49:08 +0000 Subject: [PATCH 03/12] fix CI mount error --- tests/webui-docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/webui-docker-compose.yml b/tests/webui-docker-compose.yml index a33860238..a4f79c163 100644 --- a/tests/webui-docker-compose.yml +++ b/tests/webui-docker-compose.yml @@ -50,7 +50,7 @@ services: - $HOME/test-container-cache-${CONTAINER_NAME}/dot-cache:/root/.cache - /share_nfs:/share_nfs:ro - /share_nfs/onediff_ci/sd-webui/images:/share_nfs/onediff_ci/sd-webui/images:rw - - ${REFINER_BASE}:/app/${WEBUI_SRC_DIR}/models/Stable-diffusion:ro + - ${REFINER_BASE}:/app/${WEBUI_SRC_DIR}/models/Stable-diffusion/sd_xl_refiner_1.0.safetensors:ro - ${PWD}/${WEBUI_SRC_DIR}:/app/${WEBUI_SRC_DIR} - ${PWD}/onediff_sd_webui_extensions:/app/${WEBUI_SRC_DIR}/extensions/onediff_sd_webui_extensions - $PWD:/src/onediff From 053731bd1e7a8597bc2fb69e294efd7388c59794 Mon Sep 17 00:00:00 2001 From: lijunliang Date: Wed, 5 Jun 2024 16:30:57 +0000 Subject: [PATCH 04/12] debug low ssim --- tests/sd-webui/test_api.py | 10 ++++++++++ tests/sd-webui/utils.py | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/sd-webui/test_api.py b/tests/sd-webui/test_api.py index c745ad86d..a2b55501a 100644 --- a/tests/sd-webui/test_api.py +++ b/tests/sd-webui/test_api.py @@ -16,8 +16,10 @@ get_target_image_filename, is_txt2img, post_request_and_check, + save_image, ) +import os @pytest.fixture(scope="session", autouse=True) def prepare_target_images(): @@ -52,6 +54,14 @@ def test_image_ssim(base_url, data): url = f"{base_url}/{endpoint}" generated_image = get_image_array_from_response(post_request_and_check(url, data)) target_image_path = get_target_image_filename(data) + directory, filename = os.path.split(target_image_path) + + # 添加 "generate_" 前缀 + new_filename = 'generated_' + filename + # 构建新的文件路径 + new_filepath = os.path.join(directory, new_filename) + save_image(generated_image,new_filepath) + target_image = np.array(Image.open(target_image_path)) ssim_value = cal_ssim(generated_image, target_image) assert ssim_value > 0.985 diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index 0310b6047..a5d1c9df7 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -80,7 +80,7 @@ def is_txt2img(data: Dict[str, Any]) -> bool: def is_refiner(data: Dict[str, Any]) -> bool: - return data["Refiner"]["args"][0] not in data + return data["Refiner"]["args"][0] def is_quant(data: Dict[str, Any]) -> bool: @@ -107,6 +107,11 @@ def get_image_array_from_response(response): return np.array(Image.open(io.BytesIO(imgdata))) +def save_image(image_array, file_path): + image = Image.fromarray(image_array) + image.save(file_path) + + def get_init_image(): img_path = str(Path(__file__).parent / "cat.png") return encode_file_to_base64(img_path) From b26610045324847130ef7b382fa38f3aef6524ac Mon Sep 17 00:00:00 2001 From: lijunliang Date: Thu, 6 Jun 2024 07:19:41 +0000 Subject: [PATCH 05/12] fixing using refiner --- tests/sd-webui/test_api.py | 30 ++++++++++-------------------- tests/sd-webui/utils.py | 8 ++++++-- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/tests/sd-webui/test_api.py b/tests/sd-webui/test_api.py index a2b55501a..b0d770ce5 100644 --- a/tests/sd-webui/test_api.py +++ b/tests/sd-webui/test_api.py @@ -1,25 +1,15 @@ +import os + import numpy as np import pytest from PIL import Image -from utils import ( - IMG2IMG_API_ENDPOINT, - OPTIONS_API_ENDPOINT, - SAVED_GRAPH_NAME, - TXT2IMG_API_ENDPOINT, - WEBUI_SERVER_URL, - cal_ssim, - check_and_generate_images, - get_all_args, - get_base_args, - get_data_summary, - get_image_array_from_response, - get_target_image_filename, - is_txt2img, - post_request_and_check, - save_image, -) +from utils import (IMG2IMG_API_ENDPOINT, OPTIONS_API_ENDPOINT, + SAVED_GRAPH_NAME, TXT2IMG_API_ENDPOINT, WEBUI_SERVER_URL, + cal_ssim, check_and_generate_images, get_all_args, + get_base_args, get_data_summary, + get_image_array_from_response, get_target_image_filename, + is_txt2img, post_request_and_check, save_image) -import os @pytest.fixture(scope="session", autouse=True) def prepare_target_images(): @@ -57,10 +47,10 @@ def test_image_ssim(base_url, data): directory, filename = os.path.split(target_image_path) # 添加 "generate_" 前缀 - new_filename = 'generated_' + filename + new_filename = "generated_" + filename # 构建新的文件路径 new_filepath = os.path.join(directory, new_filename) - save_image(generated_image,new_filepath) + # save_image(generated_image,new_filepath) target_image = np.array(Image.open(target_image_path)) ssim_value = cal_ssim(generated_image, target_image) diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index a5d1c9df7..e38e30d86 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -60,7 +60,11 @@ def get_extra_args() -> List[Dict[str, Any]]: ] refiner_args = [ - {"Refiner": {"args": [x, "sd_xl_refiner_1.0.safetensors", 0.5]}} + { + "refiner": x, + "refiner_checkpoint": "sd_xl_refiner_1.0", + "refiner_switch_at": 0.8, + } for x in [True, False] ] @@ -80,7 +84,7 @@ def is_txt2img(data: Dict[str, Any]) -> bool: def is_refiner(data: Dict[str, Any]) -> bool: - return data["Refiner"]["args"][0] + return data["refiner"] def is_quant(data: Dict[str, Any]) -> bool: From 89b49d074683346de19b37e4d49de566cf13b9c4 Mon Sep 17 00:00:00 2001 From: lijunliang Date: Thu, 6 Jun 2024 07:20:41 +0000 Subject: [PATCH 06/12] save imgs --- tests/sd-webui/test_api.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/sd-webui/test_api.py b/tests/sd-webui/test_api.py index b0d770ce5..7885b5fad 100644 --- a/tests/sd-webui/test_api.py +++ b/tests/sd-webui/test_api.py @@ -3,12 +3,23 @@ import numpy as np import pytest from PIL import Image -from utils import (IMG2IMG_API_ENDPOINT, OPTIONS_API_ENDPOINT, - SAVED_GRAPH_NAME, TXT2IMG_API_ENDPOINT, WEBUI_SERVER_URL, - cal_ssim, check_and_generate_images, get_all_args, - get_base_args, get_data_summary, - get_image_array_from_response, get_target_image_filename, - is_txt2img, post_request_and_check, save_image) +from utils import ( + IMG2IMG_API_ENDPOINT, + OPTIONS_API_ENDPOINT, + SAVED_GRAPH_NAME, + TXT2IMG_API_ENDPOINT, + WEBUI_SERVER_URL, + cal_ssim, + check_and_generate_images, + get_all_args, + get_base_args, + get_data_summary, + get_image_array_from_response, + get_target_image_filename, + is_txt2img, + post_request_and_check, + save_image, +) @pytest.fixture(scope="session", autouse=True) @@ -46,11 +57,9 @@ def test_image_ssim(base_url, data): target_image_path = get_target_image_filename(data) directory, filename = os.path.split(target_image_path) - # 添加 "generate_" 前缀 new_filename = "generated_" + filename - # 构建新的文件路径 new_filepath = os.path.join(directory, new_filename) - # save_image(generated_image,new_filepath) + save_image(generated_image, new_filepath) target_image = np.array(Image.open(target_image_path)) ssim_value = cal_ssim(generated_image, target_image) From d9953b42de70aa5d27d18fa912adae9cd9aa3aad Mon Sep 17 00:00:00 2001 From: lijunliang Date: Fri, 7 Jun 2024 01:54:36 +0000 Subject: [PATCH 07/12] fixing refiner --- tests/sd-webui/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index e38e30d86..39b307be9 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -60,12 +60,12 @@ def get_extra_args() -> List[Dict[str, Any]]: ] refiner_args = [ + {}, { - "refiner": x, + "refiner": True, "refiner_checkpoint": "sd_xl_refiner_1.0", "refiner_switch_at": 0.8, - } - for x in [True, False] + }, ] return [quant_args, txt2img_args, refiner_args] @@ -84,7 +84,7 @@ def is_txt2img(data: Dict[str, Any]) -> bool: def is_refiner(data: Dict[str, Any]) -> bool: - return data["refiner"] + return "refiner" in data def is_quant(data: Dict[str, Any]) -> bool: From c0d9465fc94a35cb9020a83cf7755e3c17a641e3 Mon Sep 17 00:00:00 2001 From: lijunliang Date: Fri, 7 Jun 2024 08:49:57 +0000 Subject: [PATCH 08/12] use same modle --- tests/sd-webui/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index 39b307be9..0651de601 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -29,6 +29,7 @@ def get_base_args() -> Dict[str, Any]: return { "prompt": "1girl", + "sd_model_checkpoint": "v1-5-pruned-emaonly.safetensors", "negative_prompt": "", "seed": SEED, "steps": NUM_STEPS, From f07017f2db919c3017efbc5118d16e4907f4ff88 Mon Sep 17 00:00:00 2001 From: lijunliang Date: Fri, 14 Jun 2024 02:25:42 +0000 Subject: [PATCH 09/12] remoce debug code --- tests/sd-webui/test_api.py | 7 +------ tests/sd-webui/utils.py | 5 ----- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/sd-webui/test_api.py b/tests/sd-webui/test_api.py index aea8f8a39..9d4c6606b 100644 --- a/tests/sd-webui/test_api.py +++ b/tests/sd-webui/test_api.py @@ -57,14 +57,9 @@ def test_image_ssim(base_url, data): generated_image = get_image_array_from_response(post_request_and_check(url, data)) target_image_path = get_target_image_filename(data) directory, filename = os.path.split(target_image_path) - - new_filename = "generated_" + filename - new_filepath = os.path.join(directory, new_filename) - save_image(generated_image, new_filepath) - target_image = np.array(Image.open(target_image_path)) ssim_value = cal_ssim(generated_image, target_image) - assert ssim_value > 0.985 + assert ssim_value > 0.98 def test_onediff_save_graph(url_txt2img): diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index 0651de601..37f8e0a8c 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -112,11 +112,6 @@ def get_image_array_from_response(response): return np.array(Image.open(io.BytesIO(imgdata))) -def save_image(image_array, file_path): - image = Image.fromarray(image_array) - image.save(file_path) - - def get_init_image(): img_path = str(Path(__file__).parent / "cat.png") return encode_file_to_base64(img_path) From b9b73ccc1f9abb1bf583dae49e663bed6e4ef45d Mon Sep 17 00:00:00 2001 From: lijunliang Date: Tue, 18 Jun 2024 02:09:14 +0000 Subject: [PATCH 10/12] add model pth --- .github/workflows/examples.yml | 4 +++- tests/sd-webui/test_api.py | 6 +----- tests/webui-docker-compose.yml | 2 ++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 5b6be3c24..4a49b6232 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -19,6 +19,7 @@ env: WEBUI_DEPENDENCIES_SUBDIR: repos SDXL_BASE: /share_nfs/hf_models/sd_xl_base_1.0.safetensors REFINER_BASE: /share_nfs/onediff_ci/sd-webui/safetensors/sd_xl_refiner_1.0.safetensors + AWPAINTING_BASE: /share_nfs/onediff_ci/sd-webui/safetensors/AWPainting_v1.2.safetensors UNET_INT8: /share_nfs/hf_models/unet_int8 CONTROL_LORA_OPENPOSEXL2_RANK256: /share_nfs/hf_models/controlnet/control-lora-openposeXL2-rank256.safetensors SELENIUM_CONTAINER_NAME: selenium-test @@ -382,7 +383,8 @@ jobs: run: docker exec ${{ env.CONTAINER_NAME }} python3 -m pytest -v -s tests/sd-webui/test_api.py env: REFINER_BASE: ${{ env.REFINER_BASE }} - + AWPAINTING_BASE: ${{ env.AWPAINTING_BASE }} + SDXL_BASE: ${{ env.SDXL_BASE }} - name: Show WebUI Log if: matrix.test-suite == 'webui' run: docker exec ${{ env.CONTAINER_NAME }} cat /app/${{ env.WEBUI_SRC_DIR }}/onediff_webui.log diff --git a/tests/sd-webui/test_api.py b/tests/sd-webui/test_api.py index 8ded9897d..834fb3cfa 100644 --- a/tests/sd-webui/test_api.py +++ b/tests/sd-webui/test_api.py @@ -1,9 +1,5 @@ import os -<<<<<<< HEAD - -======= from pathlib import Path ->>>>>>> 156d4f098c20c9daf13f051896646d56880f38ab import numpy as np import pytest from PIL import Image @@ -31,7 +27,7 @@ @pytest.fixture(scope="session", autouse=True) def change_model(): option_payload = { - "sd_model_checkpoint": "checkpoints/AWPainting_v1.2.safetensors", + "sd_model_checkpoint": "AWPainting_v1.2.safetensors", } post_request_and_check(f"{WEBUI_SERVER_URL}/{OPTIONS_API_ENDPOINT}", option_payload) diff --git a/tests/webui-docker-compose.yml b/tests/webui-docker-compose.yml index a4f79c163..426eb87e3 100644 --- a/tests/webui-docker-compose.yml +++ b/tests/webui-docker-compose.yml @@ -51,6 +51,8 @@ services: - /share_nfs:/share_nfs:ro - /share_nfs/onediff_ci/sd-webui/images:/share_nfs/onediff_ci/sd-webui/images:rw - ${REFINER_BASE}:/app/${WEBUI_SRC_DIR}/models/Stable-diffusion/sd_xl_refiner_1.0.safetensors:ro + - ${AWPAINTING_BASE}:/app/${WEBUI_SRC_DIR}/models/Stable-diffusion/AWPainting_v1.2.safetensors:ro + - ${SDXL_BASE}:/app/${WEBUI_SRC_DIR}/models/Stable-diffusion/sd_xl_base_1.0.safetensors:ro - ${PWD}/${WEBUI_SRC_DIR}:/app/${WEBUI_SRC_DIR} - ${PWD}/onediff_sd_webui_extensions:/app/${WEBUI_SRC_DIR}/extensions/onediff_sd_webui_extensions - $PWD:/src/onediff From c48753309c6d8bfc4ed93b4c1744403de9132d2e Mon Sep 17 00:00:00 2001 From: lijunliang Date: Wed, 19 Jun 2024 01:21:34 +0000 Subject: [PATCH 11/12] change sd_test_model --- tests/sd-webui/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sd-webui/utils.py b/tests/sd-webui/utils.py index a4e9d4a53..772ed1044 100644 --- a/tests/sd-webui/utils.py +++ b/tests/sd-webui/utils.py @@ -29,7 +29,7 @@ def get_base_args() -> Dict[str, Any]: return { "prompt": "1girl", - "sd_model_checkpoint": "v1-5-pruned-emaonly.safetensors", + "sd_model_checkpoint": "AWPainting_v1.2.safetensors", "negative_prompt": "", "seed": SEED, "steps": NUM_STEPS, From 6d29d8acb292b3c4343f71388c1a28901b86275f Mon Sep 17 00:00:00 2001 From: lijunliang Date: Fri, 21 Jun 2024 06:25:38 +0000 Subject: [PATCH 12/12] remove extra --- tests/sd-webui/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/sd-webui/test_api.py b/tests/sd-webui/test_api.py index d632c43b5..04a5f8e27 100644 --- a/tests/sd-webui/test_api.py +++ b/tests/sd-webui/test_api.py @@ -23,7 +23,6 @@ ) -THRESHOLD = 0.97 @pytest.fixture(scope="session", autouse=True) def change_model():