Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 4 additions & 12 deletions engineV2.py
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,7 @@ def check_gpu_memory(gpu_ids, num_workers_per_gpu, required_memory): # required


def init_worker_gpu(gpu_worker_list, lock, available_gpus, max_workers_per_gpu, options):
if options.log_dir:
set_test_log_path(options.log_dir)
set_engineV2()
init_log(options.log_dir, worker_tmp_logs=True)
my_pid = os.getpid()

def pid_exists(pid):
Expand Down Expand Up @@ -985,8 +983,6 @@ def main():
if options.bitwise_alignment:
options.atol = 0.0
options.rtol = 0.0
if options.log_dir:
set_test_log_path(options.log_dir)

if options.api_config:
try:
Expand All @@ -1008,8 +1004,7 @@ def main():

globals().update(_load_test_classes(options))

# set log_writer
set_engineV2()
init_log(options.log_dir, worker_tmp_logs=True)

options.api_config = options.api_config.strip()
print(
Expand Down Expand Up @@ -1097,10 +1092,7 @@ def main():
return
config_files = [options.api_config_file]

# set log_writer before resume/checkpoint handling
if options.log_dir:
set_test_log_path(options.log_dir)
set_engineV2()
init_log(options.log_dir, worker_tmp_logs=True)

# when engineV2 was interrupted, resume from .tmp dir
aggregate_logs(cleanup=True)
Expand Down Expand Up @@ -1282,7 +1274,7 @@ def cleanup_handler(*args):
finally:
print(f"{tested_case} cases have been tested.", flush=True)
log_counts = aggregate_logs(end=True)
print_log_info(all_case, log_counts)
print_log_info(max(all_case - tested_case, 0), log_counts)
end_time = time.time()
total_time = end_time - start_time
print(f"Test time: {round(total_time / 60, 3)} minutes.", flush=True)
Expand Down
22 changes: 6 additions & 16 deletions engineV4.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,7 @@ class WorkerSlot:


def _init_worker_runtime(slot_index, gpu_id, options, *, redirect_output):
if options.log_dir:
set_test_log_path(options.log_dir)
set_engineV2()
init_log(options.log_dir, worker_tmp_logs=True)

if gpu_id is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
Expand Down Expand Up @@ -258,9 +256,7 @@ def _build_sanitizer_case_command(api_config_str, options, log_dir, sanitizer_cm


def _sanitizer_worker_loop(slot_index, gpu_id, input_queue, result_queue, options):
if options.log_dir:
set_test_log_path(options.log_dir)
set_engineV2()
init_log(options.log_dir, worker_tmp_logs=True)
redirect_stdio()

child_process = None
Expand Down Expand Up @@ -1672,8 +1668,6 @@ def main():
if options.bitwise_alignment:
options.atol = 0.0
options.rtol = 0.0
if options.log_dir:
set_test_log_path(options.log_dir)

if options._sanitizer_child:
try:
Expand Down Expand Up @@ -1714,8 +1708,7 @@ def main():

globals().update(_load_test_classes(options))

# set log_writer
set_engineV2()
init_log(options.log_dir, worker_tmp_logs=True)

options.api_config = options.api_config.strip()
print(
Expand Down Expand Up @@ -1803,6 +1796,8 @@ def main():
return
config_files = [options.api_config_file]

init_log(options.log_dir, worker_tmp_logs=True)

# when engineV2 was interrupted, resume from .tmp dir
aggregate_logs(cleanup=True)
if options.use_compute_sanitizer:
Expand Down Expand Up @@ -1878,11 +1873,6 @@ def main():
if options.test_cpu:
print(f"Using {cpu_count()} CPU(s) for paddle in CPU mode.", flush=True)

# set log_writer
if options.log_dir:
set_test_log_path(options.log_dir)
set_engineV2()

# initialize worker pool (per-worker queue architecture)
pool = WorkerPool(available_gpus, max_workers_per_gpu, options)

Expand Down Expand Up @@ -2071,7 +2061,7 @@ def cleanup_handler(*args):
cleanup_sanitizer_tmp_dir()
print(f"{tested_case} cases have been tested.", flush=True)
log_counts = aggregate_logs(end=True)
print_log_info(all_case, log_counts)
print_log_info(max(all_case - tested_case, 0), log_counts)
end_time = time.time()
total_time = end_time - start_time
print(f"Test time: {round(total_time / 60, 3)} minutes.", flush=True)
Expand Down
Loading
Loading