Skip to content

Commit 02e1ccb

Browse files
alancuckinv-kkudrynski
alancucki
authored and
nv-kkudrynski
committed
[Speech/PyT] Save two dllogs: .json and .json.X
1 parent 87aa4b0 commit 02e1ccb

13 files changed

Lines changed: 67 additions & 85 deletions

File tree

‎PyTorch/SpeechRecognition/Jasper/common/tb_dllogger.py‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
import glob
33
import os
44
import re
5-
import numpy as np
5+
from pathlib import Path
66

7+
import numpy as np
78
import torch
89
from torch.utils.tensorboard import SummaryWriter
910

@@ -55,17 +56,11 @@ def log_grads(self, step, model):
5556
stat=stat)
5657

5758

58-
def unique_log_fpath(log_fpath):
59-
60-
if not os.path.isfile(log_fpath):
61-
return log_fpath
62-
63-
# Avoid overwriting old logs
64-
saved = sorted([int(re.search('\.(\d+)', f).group(1))
65-
for f in glob.glob(f'{log_fpath}.*')])
66-
67-
log_num = (saved[-1] if saved else 0) + 1
68-
return f'{log_fpath}.{log_num}'
59+
def unique_log_fpath(fpath):
60+
"""Have a unique log filename for every separate run"""
61+
log_num = max([0] + [int(re.search("\.(\d+)", Path(f).suffix).group(1))
62+
for f in glob.glob(f"{fpath}.*")])
63+
return f"{fpath}.{log_num + 1}"
6964

7065

7166
def stdout_step_format(step):
@@ -94,11 +89,12 @@ def init_log(args):
9489
enabled = (args.local_rank == 0)
9590
if enabled:
9691
fpath = args.log_file or os.path.join(args.output_dir, 'nvlog.json')
97-
backends = [JSONStreamBackend(Verbosity.DEFAULT,
98-
unique_log_fpath(fpath)),
99-
StdOutBackend(Verbosity.VERBOSE,
100-
step_format=stdout_step_format,
101-
metric_format=stdout_metric_format)]
92+
backends = [
93+
JSONStreamBackend(Verbosity.DEFAULT, fpath, append=True),
94+
JSONStreamBackend(Verbosity.DEFAULT, unique_log_fpath(fpath)),
95+
StdOutBackend(Verbosity.VERBOSE, step_format=stdout_step_format,
96+
metric_format=stdout_metric_format)
97+
]
10298
else:
10399
backends = []
104100

‎PyTorch/SpeechRecognition/Jasper/inference.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,11 @@ def main():
160160
args = parser.parse_args()
161161

162162
log_fpath = args.log_file or str(Path(args.output_dir, 'nvlog_infer.json'))
163-
log_fpath = unique_log_fpath(log_fpath)
164-
dllogger.init(backends=[JSONStreamBackend(Verbosity.DEFAULT, log_fpath),
165-
StdOutBackend(Verbosity.VERBOSE,
166-
metric_format=stdout_metric_format)])
167-
163+
dllogger.init(backends=[
164+
JSONStreamBackend(Verbosity.DEFAULT, log_fpath, append=True),
165+
JSONStreamBackend(Verbosity.DEFAULT, unique_log_fpath(log_fpath)),
166+
StdOutBackend(Verbosity.VERBOSE, metric_format=stdout_metric_format)
167+
])
168168
[dllogger.log("PARAMETER", {k: v}) for k, v in vars(args).items()]
169169

170170
for step in ['DNN', 'data+DNN', 'data']:

‎PyTorch/SpeechRecognition/Jasper/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ pyyaml>=5.4
77
soundfile
88
sox==1.4.1
99
tqdm==4.53.0
10-
git+https://github.com/NVIDIA/dllogger@v0.1.0#egg=dllogger
10+
git+https://github.com/NVIDIA/dllogger@v1.0.0#egg=dllogger

‎PyTorch/SpeechRecognition/QuartzNet/common/tb_dllogger.py‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,9 @@
1616
import glob
1717
import os
1818
import re
19-
import numpy as np
19+
from pathlib import Path
2020

21+
import numpy as np
2122
import torch
2223
from torch.utils.tensorboard import SummaryWriter
2324

@@ -69,17 +70,11 @@ def log_grads(self, step, model):
6970
stat=stat)
7071

7172

72-
def unique_log_fpath(log_fpath):
73-
74-
if not os.path.isfile(log_fpath):
75-
return log_fpath
76-
77-
# Avoid overwriting old logs
78-
saved = sorted([int(re.search('\.(\d+)', f).group(1))
79-
for f in glob.glob(f'{log_fpath}.*')])
80-
81-
log_num = (saved[-1] if saved else 0) + 1
82-
return f'{log_fpath}.{log_num}'
73+
def unique_log_fpath(fpath):
74+
"""Have a unique log filename for every separate run"""
75+
log_num = max([0] + [int(re.search("\.(\d+)", Path(f).suffix).group(1))
76+
for f in glob.glob(f"{fpath}.*")])
77+
return f"{fpath}.{log_num + 1}"
8378

8479

8580
def stdout_step_format(step):
@@ -108,11 +103,12 @@ def init_log(args):
108103
enabled = (args.local_rank == 0)
109104
if enabled:
110105
fpath = args.log_file or os.path.join(args.output_dir, 'nvlog.json')
111-
backends = [JSONStreamBackend(Verbosity.DEFAULT,
112-
unique_log_fpath(fpath)),
113-
StdOutBackend(Verbosity.VERBOSE,
114-
step_format=stdout_step_format,
115-
metric_format=stdout_metric_format)]
106+
backends = [
107+
JSONStreamBackend(Verbosity.DEFAULT, fpath, append=True),
108+
JSONStreamBackend(Verbosity.DEFAULT, unique_log_fpath(fpath)),
109+
StdOutBackend(Verbosity.VERBOSE, step_format=stdout_step_format,
110+
metric_format=stdout_metric_format)
111+
]
116112
else:
117113
backends = []
118114

‎PyTorch/SpeechRecognition/QuartzNet/inference.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -155,11 +155,11 @@ def main():
155155
args = parser.parse_args()
156156

157157
log_fpath = args.log_file or str(Path(args.output_dir, 'nvlog_infer.json'))
158-
log_fpath = unique_log_fpath(log_fpath)
159-
dllogger.init(backends=[JSONStreamBackend(Verbosity.DEFAULT, log_fpath),
160-
StdOutBackend(Verbosity.VERBOSE,
161-
metric_format=stdout_metric_format)])
162-
158+
dllogger.init(backends=[
159+
JSONStreamBackend(Verbosity.DEFAULT, log_fpath, append=True),
160+
JSONStreamBackend(Verbosity.DEFAULT, unique_log_fpath(log_fpath)),
161+
StdOutBackend(Verbosity.VERBOSE, metric_format=stdout_metric_format)
162+
])
163163
[dllogger.log("PARAMETER", {k: v}) for k, v in vars(args).items()]
164164

165165
for step in ['DNN', 'data+DNN', 'data']:

‎PyTorch/SpeechRecognition/QuartzNet/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ pyyaml>=5.4
77
soundfile
88
sox==1.4.1
99
tqdm==4.53.0
10-
git+https://github.com/NVIDIA/dllogger@v0.1.0#egg=dllogger
10+
git+https://github.com/NVIDIA/dllogger@v1.0.0#egg=dllogger

‎PyTorch/SpeechSynthesis/FastPitch/common/tb_dllogger.py‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -55,14 +55,10 @@ def log_grads(self, step, model):
5555

5656

5757
def unique_log_fpath(fpath):
58-
59-
if not Path(fpath).is_file():
60-
return fpath
61-
62-
# Avoid overwriting old logs
63-
saved = [re.search('\.(\d+)$', f) for f in glob.glob(f'{fpath}.*')]
64-
saved = [0] + [int(m.group(1)) for m in saved if m is not None]
65-
return f'{fpath}.{max(saved) + 1}'
58+
"""Have a unique log filename for every separate run"""
59+
log_num = max([0] + [int(re.search("\.(\d+)", Path(f).suffix).group(1))
60+
for f in glob.glob(f"{fpath}.*")])
61+
return f"{fpath}.{log_num + 1}"
6662

6763

6864
def stdout_step_format(step):
@@ -90,11 +86,12 @@ def stdout_metric_format(metric, metadata, value):
9086
def init(log_fpath, log_dir, enabled=True, tb_subsets=[], **tb_kw):
9187

9288
if enabled:
93-
backends = [JSONStreamBackend(Verbosity.DEFAULT,
94-
unique_log_fpath(log_fpath)),
95-
StdOutBackend(Verbosity.VERBOSE,
96-
step_format=stdout_step_format,
97-
metric_format=stdout_metric_format)]
89+
backends = [
90+
JSONStreamBackend(Verbosity.DEFAULT, log_fpath, append=True),
91+
JSONStreamBackend(Verbosity.DEFAULT, unique_log_fpath(log_fpath)),
92+
StdOutBackend(Verbosity.VERBOSE, step_format=stdout_step_format,
93+
metric_format=stdout_metric_format)
94+
]
9895
else:
9996
backends = []
10097

‎PyTorch/SpeechSynthesis/FastPitch/inference.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -310,12 +310,11 @@ def main():
310310
Path(args.output).mkdir(parents=False, exist_ok=True)
311311

312312
log_fpath = args.log_file or str(Path(args.output, 'nvlog_infer.json'))
313-
log_fpath = unique_log_fpath(log_fpath)
314-
DLLogger.init(backends=[JSONStreamBackend(Verbosity.DEFAULT, log_fpath),
315-
StdOutBackend(Verbosity.VERBOSE,
316-
metric_format=stdout_metric_format)
317-
])
318-
313+
DLLogger.init(backends=[
314+
JSONStreamBackend(Verbosity.DEFAULT, log_fpath, append=True),
315+
JSONStreamBackend(Verbosity.DEFAULT, unique_log_fpath(log_fpath)),
316+
StdOutBackend(Verbosity.VERBOSE, metric_format=stdout_metric_format)
317+
])
319318
init_inference_metadata(args.batch_size)
320319
[DLLogger.log("PARAMETER", {k: v}) for k, v in vars(args).items()]
321320

‎PyTorch/SpeechSynthesis/FastPitch/requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
git+https://github.com/NVIDIA/dllogger#egg=dllogger
21
inflect
32
librosa==0.9.0
43
matplotlib
54
numpy
65
pynvml==11.0.0
76
scipy
87
tensorboardX==2.0
8+
git+https://github.com/NVIDIA/dllogger@v1.0.0#egg=dllogger

‎PyTorch/SpeechSynthesis/HiFi-GAN/common/tb_dllogger.py‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,10 @@ def log_samples(self, step, sample_ind, audio, spec, rate):
8080

8181

8282
def unique_log_fpath(fpath):
83-
84-
if not Path(fpath).is_file():
85-
return fpath
86-
87-
# Avoid overwriting old logs
88-
saved = [re.search('\.(\d+)$', f) for f in glob.glob(f'{fpath}.*')]
89-
saved = [0] + [int(m.group(1)) for m in saved if m is not None]
90-
return f'{fpath}.{max(saved) + 1}'
83+
"""Have a unique log filename for every separate run"""
84+
log_num = max([0] + [int(re.search("\.(\d+)", Path(f).suffix).group(1))
85+
for f in glob.glob(f"{fpath}.*")])
86+
return f"{fpath}.{log_num + 1}"
9187

9288

9389
def stdout_step_format(step):

0 commit comments

Comments
 (0)