Skip to content

Commit c1b6fd7

Browse files
cpcloudclaude
cpcloud
and
claude
authored
Fix IPC memory pool leak in nightly repeated test runs (#1905)
* Fix IPC memory pool leak in nightly repeated test runs IPC test fixtures and test methods create DeviceMemoryResource and PinnedMemoryResource instances that are never closed. Under single-run CI this is harmless, but nightly CI runs tests with pytest-repeat --count=100, accumulating ~200+ leaked CUDA mempools and POSIX fd exports per test file until the process segfaults around iteration 83. Convert the ipc_memory_resource fixture from return to yield with explicit close() teardown, and close locally-created memory resources in test_errors, test_send_buffers, and test_workerpool after use. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Wrap IPC memory resource cleanup in try/finally Ensure MR.close() runs even when tests fail (timeout, assertion, queue error) so pools don't leak on the unhappy path either. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 7a9a248 commit c1b6fd7

4 files changed

Lines changed: 83 additions & 56 deletions

File tree

‎cuda_core/tests/conftest.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,8 @@ def ipc_memory_resource(request, ipc_device):
183183
mr = PinnedMemoryResource(options=options)
184184

185185
assert mr.is_ipc_enabled
186-
return mr
186+
yield mr
187+
mr.close()
187188

188189

189190
@pytest.fixture

‎cuda_core/tests/memory_ipc/test_errors.py‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,27 @@ def test_main(self, ipc_device, ipc_memory_resource):
2626
# from PARENT_ACTION.
2727
self.device = ipc_device
2828
self.mr = ipc_memory_resource
29+
self._extra_mrs = []
2930

30-
# Start a child process to generate error info.
31-
pipe = [multiprocessing.Queue() for _ in range(2)]
32-
process = multiprocessing.Process(target=self.child_main, args=(pipe, self.device, self.mr))
33-
process.start()
31+
try:
32+
# Start a child process to generate error info.
33+
pipe = [multiprocessing.Queue() for _ in range(2)]
34+
process = multiprocessing.Process(target=self.child_main, args=(pipe, self.device, self.mr))
35+
process.start()
3436

35-
# Interact.
36-
self.PARENT_ACTION(pipe[0])
37+
# Interact.
38+
self.PARENT_ACTION(pipe[0])
3739

38-
# Check the error.
39-
exc_type, exc_msg = pipe[1].get(timeout=CHILD_TIMEOUT_SEC)
40-
self.ASSERT(exc_type, exc_msg)
40+
# Check the error.
41+
exc_type, exc_msg = pipe[1].get(timeout=CHILD_TIMEOUT_SEC)
42+
self.ASSERT(exc_type, exc_msg)
4143

42-
# Wait for the child process.
43-
process.join(timeout=CHILD_TIMEOUT_SEC)
44-
assert process.exitcode == 0
44+
# Wait for the child process.
45+
process.join(timeout=CHILD_TIMEOUT_SEC)
46+
assert process.exitcode == 0
47+
finally:
48+
for mr in self._extra_mrs:
49+
mr.close()
4550

4651
def child_main(self, pipe, device, mr):
4752
"""Child process that pushes IPC errors to a shared pipe for testing."""
@@ -78,6 +83,7 @@ class TestImportWrongMR(ChildErrorHarness):
7883
def PARENT_ACTION(self, queue):
7984
options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
8085
mr2 = DeviceMemoryResource(self.device, options=options)
86+
self._extra_mrs.append(mr2)
8187
buffer = mr2.allocate(NBYTES)
8288
queue.put([self.mr, buffer.get_ipc_descriptor()]) # Note: mr does not own this buffer
8389

@@ -117,6 +123,7 @@ class TestDanglingBuffer(ChildErrorHarness):
117123
def PARENT_ACTION(self, queue):
118124
options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
119125
mr2 = DeviceMemoryResource(self.device, options=options)
126+
self._extra_mrs.append(mr2)
120127
self.buffer = mr2.allocate(NBYTES)
121128
buffer_s = pickle.dumps(self.buffer)
122129
queue.put(buffer_s) # Note: mr2 not sent

‎cuda_core/tests/memory_ipc/test_send_buffers.py‎

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,25 +26,29 @@ def test_main(self, ipc_device, nmrs):
2626
options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
2727
mrs = [DeviceMemoryResource(device, options=options) for _ in range(nmrs)]
2828

29-
# Allocate and fill memory.
30-
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
31-
pgen = PatternGen(device, NBYTES)
32-
for buffer in buffers:
33-
pgen.fill_buffer(buffer, seed=False)
34-
35-
# Start the child process.
36-
process = mp.Process(target=self.child_main, args=(device, buffers))
37-
process.start()
38-
39-
# Wait for the child process.
40-
process.join(timeout=CHILD_TIMEOUT_SEC)
41-
assert process.exitcode == 0
42-
43-
# Verify that the buffers were modified.
44-
pgen = PatternGen(device, NBYTES)
45-
for buffer in buffers:
46-
pgen.verify_buffer(buffer, seed=True)
47-
buffer.close()
29+
try:
30+
# Allocate and fill memory.
31+
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
32+
pgen = PatternGen(device, NBYTES)
33+
for buffer in buffers:
34+
pgen.fill_buffer(buffer, seed=False)
35+
36+
# Start the child process.
37+
process = mp.Process(target=self.child_main, args=(device, buffers))
38+
process.start()
39+
40+
# Wait for the child process.
41+
process.join(timeout=CHILD_TIMEOUT_SEC)
42+
assert process.exitcode == 0
43+
44+
# Verify that the buffers were modified.
45+
pgen = PatternGen(device, NBYTES)
46+
for buffer in buffers:
47+
pgen.verify_buffer(buffer, seed=True)
48+
buffer.close()
49+
finally:
50+
for mr in mrs:
51+
mr.close()
4852

4953
def child_main(self, device, buffers):
5054
device.set_current()

‎cuda_core/tests/memory_ipc/test_workerpool.py‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,20 @@ def test_main(self, ipc_device, nmrs):
3333
device = ipc_device
3434
options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
3535
mrs = [DeviceMemoryResource(device, options=options) for _ in range(nmrs)]
36-
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
3736

38-
with mp.Pool(NWORKERS) as pool:
39-
pool.map(self.process_buffer, buffers)
37+
try:
38+
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
4039

41-
pgen = PatternGen(device, NBYTES)
42-
for buffer in buffers:
43-
pgen.verify_buffer(buffer, seed=True)
44-
buffer.close()
40+
with mp.Pool(NWORKERS) as pool:
41+
pool.map(self.process_buffer, buffers)
42+
43+
pgen = PatternGen(device, NBYTES)
44+
for buffer in buffers:
45+
pgen.verify_buffer(buffer, seed=True)
46+
buffer.close()
47+
finally:
48+
for mr in mrs:
49+
mr.close()
4550

4651
def process_buffer(self, buffer):
4752
device = Device(buffer.memory_resource.device_id)
@@ -70,18 +75,23 @@ def test_main(self, ipc_device, nmrs):
7075
device = ipc_device
7176
options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
7277
mrs = [DeviceMemoryResource(device, options=options) for _ in range(nmrs)]
73-
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
7478

75-
with mp.Pool(NWORKERS, initializer=self.init_worker, initargs=(mrs,)) as pool:
76-
pool.starmap(
77-
self.process_buffer,
78-
[(mrs.index(buffer.memory_resource), buffer.get_ipc_descriptor()) for buffer in buffers],
79-
)
79+
try:
80+
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
8081

81-
pgen = PatternGen(device, NBYTES)
82-
for buffer in buffers:
83-
pgen.verify_buffer(buffer, seed=True)
84-
buffer.close()
82+
with mp.Pool(NWORKERS, initializer=self.init_worker, initargs=(mrs,)) as pool:
83+
pool.starmap(
84+
self.process_buffer,
85+
[(mrs.index(buffer.memory_resource), buffer.get_ipc_descriptor()) for buffer in buffers],
86+
)
87+
88+
pgen = PatternGen(device, NBYTES)
89+
for buffer in buffers:
90+
pgen.verify_buffer(buffer, seed=True)
91+
buffer.close()
92+
finally:
93+
for mr in mrs:
94+
mr.close()
8595

8696
def process_buffer(self, mr_idx, buffer_desc):
8797
mr = self.mrs[mr_idx]
@@ -115,15 +125,20 @@ def test_main(self, ipc_device, nmrs):
115125
device = ipc_device
116126
options = DeviceMemoryResourceOptions(max_size=POOL_SIZE, ipc_enabled=True)
117127
mrs = [DeviceMemoryResource(device, options=options) for _ in range(nmrs)]
118-
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
119128

120-
with mp.Pool(NWORKERS, initializer=self.init_worker, initargs=(mrs,)) as pool:
121-
pool.starmap(self.process_buffer, [(device, pickle.dumps(buffer)) for buffer in buffers])
129+
try:
130+
buffers = [mr.allocate(NBYTES) for mr, _ in zip(cycle(mrs), range(NTASKS))]
122131

123-
pgen = PatternGen(device, NBYTES)
124-
for buffer in buffers:
125-
pgen.verify_buffer(buffer, seed=True)
126-
buffer.close()
132+
with mp.Pool(NWORKERS, initializer=self.init_worker, initargs=(mrs,)) as pool:
133+
pool.starmap(self.process_buffer, [(device, pickle.dumps(buffer)) for buffer in buffers])
134+
135+
pgen = PatternGen(device, NBYTES)
136+
for buffer in buffers:
137+
pgen.verify_buffer(buffer, seed=True)
138+
buffer.close()
139+
finally:
140+
for mr in mrs:
141+
mr.close()
127142

128143
def process_buffer(self, device, buffer_s):
129144
device.set_current()

0 commit comments

Comments
 (0)