fix(docreader): kill timed-out converter process groups (#2990)

* fix(docreader): kill timed-out converter process groups

(cherry picked from commit 403ee6dd90ad6836f4f073f4f06839884133a7e4)

* fix(docreader): bound timeout reap so converter cleanup cannot hang

Unbounded communicate() after killpg waited for pipe EOF and could
block a DocReader worker if a descendant left the process group.

---------

Co-authored-by: wizardchen <wizardchen@tencent.com>
main
YixuanLieu 2026-09-23 12:02:53 +08:00 committed by GitHub
parent acd1a2a5a4
commit 06ae6bcf04
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 365 additions and 1 deletions

View File

@ -1,5 +1,6 @@
import logging
import os
import signal
import subprocess
import time
import uuid
@ -75,22 +76,93 @@ class SandboxExecutor:
if self.proxy:
logger.info(f"Using proxy: {self.proxy}")
is_posix = os.name == "posix"
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
# On POSIX, start a fresh group so a timed-out command can be
# terminated together with children it spawned (e.g. soffice.bin).
start_new_session=is_posix,
)
# On POSIX start_new_session makes the direct child the group leader, so
# retain this ID even if the direct child exits before cleanup starts.
process_group_id = process.pid if is_posix else None
try:
stdout, stderr = process.communicate(timeout=self.default_timeout)
return stdout, stderr, process.returncode
except subprocess.TimeoutExpired:
process.kill()
# Terminate the POSIX process group (or the direct child elsewhere),
# then reap with a bounded wait. Do not drain pipes here:
# communicate() waits for EOF and can hang if a descendant left
# the group (setsid) or otherwise kept a PIPE writer open.
try:
self._kill_timed_out_command(process, process_group_id)
finally:
self._reap_timed_out_process(process)
raise RuntimeError(
f"Command execution timeout after {self.default_timeout} seconds"
)
def _kill_timed_out_command(
self, process: subprocess.Popen, process_group_id: Optional[int]
) -> None:
if process_group_id is None:
try:
process.kill()
except OSError:
pass
return
self._signal_process_group(process_group_id, signal.SIGTERM)
grace_period = 0.5
grace_deadline = time.monotonic() + grace_period
try:
process.wait(timeout=grace_period)
except subprocess.TimeoutExpired:
pass
remaining_grace_period = grace_deadline - time.monotonic()
if remaining_grace_period > 0:
time.sleep(remaining_grace_period)
# The direct child can exit after SIGTERM while a descendant
# ignores it. Always finish the grace period with SIGKILL.
self._signal_process_group(process_group_id, signal.SIGKILL)
@staticmethod
def _signal_process_group(process_group_id: int, sig: int) -> None:
try:
os.killpg(process_group_id, sig)
except OSError:
pass
@staticmethod
def _reap_timed_out_process(
process: subprocess.Popen, timeout: float = 1.0
) -> None:
try:
process.wait(timeout=timeout)
except subprocess.TimeoutExpired:
try:
process.kill()
except OSError:
pass
try:
process.wait(timeout=timeout)
except subprocess.TimeoutExpired:
logger.warning(
"Timed-out command pid=%s did not exit after SIGKILL",
process.pid,
)
finally:
for pipe in (process.stdout, process.stderr):
if pipe is not None:
try:
pipe.close()
except OSError:
pass
logger = logging.getLogger(__name__)

View File

@ -0,0 +1,292 @@
import os
import shlex
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import ANY, Mock, call, patch
from docreader.parser.doc_parser import SandboxExecutor
def _mock_process(pid=4321):
process = Mock(pid=pid)
process.stdout = Mock()
process.stderr = Mock()
return process
@unittest.skipUnless(os.name == "posix", "process-group cleanup is POSIX-specific")
class TestSandboxExecutorProcessGroup(unittest.TestCase):
def test_timeout_kills_whole_process_group(self):
"""A timed-out command must terminate its parent and descendants."""
with tempfile.TemporaryDirectory() as temp_dir:
child_pid_file = Path(temp_dir) / "child.pid"
command = [
"sh",
"-c",
f"sleep 30 & echo $! > {shlex.quote(str(child_pid_file))}; wait",
]
executor = SandboxExecutor(default_timeout=1)
with self.assertRaises(RuntimeError):
executor.execute_in_sandbox(command)
child_pid = int(child_pid_file.read_text().strip())
deadline = time.monotonic() + 1
while True:
try:
os.kill(child_pid, 0)
except ProcessLookupError:
break
if time.monotonic() >= deadline:
self.fail("timed-out command left a child process running")
time.sleep(0.01)
def test_timeout_returns_when_descendant_leaves_process_group(self):
"""setsid descendants can survive killpg; cleanup must still return."""
with tempfile.TemporaryDirectory() as temp_dir:
child_pid_file = Path(temp_dir) / "child.pid"
command = [
sys.executable,
"-c",
(
"import os, time\n"
f"pid_file = {str(child_pid_file)!r}\n"
"if os.fork() == 0:\n"
" os.setsid()\n"
" with open(pid_file, 'w', encoding='utf-8') as handle:\n"
" handle.write(str(os.getpid()))\n"
" time.sleep(60)\n"
"time.sleep(30)\n"
),
]
executor = SandboxExecutor(default_timeout=1)
started = time.monotonic()
with self.assertRaises(RuntimeError):
executor.execute_in_sandbox(command)
self.assertLess(
time.monotonic() - started,
4,
"timeout cleanup hung waiting on an escaped descendant",
)
deadline = time.monotonic() + 2
while not child_pid_file.exists() or not child_pid_file.stat().st_size:
if time.monotonic() >= deadline:
self.fail("escaped child did not write its pid file")
time.sleep(0.01)
child_pid = int(child_pid_file.read_text().strip())
try:
os.kill(child_pid, 0)
except ProcessLookupError:
self.fail("expected setsid descendant to survive process-group kill")
try:
os.kill(child_pid, signal.SIGKILL)
except ProcessLookupError:
pass
class TestSandboxExecutorMock(unittest.TestCase):
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
def test_posix_timeout_terminates_group_and_reaps_process(self):
process = _mock_process()
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
process.wait.side_effect = [
subprocess.TimeoutExpired(["soffice"], 0.5),
0,
]
with (
patch("docreader.parser.doc_parser.os.name", "posix"),
patch(
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
) as popen,
patch("docreader.parser.doc_parser.os.killpg") as killpg,
patch(
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
),
patch("docreader.parser.doc_parser.time.sleep") as sleep,
):
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
popen.assert_called_once_with(
["soffice"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=ANY,
start_new_session=True,
)
killpg.assert_has_calls(
[
call(4321, signal.SIGTERM),
call(4321, signal.SIGKILL),
]
)
sleep.assert_not_called()
self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])
self.assertEqual(
process.wait.call_args_list,
[call(timeout=0.5), call(timeout=1.0)],
)
process.stdout.close.assert_called_once_with()
process.stderr.close.assert_called_once_with()
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
def test_posix_timeout_waits_full_grace_period_when_child_exits_early(self):
process = _mock_process()
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
process.wait.side_effect = [0, 0]
with (
patch("docreader.parser.doc_parser.os.name", "posix"),
patch(
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
),
patch("docreader.parser.doc_parser.os.killpg") as killpg,
patch(
"docreader.parser.doc_parser.time.monotonic", side_effect=[10, 10.2]
),
patch("docreader.parser.doc_parser.time.sleep") as sleep,
):
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
self.assertAlmostEqual(sleep.call_args.args[0], 0.3)
killpg.assert_has_calls(
[
call(4321, signal.SIGTERM),
call(4321, signal.SIGKILL),
]
)
self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])
self.assertEqual(
process.wait.call_args_list,
[call(timeout=0.5), call(timeout=1.0)],
)
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
def test_posix_timeout_reaps_when_group_already_exited(self):
process = _mock_process()
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
process.wait.side_effect = [
subprocess.TimeoutExpired(["soffice"], 0.5),
0,
]
with (
patch("docreader.parser.doc_parser.os.name", "posix"),
patch(
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
),
patch(
"docreader.parser.doc_parser.os.killpg",
side_effect=ProcessLookupError,
) as killpg,
patch(
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
),
patch("docreader.parser.doc_parser.time.sleep") as sleep,
):
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
self.assertEqual(killpg.call_count, 2)
sleep.assert_not_called()
self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0))
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
def test_posix_timeout_reaps_when_killpg_raises_permission_error(self):
process = _mock_process()
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
process.wait.side_effect = [
subprocess.TimeoutExpired(["soffice"], 0.5),
0,
]
with (
patch("docreader.parser.doc_parser.os.name", "posix"),
patch(
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
),
patch(
"docreader.parser.doc_parser.os.killpg",
side_effect=PermissionError,
) as killpg,
patch(
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
),
patch("docreader.parser.doc_parser.time.sleep") as sleep,
):
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
self.assertEqual(killpg.call_count, 2)
sleep.assert_not_called()
self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0))
process.stdout.close.assert_called_once_with()
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
def test_posix_timeout_returns_when_final_wait_times_out(self):
process = _mock_process()
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
process.wait.side_effect = [
subprocess.TimeoutExpired(["soffice"], 0.5),
subprocess.TimeoutExpired(["soffice"], 1.0),
subprocess.TimeoutExpired(["soffice"], 1.0),
]
with (
patch("docreader.parser.doc_parser.os.name", "posix"),
patch(
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
),
patch("docreader.parser.doc_parser.os.killpg"),
patch(
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
),
patch("docreader.parser.doc_parser.time.sleep"),
):
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
self.assertEqual(process.kill.call_count, 1)
process.stdout.close.assert_called_once_with()
process.stderr.close.assert_called_once_with()
def test_non_posix_timeout_kills_direct_child_and_reaps_process(self):
process = _mock_process()
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
process.wait.return_value = 0
with (
patch("docreader.parser.doc_parser.os.name", "nt"),
patch(
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
) as popen,
patch("docreader.parser.doc_parser.os.killpg", create=True) as killpg,
):
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
popen.assert_called_once_with(
["soffice"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=ANY,
start_new_session=False,
)
process.kill.assert_called_once_with()
killpg.assert_not_called()
self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])
self.assertEqual(process.wait.call_args_list, [call(timeout=1.0)])
process.stdout.close.assert_called_once_with()
process.stderr.close.assert_called_once_with()
if __name__ == "__main__":
unittest.main()