fix(docreader): kill timed-out converter process groups (#2990)
* fix(docreader): kill timed-out converter process groups (cherry picked from commit 403ee6dd90ad6836f4f073f4f06839884133a7e4) * fix(docreader): bound timeout reap so converter cleanup cannot hang Unbounded communicate() after killpg waited for pipe EOF and could block a DocReader worker if a descendant left the process group. --------- Co-authored-by: wizardchen <wizardchen@tencent.com>main
parent
acd1a2a5a4
commit
06ae6bcf04
|
|
@ -1,5 +1,6 @@
|
|||
import logging
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
|
@ -75,22 +76,93 @@ class SandboxExecutor:
|
|||
if self.proxy:
|
||||
logger.info(f"Using proxy: {self.proxy}")
|
||||
|
||||
is_posix = os.name == "posix"
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=env,
|
||||
# On POSIX, start a fresh group so a timed-out command can be
|
||||
# terminated together with children it spawned (e.g. soffice.bin).
|
||||
start_new_session=is_posix,
|
||||
)
|
||||
# On POSIX start_new_session makes the direct child the group leader, so
|
||||
# retain this ID even if the direct child exits before cleanup starts.
|
||||
process_group_id = process.pid if is_posix else None
|
||||
|
||||
try:
|
||||
stdout, stderr = process.communicate(timeout=self.default_timeout)
|
||||
return stdout, stderr, process.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
# Terminate the POSIX process group (or the direct child elsewhere),
|
||||
# then reap with a bounded wait. Do not drain pipes here:
|
||||
# communicate() waits for EOF and can hang if a descendant left
|
||||
# the group (setsid) or otherwise kept a PIPE writer open.
|
||||
try:
|
||||
self._kill_timed_out_command(process, process_group_id)
|
||||
finally:
|
||||
self._reap_timed_out_process(process)
|
||||
raise RuntimeError(
|
||||
f"Command execution timeout after {self.default_timeout} seconds"
|
||||
)
|
||||
|
||||
def _kill_timed_out_command(
|
||||
self, process: subprocess.Popen, process_group_id: Optional[int]
|
||||
) -> None:
|
||||
if process_group_id is None:
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
|
||||
self._signal_process_group(process_group_id, signal.SIGTERM)
|
||||
grace_period = 0.5
|
||||
grace_deadline = time.monotonic() + grace_period
|
||||
try:
|
||||
process.wait(timeout=grace_period)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
remaining_grace_period = grace_deadline - time.monotonic()
|
||||
if remaining_grace_period > 0:
|
||||
time.sleep(remaining_grace_period)
|
||||
# The direct child can exit after SIGTERM while a descendant
|
||||
# ignores it. Always finish the grace period with SIGKILL.
|
||||
self._signal_process_group(process_group_id, signal.SIGKILL)
|
||||
|
||||
@staticmethod
|
||||
def _signal_process_group(process_group_id: int, sig: int) -> None:
|
||||
try:
|
||||
os.killpg(process_group_id, sig)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _reap_timed_out_process(
|
||||
process: subprocess.Popen, timeout: float = 1.0
|
||||
) -> None:
|
||||
try:
|
||||
process.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
process.kill()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning(
|
||||
"Timed-out command pid=%s did not exit after SIGKILL",
|
||||
process.pid,
|
||||
)
|
||||
finally:
|
||||
for pipe in (process.stdout, process.stderr):
|
||||
if pipe is not None:
|
||||
try:
|
||||
pipe.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,292 @@
|
|||
import os
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import ANY, Mock, call, patch
|
||||
|
||||
from docreader.parser.doc_parser import SandboxExecutor
|
||||
|
||||
|
||||
def _mock_process(pid=4321):
|
||||
process = Mock(pid=pid)
|
||||
process.stdout = Mock()
|
||||
process.stderr = Mock()
|
||||
return process
|
||||
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "process-group cleanup is POSIX-specific")
|
||||
class TestSandboxExecutorProcessGroup(unittest.TestCase):
|
||||
def test_timeout_kills_whole_process_group(self):
|
||||
"""A timed-out command must terminate its parent and descendants."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
child_pid_file = Path(temp_dir) / "child.pid"
|
||||
command = [
|
||||
"sh",
|
||||
"-c",
|
||||
f"sleep 30 & echo $! > {shlex.quote(str(child_pid_file))}; wait",
|
||||
]
|
||||
executor = SandboxExecutor(default_timeout=1)
|
||||
|
||||
with self.assertRaises(RuntimeError):
|
||||
executor.execute_in_sandbox(command)
|
||||
|
||||
child_pid = int(child_pid_file.read_text().strip())
|
||||
deadline = time.monotonic() + 1
|
||||
while True:
|
||||
try:
|
||||
os.kill(child_pid, 0)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
self.fail("timed-out command left a child process running")
|
||||
time.sleep(0.01)
|
||||
|
||||
def test_timeout_returns_when_descendant_leaves_process_group(self):
|
||||
"""setsid descendants can survive killpg; cleanup must still return."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
child_pid_file = Path(temp_dir) / "child.pid"
|
||||
command = [
|
||||
sys.executable,
|
||||
"-c",
|
||||
(
|
||||
"import os, time\n"
|
||||
f"pid_file = {str(child_pid_file)!r}\n"
|
||||
"if os.fork() == 0:\n"
|
||||
" os.setsid()\n"
|
||||
" with open(pid_file, 'w', encoding='utf-8') as handle:\n"
|
||||
" handle.write(str(os.getpid()))\n"
|
||||
" time.sleep(60)\n"
|
||||
"time.sleep(30)\n"
|
||||
),
|
||||
]
|
||||
executor = SandboxExecutor(default_timeout=1)
|
||||
started = time.monotonic()
|
||||
with self.assertRaises(RuntimeError):
|
||||
executor.execute_in_sandbox(command)
|
||||
self.assertLess(
|
||||
time.monotonic() - started,
|
||||
4,
|
||||
"timeout cleanup hung waiting on an escaped descendant",
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 2
|
||||
while not child_pid_file.exists() or not child_pid_file.stat().st_size:
|
||||
if time.monotonic() >= deadline:
|
||||
self.fail("escaped child did not write its pid file")
|
||||
time.sleep(0.01)
|
||||
child_pid = int(child_pid_file.read_text().strip())
|
||||
try:
|
||||
os.kill(child_pid, 0)
|
||||
except ProcessLookupError:
|
||||
self.fail("expected setsid descendant to survive process-group kill")
|
||||
try:
|
||||
os.kill(child_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
class TestSandboxExecutorMock(unittest.TestCase):
|
||||
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
|
||||
def test_posix_timeout_terminates_group_and_reaps_process(self):
|
||||
process = _mock_process()
|
||||
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
|
||||
process.wait.side_effect = [
|
||||
subprocess.TimeoutExpired(["soffice"], 0.5),
|
||||
0,
|
||||
]
|
||||
|
||||
with (
|
||||
patch("docreader.parser.doc_parser.os.name", "posix"),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
|
||||
) as popen,
|
||||
patch("docreader.parser.doc_parser.os.killpg") as killpg,
|
||||
patch(
|
||||
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
|
||||
),
|
||||
patch("docreader.parser.doc_parser.time.sleep") as sleep,
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
|
||||
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
|
||||
|
||||
popen.assert_called_once_with(
|
||||
["soffice"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=ANY,
|
||||
start_new_session=True,
|
||||
)
|
||||
killpg.assert_has_calls(
|
||||
[
|
||||
call(4321, signal.SIGTERM),
|
||||
call(4321, signal.SIGKILL),
|
||||
]
|
||||
)
|
||||
sleep.assert_not_called()
|
||||
self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])
|
||||
self.assertEqual(
|
||||
process.wait.call_args_list,
|
||||
[call(timeout=0.5), call(timeout=1.0)],
|
||||
)
|
||||
process.stdout.close.assert_called_once_with()
|
||||
process.stderr.close.assert_called_once_with()
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
|
||||
def test_posix_timeout_waits_full_grace_period_when_child_exits_early(self):
|
||||
process = _mock_process()
|
||||
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
|
||||
process.wait.side_effect = [0, 0]
|
||||
|
||||
with (
|
||||
patch("docreader.parser.doc_parser.os.name", "posix"),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
|
||||
),
|
||||
patch("docreader.parser.doc_parser.os.killpg") as killpg,
|
||||
patch(
|
||||
"docreader.parser.doc_parser.time.monotonic", side_effect=[10, 10.2]
|
||||
),
|
||||
patch("docreader.parser.doc_parser.time.sleep") as sleep,
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
|
||||
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
|
||||
|
||||
self.assertAlmostEqual(sleep.call_args.args[0], 0.3)
|
||||
killpg.assert_has_calls(
|
||||
[
|
||||
call(4321, signal.SIGTERM),
|
||||
call(4321, signal.SIGKILL),
|
||||
]
|
||||
)
|
||||
self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])
|
||||
self.assertEqual(
|
||||
process.wait.call_args_list,
|
||||
[call(timeout=0.5), call(timeout=1.0)],
|
||||
)
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
|
||||
def test_posix_timeout_reaps_when_group_already_exited(self):
|
||||
process = _mock_process()
|
||||
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
|
||||
process.wait.side_effect = [
|
||||
subprocess.TimeoutExpired(["soffice"], 0.5),
|
||||
0,
|
||||
]
|
||||
|
||||
with (
|
||||
patch("docreader.parser.doc_parser.os.name", "posix"),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
|
||||
),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.os.killpg",
|
||||
side_effect=ProcessLookupError,
|
||||
) as killpg,
|
||||
patch(
|
||||
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
|
||||
),
|
||||
patch("docreader.parser.doc_parser.time.sleep") as sleep,
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
|
||||
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
|
||||
|
||||
self.assertEqual(killpg.call_count, 2)
|
||||
sleep.assert_not_called()
|
||||
self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0))
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
|
||||
def test_posix_timeout_reaps_when_killpg_raises_permission_error(self):
|
||||
process = _mock_process()
|
||||
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
|
||||
process.wait.side_effect = [
|
||||
subprocess.TimeoutExpired(["soffice"], 0.5),
|
||||
0,
|
||||
]
|
||||
|
||||
with (
|
||||
patch("docreader.parser.doc_parser.os.name", "posix"),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
|
||||
),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.os.killpg",
|
||||
side_effect=PermissionError,
|
||||
) as killpg,
|
||||
patch(
|
||||
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
|
||||
),
|
||||
patch("docreader.parser.doc_parser.time.sleep") as sleep,
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
|
||||
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
|
||||
|
||||
self.assertEqual(killpg.call_count, 2)
|
||||
sleep.assert_not_called()
|
||||
self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0))
|
||||
process.stdout.close.assert_called_once_with()
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "POSIX signals are required")
|
||||
def test_posix_timeout_returns_when_final_wait_times_out(self):
|
||||
process = _mock_process()
|
||||
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
|
||||
process.wait.side_effect = [
|
||||
subprocess.TimeoutExpired(["soffice"], 0.5),
|
||||
subprocess.TimeoutExpired(["soffice"], 1.0),
|
||||
subprocess.TimeoutExpired(["soffice"], 1.0),
|
||||
]
|
||||
|
||||
with (
|
||||
patch("docreader.parser.doc_parser.os.name", "posix"),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
|
||||
),
|
||||
patch("docreader.parser.doc_parser.os.killpg"),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]
|
||||
),
|
||||
patch("docreader.parser.doc_parser.time.sleep"),
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
|
||||
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
|
||||
|
||||
self.assertEqual(process.kill.call_count, 1)
|
||||
process.stdout.close.assert_called_once_with()
|
||||
process.stderr.close.assert_called_once_with()
|
||||
|
||||
def test_non_posix_timeout_kills_direct_child_and_reaps_process(self):
|
||||
process = _mock_process()
|
||||
process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]
|
||||
process.wait.return_value = 0
|
||||
|
||||
with (
|
||||
patch("docreader.parser.doc_parser.os.name", "nt"),
|
||||
patch(
|
||||
"docreader.parser.doc_parser.subprocess.Popen", return_value=process
|
||||
) as popen,
|
||||
patch("docreader.parser.doc_parser.os.killpg", create=True) as killpg,
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "timeout after 1"):
|
||||
SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])
|
||||
|
||||
popen.assert_called_once_with(
|
||||
["soffice"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=ANY,
|
||||
start_new_session=False,
|
||||
)
|
||||
process.kill.assert_called_once_with()
|
||||
killpg.assert_not_called()
|
||||
self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])
|
||||
self.assertEqual(process.wait.call_args_list, [call(timeout=1.0)])
|
||||
process.stdout.close.assert_called_once_with()
|
||||
process.stderr.close.assert_called_once_with()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue