From 06ae6bcf04e5ce3543c050383f7af8a033abf317 Mon Sep 17 00:00:00 2001 From: YixuanLieu <52775926+baka-world@users.noreply.github.com> Date: Wed, 23 Sep 2026 12:02:53 +0800 Subject: [PATCH] fix(docreader): kill timed-out converter process groups (#2990) * fix(docreader): kill timed-out converter process groups (cherry picked from commit 403ee6dd90ad6836f4f073f4f06839884133a7e4) * fix(docreader): bound timeout reap so converter cleanup cannot hang Unbounded communicate() after killpg waited for pipe EOF and could block a DocReader worker if a descendant left the process group. --------- Co-authored-by: wizardchen --- docreader/parser/doc_parser.py | 74 +++++- docreader/tests/test_sandbox_executor.py | 292 +++++++++++++++++++++++ 2 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 docreader/tests/test_sandbox_executor.py diff --git a/docreader/parser/doc_parser.py b/docreader/parser/doc_parser.py index 8c0882a3..013aa3ee 100644 --- a/docreader/parser/doc_parser.py +++ b/docreader/parser/doc_parser.py @@ -1,5 +1,6 @@ import logging import os +import signal import subprocess import time import uuid @@ -75,22 +76,93 @@ class SandboxExecutor: if self.proxy: logger.info(f"Using proxy: {self.proxy}") + is_posix = os.name == "posix" process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, + # On POSIX, start a fresh group so a timed-out command can be + # terminated together with children it spawned (e.g. soffice.bin). + start_new_session=is_posix, ) + # On POSIX start_new_session makes the direct child the group leader, so + # retain this ID even if the direct child exits before cleanup starts. + process_group_id = process.pid if is_posix else None try: stdout, stderr = process.communicate(timeout=self.default_timeout) return stdout, stderr, process.returncode except subprocess.TimeoutExpired: - process.kill() + # Terminate the POSIX process group (or the direct child elsewhere), + # then reap with a bounded wait. Do not drain pipes here: + # communicate() waits for EOF and can hang if a descendant left + # the group (setsid) or otherwise kept a PIPE writer open. + try: + self._kill_timed_out_command(process, process_group_id) + finally: + self._reap_timed_out_process(process) raise RuntimeError( f"Command execution timeout after {self.default_timeout} seconds" ) + def _kill_timed_out_command( + self, process: subprocess.Popen, process_group_id: Optional[int] + ) -> None: + if process_group_id is None: + try: + process.kill() + except OSError: + pass + return + + self._signal_process_group(process_group_id, signal.SIGTERM) + grace_period = 0.5 + grace_deadline = time.monotonic() + grace_period + try: + process.wait(timeout=grace_period) + except subprocess.TimeoutExpired: + pass + remaining_grace_period = grace_deadline - time.monotonic() + if remaining_grace_period > 0: + time.sleep(remaining_grace_period) + # The direct child can exit after SIGTERM while a descendant + # ignores it. Always finish the grace period with SIGKILL. + self._signal_process_group(process_group_id, signal.SIGKILL) + + @staticmethod + def _signal_process_group(process_group_id: int, sig: int) -> None: + try: + os.killpg(process_group_id, sig) + except OSError: + pass + + @staticmethod + def _reap_timed_out_process( + process: subprocess.Popen, timeout: float = 1.0 + ) -> None: + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + try: + process.kill() + except OSError: + pass + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning( + "Timed-out command pid=%s did not exit after SIGKILL", + process.pid, + ) + finally: + for pipe in (process.stdout, process.stderr): + if pipe is not None: + try: + pipe.close() + except OSError: + pass + logger = logging.getLogger(__name__) diff --git a/docreader/tests/test_sandbox_executor.py b/docreader/tests/test_sandbox_executor.py new file mode 100644 index 00000000..6347785c --- /dev/null +++ b/docreader/tests/test_sandbox_executor.py @@ -0,0 +1,292 @@ +import os +import shlex +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import ANY, Mock, call, patch + +from docreader.parser.doc_parser import SandboxExecutor + + +def _mock_process(pid=4321): + process = Mock(pid=pid) + process.stdout = Mock() + process.stderr = Mock() + return process + + +@unittest.skipUnless(os.name == "posix", "process-group cleanup is POSIX-specific") +class TestSandboxExecutorProcessGroup(unittest.TestCase): + def test_timeout_kills_whole_process_group(self): + """A timed-out command must terminate its parent and descendants.""" + with tempfile.TemporaryDirectory() as temp_dir: + child_pid_file = Path(temp_dir) / "child.pid" + command = [ + "sh", + "-c", + f"sleep 30 & echo $! > {shlex.quote(str(child_pid_file))}; wait", + ] + executor = SandboxExecutor(default_timeout=1) + + with self.assertRaises(RuntimeError): + executor.execute_in_sandbox(command) + + child_pid = int(child_pid_file.read_text().strip()) + deadline = time.monotonic() + 1 + while True: + try: + os.kill(child_pid, 0) + except ProcessLookupError: + break + if time.monotonic() >= deadline: + self.fail("timed-out command left a child process running") + time.sleep(0.01) + + def test_timeout_returns_when_descendant_leaves_process_group(self): + """setsid descendants can survive killpg; cleanup must still return.""" + with tempfile.TemporaryDirectory() as temp_dir: + child_pid_file = Path(temp_dir) / "child.pid" + command = [ + sys.executable, + "-c", + ( + "import os, time\n" + f"pid_file = {str(child_pid_file)!r}\n" + "if os.fork() == 0:\n" + " os.setsid()\n" + " with open(pid_file, 'w', encoding='utf-8') as handle:\n" + " handle.write(str(os.getpid()))\n" + " time.sleep(60)\n" + "time.sleep(30)\n" + ), + ] + executor = SandboxExecutor(default_timeout=1) + started = time.monotonic() + with self.assertRaises(RuntimeError): + executor.execute_in_sandbox(command) + self.assertLess( + time.monotonic() - started, + 4, + "timeout cleanup hung waiting on an escaped descendant", + ) + + deadline = time.monotonic() + 2 + while not child_pid_file.exists() or not child_pid_file.stat().st_size: + if time.monotonic() >= deadline: + self.fail("escaped child did not write its pid file") + time.sleep(0.01) + child_pid = int(child_pid_file.read_text().strip()) + try: + os.kill(child_pid, 0) + except ProcessLookupError: + self.fail("expected setsid descendant to survive process-group kill") + try: + os.kill(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +class TestSandboxExecutorMock(unittest.TestCase): + @unittest.skipUnless(os.name == "posix", "POSIX signals are required") + def test_posix_timeout_terminates_group_and_reaps_process(self): + process = _mock_process() + process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)] + process.wait.side_effect = [ + subprocess.TimeoutExpired(["soffice"], 0.5), + 0, + ] + + with ( + patch("docreader.parser.doc_parser.os.name", "posix"), + patch( + "docreader.parser.doc_parser.subprocess.Popen", return_value=process + ) as popen, + patch("docreader.parser.doc_parser.os.killpg") as killpg, + patch( + "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5] + ), + patch("docreader.parser.doc_parser.time.sleep") as sleep, + ): + with self.assertRaisesRegex(RuntimeError, "timeout after 1"): + SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"]) + + popen.assert_called_once_with( + ["soffice"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=ANY, + start_new_session=True, + ) + killpg.assert_has_calls( + [ + call(4321, signal.SIGTERM), + call(4321, signal.SIGKILL), + ] + ) + sleep.assert_not_called() + self.assertEqual(process.communicate.call_args_list, [call(timeout=1)]) + self.assertEqual( + process.wait.call_args_list, + [call(timeout=0.5), call(timeout=1.0)], + ) + process.stdout.close.assert_called_once_with() + process.stderr.close.assert_called_once_with() + + @unittest.skipUnless(os.name == "posix", "POSIX signals are required") + def test_posix_timeout_waits_full_grace_period_when_child_exits_early(self): + process = _mock_process() + process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)] + process.wait.side_effect = [0, 0] + + with ( + patch("docreader.parser.doc_parser.os.name", "posix"), + patch( + "docreader.parser.doc_parser.subprocess.Popen", return_value=process + ), + patch("docreader.parser.doc_parser.os.killpg") as killpg, + patch( + "docreader.parser.doc_parser.time.monotonic", side_effect=[10, 10.2] + ), + patch("docreader.parser.doc_parser.time.sleep") as sleep, + ): + with self.assertRaisesRegex(RuntimeError, "timeout after 1"): + SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"]) + + self.assertAlmostEqual(sleep.call_args.args[0], 0.3) + killpg.assert_has_calls( + [ + call(4321, signal.SIGTERM), + call(4321, signal.SIGKILL), + ] + ) + self.assertEqual(process.communicate.call_args_list, [call(timeout=1)]) + self.assertEqual( + process.wait.call_args_list, + [call(timeout=0.5), call(timeout=1.0)], + ) + + @unittest.skipUnless(os.name == "posix", "POSIX signals are required") + def test_posix_timeout_reaps_when_group_already_exited(self): + process = _mock_process() + process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)] + process.wait.side_effect = [ + subprocess.TimeoutExpired(["soffice"], 0.5), + 0, + ] + + with ( + patch("docreader.parser.doc_parser.os.name", "posix"), + patch( + "docreader.parser.doc_parser.subprocess.Popen", return_value=process + ), + patch( + "docreader.parser.doc_parser.os.killpg", + side_effect=ProcessLookupError, + ) as killpg, + patch( + "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5] + ), + patch("docreader.parser.doc_parser.time.sleep") as sleep, + ): + with self.assertRaisesRegex(RuntimeError, "timeout after 1"): + SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"]) + + self.assertEqual(killpg.call_count, 2) + sleep.assert_not_called() + self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0)) + + @unittest.skipUnless(os.name == "posix", "POSIX signals are required") + def test_posix_timeout_reaps_when_killpg_raises_permission_error(self): + process = _mock_process() + process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)] + process.wait.side_effect = [ + subprocess.TimeoutExpired(["soffice"], 0.5), + 0, + ] + + with ( + patch("docreader.parser.doc_parser.os.name", "posix"), + patch( + "docreader.parser.doc_parser.subprocess.Popen", return_value=process + ), + patch( + "docreader.parser.doc_parser.os.killpg", + side_effect=PermissionError, + ) as killpg, + patch( + "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5] + ), + patch("docreader.parser.doc_parser.time.sleep") as sleep, + ): + with self.assertRaisesRegex(RuntimeError, "timeout after 1"): + SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"]) + + self.assertEqual(killpg.call_count, 2) + sleep.assert_not_called() + self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0)) + process.stdout.close.assert_called_once_with() + + @unittest.skipUnless(os.name == "posix", "POSIX signals are required") + def test_posix_timeout_returns_when_final_wait_times_out(self): + process = _mock_process() + process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)] + process.wait.side_effect = [ + subprocess.TimeoutExpired(["soffice"], 0.5), + subprocess.TimeoutExpired(["soffice"], 1.0), + subprocess.TimeoutExpired(["soffice"], 1.0), + ] + + with ( + patch("docreader.parser.doc_parser.os.name", "posix"), + patch( + "docreader.parser.doc_parser.subprocess.Popen", return_value=process + ), + patch("docreader.parser.doc_parser.os.killpg"), + patch( + "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5] + ), + patch("docreader.parser.doc_parser.time.sleep"), + ): + with self.assertRaisesRegex(RuntimeError, "timeout after 1"): + SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"]) + + self.assertEqual(process.kill.call_count, 1) + process.stdout.close.assert_called_once_with() + process.stderr.close.assert_called_once_with() + + def test_non_posix_timeout_kills_direct_child_and_reaps_process(self): + process = _mock_process() + process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)] + process.wait.return_value = 0 + + with ( + patch("docreader.parser.doc_parser.os.name", "nt"), + patch( + "docreader.parser.doc_parser.subprocess.Popen", return_value=process + ) as popen, + patch("docreader.parser.doc_parser.os.killpg", create=True) as killpg, + ): + with self.assertRaisesRegex(RuntimeError, "timeout after 1"): + SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"]) + + popen.assert_called_once_with( + ["soffice"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=ANY, + start_new_session=False, + ) + process.kill.assert_called_once_with() + killpg.assert_not_called() + self.assertEqual(process.communicate.call_args_list, [call(timeout=1)]) + self.assertEqual(process.wait.call_args_list, [call(timeout=1.0)]) + process.stdout.close.assert_called_once_with() + process.stderr.close.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main()