From a3e7d0148ed1d2d261667e487e028ae8e9595ba9 Mon Sep 17 00:00:00 2001 From: Bengang Yuan Date: Thu, 29 Oct 2026 09:04:23 +0000 Subject: [PATCH 1/2] CA-432805: Exit fuse without flushing channels Stdlib.exit calls flush_all which does a blocking write() on every unflushed out_channel. One stale channel whose fd was reused by an unread socket blocks it forever. Use Unix._exit instead which doesn't call flush_all. Signed-off-by: Bengang Yuan --- ocaml/database/db_connections.ml | 6 ++++-- ocaml/xapi/xapi_fuse.ml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ocaml/database/db_connections.ml b/ocaml/database/db_connections.ml index aa7152eaa7d..714bf3e6bd6 100644 --- a/ocaml/database/db_connections.ml +++ b/ocaml/database/db_connections.ml @@ -87,6 +87,8 @@ let pre_exit_hook () = ensure that they flush a consistent snapshot. Backends must also ensure that they do not hold the global db_lock whilst they are writing to non-local storage. + Exits use Unix._exit: the database is already flushed, and the flush_all in + Stdlib.exit can block forever on a stale channel while Db_lock is held. *) let flush_dirty_and_maybe_exit dbconn exit_spec = Db_conn_store.with_db_conn_lock dbconn (fun () -> @@ -101,7 +103,7 @@ let flush_dirty_and_maybe_exit dbconn exit_spec = if refcount = 0 then ( debug "refcount is 0; exiting" ; pre_exit_hook () ; - exit 0 + Unix._exit 0 ) else debug "refcount is %d; not exiting" refcount ) ; @@ -111,7 +113,7 @@ let flush_dirty_and_maybe_exit dbconn exit_spec = | None -> () | Some ret_code -> - pre_exit_hook () ; exit ret_code + pre_exit_hook () ; Unix._exit ret_code ) ; was_anything_flushed ) diff --git a/ocaml/xapi/xapi_fuse.ml b/ocaml/xapi/xapi_fuse.ml index 49f6f19c1eb..34b28004739 100644 --- a/ocaml/xapi/xapi_fuse.ml +++ b/ocaml/xapi/xapi_fuse.ml @@ -82,7 +82,7 @@ let light_fuse_and_run ?(fuse_length = !Constants.fuse_time) () = "Caught an exception flushing database (perhaps it hasn't been \ initialised yet): %s; restarting immediately" (ExnHelper.string_of_exn e) ; - exit Xapi_globs.restart_return_code + Unix._exit Xapi_globs.restart_return_code ) () ) From eb672a2fddbad43142adbc92a421e604a27599e4 Mon Sep 17 00:00:00 2001 From: Bengang Yuan Date: Thu, 29 Oct 2026 09:04:24 +0000 Subject: [PATCH 2/2] CA-432805: Force exit when the fuse hangs Start a timer in the fuse process to force exiting the process if it is still running fuse_exit_grace seconds (default 60) after the fuse delay. Signed-off-by: Bengang Yuan --- ocaml/xapi-consts/constants.ml | 3 +++ ocaml/xapi/xapi_fuse.ml | 14 ++++++++++++++ ocaml/xapi/xapi_globs.ml | 1 + scripts/xapi.conf | 4 ++++ 4 files changed, 22 insertions(+) diff --git a/ocaml/xapi-consts/constants.ml b/ocaml/xapi-consts/constants.ml index 7089079a1a8..839b66ce92c 100644 --- a/ocaml/xapi-consts/constants.ml +++ b/ocaml/xapi-consts/constants.ml @@ -327,6 +327,9 @@ let cli_report_ignored_parameters = ref Off (* the time taken to wait before restarting in a different mode for pool eject/join operations *) let fuse_time = ref 10. +(* the time the fuse allows for flush and exit before forcing the process out *) +let fuse_exit_grace = ref 60. + (* Names of storage parameters *) let _sm_vm_hint = "vmhint" diff --git a/ocaml/xapi/xapi_fuse.ml b/ocaml/xapi/xapi_fuse.ml index 34b28004739..2374ddf5a5d 100644 --- a/ocaml/xapi/xapi_fuse.ml +++ b/ocaml/xapi/xapi_fuse.ml @@ -42,8 +42,21 @@ let once kind f = ) ) +(* The exit may hang, and nothing else would kill the process if it did, so + add a timer that forces the process to exit. *) +let force_exit_after_grace ~fuse_length ret_code = + let wait = fuse_length +. !Constants.fuse_exit_grace in + try + ignore (Thread.create (fun () -> Thread.delay wait ; Unix._exit ret_code) ()) ; + debug "%s: forcing exit %d if still running in %.0fs" __FUNCTION__ ret_code + wait + with e -> + warn "%s: could not start the exit timer: %s" __FUNCTION__ + (Printexc.to_string e) + let light_fuse_and_run ?(fuse_length = !Constants.fuse_time) () = once `Exit @@ fun () -> + force_exit_after_grace ~fuse_length Xapi_globs.restart_return_code ; debug "light_fuse_and_run: calling Rrdd.backup_rrds to save current RRDs locally" ; let delay_so_far = @@ -116,6 +129,7 @@ let light_fuse_and_reboot ?(fuse_length = !Constants.fuse_time) () = let light_fuse_and_dont_restart ?(fuse_length = !Constants.fuse_time) () = once `Exit @@ fun () -> + force_exit_after_grace ~fuse_length 0 ; ignore (Thread.create (fun () -> diff --git a/ocaml/xapi/xapi_globs.ml b/ocaml/xapi/xapi_globs.ml index baa64de53ff..0ffc882846b 100644 --- a/ocaml/xapi/xapi_globs.ml +++ b/ocaml/xapi/xapi_globs.ml @@ -1239,6 +1239,7 @@ let xapi_globs_spec = , LongDurationFromSeconds host_assumed_dead_interval ) ; ("fuse_time", Float Constants.fuse_time) + ; ("fuse_exit_grace", Float Constants.fuse_exit_grace) ; ("db_restore_fuse_time", Float Constants.db_restore_fuse_time) ; ("inactive_session_timeout", Float inactive_session_timeout) ; ("pending_task_timeout", Float pending_task_timeout) diff --git a/scripts/xapi.conf b/scripts/xapi.conf index 94bff038b55..e4d8166c6ba 100644 --- a/scripts/xapi.conf +++ b/scripts/xapi.conf @@ -316,6 +316,10 @@ sm-plugins=ext nfs iscsi lvmoiscsi dummy file hba rawhba udev iso lvm lvmohba lv # allows the current API call to succeed. # fuse_time = 10 +# time after the fuse delay before xapi is forced to exit if flushing the +# database and exiting has not finished. +# fuse_exit_grace = 60 + # time after receiving a pool database restore before automatically restarting # xapi to read it. # db_restore_fuse_time = 30