From b958871fc5c9cffec0523ce9b74848807bd410bd Mon Sep 17 00:00:00 2001 From: Sean Cheah Date: Wed, 9 Sep 2026 21:26:17 -0700 Subject: [PATCH 1/3] Sherlock: only CPUs with AVX-512 --- runscripts/workflow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runscripts/workflow.py b/runscripts/workflow.py index 754f40c03..242de0c1e 100644 --- a/runscripts/workflow.py +++ b/runscripts/workflow.py @@ -109,7 +109,9 @@ def hyperqueue_snippets(outdir: str) -> tuple[str, str]: # See https://github.com/CovertLab/vEcoli/pull/331 "CLUSTER_OPTIONS": { "prefer": '"CPU_GEN:GEN|CPU_GEN:SPR"', - "constraint": '"CPU_GEN:RME|CPU_GEN:MLN|CPU_GEN:BGM|CPU_GEN:SIE|CPU_GEN:GEN|CPU_GEN:SPR"', + # Removed EPYC Rome and Milan as they do not support AVX-512, + # causing unpredictable floating point differences + "constraint": '"CPU_GEN:BGM|CPU_GEN:SIE|CPU_GEN:GEN|CPU_GEN:SPR"', }, "QUEUE": "owners,normal", }, From 055ce0a8cbc379a813676e0e75f6308eda7d1bc1 Mon Sep 17 00:00:00 2001 From: Sean Cheah Date: Sat, 19 Sep 2026 10:23:16 -0700 Subject: [PATCH 2/3] Docs: Sherlock CPU generation constraint --- doc/hpc.rst | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/doc/hpc.rst b/doc/hpc.rst index 2b9812c29..4c943be14 100644 --- a/doc/hpc.rst +++ b/doc/hpc.rst @@ -286,6 +286,21 @@ than a few minutes from their workflow configuration. Instead, either run these manually following :ref:`sherlock-interactive` or create a SLURM batch script to run these analyses following :ref:`sherlock-noninteractive`. +.. _reproducibility: + +Reproducibility +=============== + +To ensure reproducibility, workflow steps on Sherlock are intentionally constrained to nodes +with CPUs that have been tested to produce numerically identical output for a 4-seed, +4-generation baseline glucose workflow. This is necessary because older CPUs lack support +for features like AVX-512 instructions that NumPy and OpenBLAS can utilize on newer CPUs. +As such, older CPUs tend to use different code paths in NumPy and OpenBLAS, leading to +unpredictable floating point differences. See ``sherlock --> nf_config_overrides --> CLUSTER_OPTIONS`` +under the source code for :py:data:`runscripts.workflow.CLUSTER_PRESETS` for the current +list of supported CPU generations. As new CPU models are introduced, they will need to be +tested for consistency with existing ones before being added to this list. + .. _sherlock-interactive: Interactive Container @@ -294,12 +309,13 @@ Interactive Container .. warning:: The following steps should be run on a compute node. See the `Sherlock documentation `_ - for details. + for details. Be sure to use the ``--constraint`` flag to request nodes with + supported CPU generations as described in :ref:`reproducibility`. The maximum resource request for an interactive compute node is 2 hours, 4 CPU cores, and 8GB RAM/core. Scripts that require more -resources should be submitted as SLURM batch scripts to the ``mcovert`` -or ``owners`` partition (see :ref:`sherlock-noninteractive`). +resources should be submitted as SLURM batch scripts to the ``owners`` +partition (see :ref:`sherlock-noninteractive`). To run scripts on Sherlock, you must have either: @@ -404,11 +420,22 @@ to include one of the following directives at the top of your script: Importantly, if all 32 cores are in use by ``mcovert`` users, not only will your script have to wait for resources to free up, so will any workflows. As such, treat this partition as a limited resource reserved for high priority jobs. + +.. warning:: + As of Sep 19, 2026, the ``mcovert`` partition does not contain any nodes with + a CPU generation supported by ``workflow.py`` (see :ref:`reproducibility`). + Scripts manually run on this partition will diverge from results generated by + ``workflow.py``. + - ``#SBATCH --partition=normal``: Potentially longer queue time than either of the two options above but no risk of preemption. - ``#SBATCH --partition=owners,normal``: Uses either the ``owners`` or ``normal`` partition. This is the recommended option for the vast majority of scripts. +If not using the deprecated ``mcovert`` partition, also include the +``#SBATCH --constraint=`` directive to request +nodes with a supported CPU generation as described in :ref:`reproducibility`. + Following is a sample of sbatch scripts for requiring more resources to analysis simulation results: .. code-block:: bash @@ -420,6 +447,7 @@ Following is a sample of sbatch scripts for requiring more resources to analysis #SBATCH --time=20:00 #SBATCH --ntasks=1 #SBATCH --partition=owners,normal + #SBATCH --constraint="CPU_GEN:BGM|CPU_GEN:SIE|CPU_GEN:GEN|CPU_GEN:SPR" #SBATCH --cpus-per-task=4 #SBATCH --mem=64GB From 0f0dbe352984314ba0cc93c3f8f78563b7188a27 Mon Sep 17 00:00:00 2001 From: Sean Cheah Date: Sun, 20 Sep 2026 16:29:47 -0700 Subject: [PATCH 3/3] Link to PR --- runscripts/workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/runscripts/workflow.py b/runscripts/workflow.py index 242de0c1e..58d5182ec 100644 --- a/runscripts/workflow.py +++ b/runscripts/workflow.py @@ -111,6 +111,7 @@ def hyperqueue_snippets(outdir: str) -> tuple[str, str]: "prefer": '"CPU_GEN:GEN|CPU_GEN:SPR"', # Removed EPYC Rome and Milan as they do not support AVX-512, # causing unpredictable floating point differences + # See https://github.com/CovertLab/vEcoli/pull/444 "constraint": '"CPU_GEN:BGM|CPU_GEN:SIE|CPU_GEN:GEN|CPU_GEN:SPR"', }, "QUEUE": "owners,normal",