Fix gaussian galloc underallocation

arc/job/adapter.py

@@ -589,9 +589,9 @@ def set_cpu_and_mem(self):
         self.cpu_cores = self.cpu_cores or job_cpu_cores
         max_mem = servers[self.server].get('memory', None) if self.server is not None else 32.0  # Max memory per node in GB.
         job_max_server_node_memory_allocation = default_job_settings.get('job_max_server_node_memory_allocation', 0.95)
-        if max_mem is not None and self.job_memory_gb > max_mem * job_max_server_node_memory_allocation:
+        if max_mem is not None and self.job_memory_gb >= max_mem * job_max_server_node_memory_allocation:
             logger.warning(f'The memory for job {self.job_name} using {self.job_adapter} ({self.job_memory_gb} GB) '
-                           f'exceeds {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on '
+                           f'reaches {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on '
                            f'{self.server}. Setting it to {job_max_server_node_memory_allocation * max_mem:.2f} GB.')
             self.job_memory_gb = job_max_server_node_memory_allocation * max_mem
             total_submit_script_memory_mib = math.ceil(self.job_memory_gb * MEMORY_GB_TO_MIB * CAPPED_JOB_MEMORY_OVERHEAD)

arc/job/adapter_test.py

@@ -22,7 +22,8 @@
 from arc.level import Level
 from arc.species import ARCSpecies
 
-servers, submit_filenames = settings['servers'], settings['submit_filenames']
+default_job_settings, servers, submit_filenames = \
+    settings['default_job_settings'], settings['servers'], settings['submit_filenames']
 
 
 class TestEnumerationClasses(unittest.TestCase):
@@ -250,6 +251,23 @@ def test_set_cpu_and_mem(self):
         self.assertEqual(self.job_4.submit_script_memory, expected_memory)
         self.job_4.server = 'local'
 
+        capped_memory_gb = servers['server2']['memory'] \
+            * default_job_settings['job_max_server_node_memory_allocation']
+        original_job_memory_gb = self.job_4.job_memory_gb
+        self.addCleanup(setattr, self.job_4, 'job_memory_gb', original_job_memory_gb)
+        self.addCleanup(setattr, self.job_4, 'server', 'local')
+        self.job_4.server = 'server2'
+        self.job_4.cpu_cores = None
+        self.job_4.job_memory_gb = capped_memory_gb
+        self.job_4.set_cpu_and_mem()
+        self.assertEqual(self.job_4.job_memory_gb, capped_memory_gb)
+        self.assertEqual(self.job_4.submit_script_memory_mib, math.ceil(capped_memory_gb * 1024 * 1.05))
+        self.assertLess(self.job_4.submit_script_memory_mib, servers['server2']['memory'] * 1024)
+        self.assertIn('max_total_job_memory', self.job_4.job_status[1]['keywords'])
+        self.job_4.job_status[1]['keywords'].remove('max_total_job_memory')
+        self.job_4.job_memory_gb = original_job_memory_gb
+        self.job_4.server = 'local'
+
     def test_set_file_paths(self):
         """Test setting up the job's paths"""
         self.assertEqual(self.job_1.local_path, os.path.join(self.job_1.project_directory, 'calcs', 'Species',

arc/job/trsh.py

@@ -202,7 +202,7 @@ def determine_ess_status(output_path: str,
                     line = ''
                 elif 'malloc failed' in line or 'galloc' in line:
                     keywords = ['Memory']
-                    error = 'Memory allocation failed (did you ask for too much?)'
+                    error = 'Gaussian could not allocate the memory it was given, it requires more memory.'
                     line = ''
                 elif 'PGFIO/stdio: No such file or directory' in line:
                     keywords = ['Scratch']
@@ -989,14 +989,16 @@ def trsh_ess_job(label: str,
             else:
                 couldnt_trsh = True
                 output_errors.append(
-                    f'Error: Could not troubleshoot {job_type} for {label}! Gaussian exhausted memory even after ARC '
-                    f'reached the configured node-memory cap ({max_mem_allocation:.2f} GB total allocation) while '
-                    f'still reserving scheduler headroom. Use a higher-memory node or lower the job cost; '
+                    f'Error: Could not troubleshoot {job_type} for {label}! Gaussian could not allocate the memory it '
+                    f'was given, and this job already requests the configured per-job memory cap on {server}, '
+                    f'{max_mem_allocation:.2f} GB. It needs more memory than the cap allows. Either raise the cap by '
+                    f'changing servers[\'{server}\'][\'memory\'] (currently {max_mem} GB) or '
+                    f'job_max_server_node_memory_allocation, or use a cheaper method or level of theory; '
                 )
                 logger.error(
-                    f'Could not troubleshoot {job_type} job in {software} for {label}. ARC already reached the '
-                    f'configured node-memory cap ({max_mem_allocation:.2f} GB total allocation) and still preserved '
-                    f'Gaussian headroom.'
+                    f'Could not troubleshoot {job_type} job in {software} for {label}. Gaussian could not allocate '
+                    f'{memory_gb} GB, which is the configured per-job memory cap on {server} '
+                    f'({max_mem_allocation:.2f} GB).'
                 )
 
         if attempted_ess_trsh_methods:

arc/job/trsh_test.py

@@ -433,7 +433,7 @@ def test_trsh_ess_job(self):
 
         # Gaussian: test 6
         job_status = {'keywords': ['Memory', 'max_total_job_memory'],
-                      'error': 'Memory allocation failed (did you ask for too much?)'}
+                      'error': 'Gaussian could not allocate the memory it was given, it requires more memory.'}
         capped_memory_gb = settings['default_job_settings']['job_max_server_node_memory_allocation'] * \
             settings['servers']['server2']['memory']
         output_errors, ess_trsh_methods, remove_checkfile, level_of_theory, software, job_type, fine, trsh_keyword, \
@@ -443,7 +443,8 @@ def test_trsh_ess_job(self):
 
         self.assertTrue(couldnt_trsh)
         self.assertEqual(memory, capped_memory_gb)
-        self.assertIn('Use a higher-memory node or lower the job cost', output_errors[0])
+        self.assertIn('job_max_server_node_memory_allocation', output_errors[0])
+        self.assertNotIn('higher-memory node', output_errors[0])
 
         # Gaussian: test 7 - part 1
         job_status = {'keywords': ['SCF', 'GL502', 'NoSymm']}
@@ -832,6 +833,48 @@ def test_trsh_ess_job(self):
                               num_heavy_atoms, cpu_cores, ess_trsh_methods,
                               is_h=True, is_monoatomic=True)
 
+    def test_trsh_ess_job_gaussian_galloc_escalates_memory_to_the_cap(self):
+        """Test that a Gaussian galloc failure escalates the job memory up to the configured cap, then gives up."""
+        label, level_of_theory, job_type, software = 'TS0', {'method': 'wb97xd', 'basis': 'def2tzvp'}, 'conf_opt', \
+            'gaussian'
+        server, num_heavy_atoms, fine = 'server2', 5, False
+        path = os.path.join(self.base_path['gaussian'], 'galloc.out')
+        status, keywords, error, _ = trsh.determine_ess_status(output_path=path,
+                                                               species_label=label,
+                                                               job_type=job_type,
+                                                               software=software)
+        self.assertEqual(status, 'errored')
+        self.assertEqual(keywords, ['Memory'])
+        self.assertNotIn('too high', error)
+        self.assertNotIn('too much', error)
+        job_status = {'keywords': keywords, 'error': error}
+        capped_memory_gb = settings['default_job_settings']['job_max_server_node_memory_allocation'] * \
+            settings['servers']['server2']['memory']
+
+        memory, cpu_cores, ess_trsh_methods = 42.0, 12, []
+        for expected_memory in [84.0, 168.0, capped_memory_gb]:
+            output_errors, ess_trsh_methods, _, _, _, _, _, _, memory, _, cpu_cores, couldnt_trsh = \
+                trsh.trsh_ess_job(label, level_of_theory, server, job_status, job_type, software, fine, memory,
+                                  num_heavy_atoms, cpu_cores, ess_trsh_methods)
+            self.assertFalse(couldnt_trsh)
+            self.assertEqual(memory, expected_memory)
+            self.assertEqual(cpu_cores, 12)
+            self.assertEqual(ess_trsh_methods[-1], 'memory')
+            self.assertEqual(output_errors, [])
+
+        output_errors, ess_trsh_methods, _, _, _, _, _, _, memory, _, cpu_cores, couldnt_trsh = \
+            trsh.trsh_ess_job(label, level_of_theory, server, job_status, job_type, software, fine, memory,
+                              num_heavy_atoms, cpu_cores, ess_trsh_methods)
+        self.assertTrue(couldnt_trsh)
+        self.assertEqual(memory, capped_memory_gb)
+        self.assertEqual(cpu_cores, 12)
+        self.assertIn('all_attempted', ess_trsh_methods)
+        self.assertIn(f'{capped_memory_gb:.2f} GB', output_errors[0])
+        self.assertIn("servers['server2']['memory']", output_errors[0])
+        self.assertIn('job_max_server_node_memory_allocation', output_errors[0])
+        self.assertIn('cheaper method or level of theory', output_errors[0])
+        self.assertNotIn('higher-memory node', output_errors[0])
+
     def test_trsh_ess_job_terachem_trsh_attempt_only(self):
         """Isolate the terachem trsh_attempt-only case from Gaussian stateful flow."""
         label = 'ethanol'

arc/testing/trsh/gaussian/galloc.out

@@ -0,0 +1,109 @@
+ Entering Gaussian System, Link 0=g16
+ Initial command:
+ /usr/local/g16/l1.exe "/gtmp/alon/scratch/g16/4393461.zeus-master/Gau-173582.inp" -scrdir="/gtmp/alon/scratch/g16/4393461.zeus-master/"
+ Entering Link 1 = /usr/local/g16/l1.exe PID=    173583.
+
+ Copyright (c) 1988-2019, Gaussian, Inc.  All Rights Reserved.
+
+ This is part of the Gaussian(R) 16 program.  It is based on
+ the Gaussian(R) 09 system (copyright 2009, Gaussian, Inc.),
+ the Gaussian(R) 03 system (copyright 2003, Gaussian, Inc.),
+ the Gaussian(R) 98 system (copyright 1998, Gaussian, Inc.),
+ the Gaussian(R) 94 system (copyright 1995, Gaussian, Inc.),
+ the Gaussian 92(TM) system (copyright 1992, Gaussian, Inc.),
+ the Gaussian 90(TM) system (copyright 1990, Gaussian, Inc.),
+ the Gaussian 88(TM) system (copyright 1988, Gaussian, Inc.),
+ the Gaussian 86(TM) system (copyright 1986, Carnegie Mellon
+ University), and the Gaussian 82(TM) system (copyright 1983,
+ Carnegie Mellon University). Gaussian is a federally registered
+ trademark of Gaussian, Inc.
+
+ This software contains proprietary and confidential information,
+ including trade secrets, belonging to Gaussian, Inc.
+
+ This software is provided under written license and may be
+ used, copied, transmitted, or stored only in accord with that
+ written license.
+
+ The following legend is applicable only to US Government
+ contracts under FAR:
+
+                    RESTRICTED RIGHTS LEGEND
+
+ Use, reproduction and disclosure by the US Government is
+ subject to restrictions as set forth in subparagraphs (a)
+ and (c) of the Commercial Computer Software - Restricted
+ Rights clause in FAR 52.227-19.
+
+ Gaussian, Inc.
+ 340 Quinnipiac St., Bldg. 40, Wallingford CT 06492
+
+
+ ---------------------------------------------------------------
+ Warning -- This program may not be used in any manner that
+ competes with the business of Gaussian, Inc. or will provide
+ assistance to any competitor of Gaussian, Inc.  The licensee
+ of this program is prohibited from giving any competitor of
+ Gaussian, Inc. access to this program.  By using this program,
+ the user acknowledges that Gaussian, Inc. is engaged in the
+ business of creating and licensing software in the field of
+ computational chemistry and represents and warrants to the
+ licensee that it is not a competitor of Gaussian, Inc. and that
+ it will not use this program in any manner prohibited above.
+ ---------------------------------------------------------------
+
+
+ Cite this work as:
+ Gaussian 16, Revision C.01,
+ M. J. Frisch, G. W. Trucks, H. B. Schlegel, G. E. Scuseria, 
+ M. A. Robb, J. R. Cheeseman, G. Scalmani, V. Barone, 
+ G. A. Petersson, H. Nakatsuji, X. Li, M. Caricato, A. V. Marenich, 
+ J. Bloino, B. G. Janesko, R. Gomperts, B. Mennucci, H. P. Hratchian, 
+ J. V. Ortiz, A. F. Izmaylov, J. L. Sonnenberg, D. Williams-Young, 
+ F. Ding, F. Lipparini, F. Egidi, J. Goings, B. Peng, A. Petrone, 
+ T. Henderson, D. Ranasinghe, V. G. Zakrzewski, J. Gao, N. Rega, 
+ G. Zheng, W. Liang, M. Hada, M. Ehara, K. Toyota, R. Fukuda, 
+ J. Hasegawa, M. Ishida, T. Nakajima, Y. Honda, O. Kitao, H. Nakai, 
+ T. Vreven, K. Throssell, J. A. Montgomery, Jr., J. E. Peralta, 
+ F. Ogliaro, M. J. Bearpark, J. J. Heyd, E. N. Brothers, K. N. Kudin, 
+ V. N. Staroverov, T. A. Keith, R. Kobayashi, J. Normand, 
+ K. Raghavachari, A. P. Rendell, J. C. Burant, S. S. Iyengar, 
+ J. Tomasi, M. Cossi, J. M. Millam, M. Klene, C. Adamo, R. Cammi, 
+ J. W. Ochterski, R. L. Martin, K. Morokuma, O. Farkas, 
+ J. B. Foresman, and D. J. Fox, Gaussian, Inc., Wallingford CT, 2019.
+
+ ******************************************
+ Gaussian 16:  EM64L-G16RevC.01  3-Jul-2019
+                 4-Jul-2026 
+ ******************************************
+ %chk=check.chk
+ %mem=61637mb
+ %NProcShared=16
+ Will use up to   16 processors via shared memory.
+ --------------------------------------------------------
+ #P opt=(calcfc) guess=read wb97xd/def2tzvp IOp(2/9=2000)
+ --------------------------------------------------------
+ 1/10=4,18=20,19=15,26=3,38=1/1,3;
+ 2/9=2000,12=2,17=6,18=5,40=1/2;
+ 3/5=44,7=101,11=2,25=1,30=1,71=2,74=-58,116=-2,140=1/1,2,3;
+ 4/5=1/1;
+ 5/5=2,38=6/2;
+ 8/6=4,10=90,11=11/1;
+ 11/6=1,8=1,9=11,15=111,16=1/1,2,10;
+ 10/6=1,13=1/2;
+ 6/7=2,8=2,9=2,10=2,28=1/1;
+ 7/10=1,25=1/1,2,3,16;
+ 1/10=4,18=20,19=15,26=3/3(2);
+ 2/9=2000/2;
+ 99//99;
+ 2/9=2000/2;
+ 3/5=44,7=101,11=2,25=1,30=1,71=1,74=-58/1,2,3;
+ 4/5=5,16=3,69=1/1;
+ 5/5=2,38=5/2;
+ 7//1,2,3,16;
+ 1/18=20,19=15,26=3/3(-5);
+ 2/9=2000/2;
+ 6/7=2,8=2,9=2,10=2,19=2,28=1/1;
+ 99/9=1/99;
+ Leave Link    1 at Sat Jul  4 12:46:50 2026, MaxMem=  8078884864 cpu:               0.7 elap:               0.1
+galloc:  could not allocate memory.