diff --git a/arc/job/adapter.py b/arc/job/adapter.py index 9f47ea0c6a..80f9c02176 100644 --- a/arc/job/adapter.py +++ b/arc/job/adapter.py @@ -589,9 +589,9 @@ def set_cpu_and_mem(self): self.cpu_cores = self.cpu_cores or job_cpu_cores max_mem = servers[self.server].get('memory', None) if self.server is not None else 32.0 # Max memory per node in GB. job_max_server_node_memory_allocation = default_job_settings.get('job_max_server_node_memory_allocation', 0.95) - if max_mem is not None and self.job_memory_gb > max_mem * job_max_server_node_memory_allocation: + if max_mem is not None and self.job_memory_gb >= max_mem * job_max_server_node_memory_allocation: logger.warning(f'The memory for job {self.job_name} using {self.job_adapter} ({self.job_memory_gb} GB) ' - f'exceeds {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' + f'reaches {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' f'{self.server}. Setting it to {job_max_server_node_memory_allocation * max_mem:.2f} GB.') self.job_memory_gb = job_max_server_node_memory_allocation * max_mem total_submit_script_memory_mib = math.ceil(self.job_memory_gb * MEMORY_GB_TO_MIB * CAPPED_JOB_MEMORY_OVERHEAD) diff --git a/arc/job/adapter_test.py b/arc/job/adapter_test.py index 3e3536ea09..24577a9a5c 100644 --- a/arc/job/adapter_test.py +++ b/arc/job/adapter_test.py @@ -22,7 +22,8 @@ from arc.level import Level from arc.species import ARCSpecies -servers, submit_filenames = settings['servers'], settings['submit_filenames'] +default_job_settings, servers, submit_filenames = \ + settings['default_job_settings'], settings['servers'], settings['submit_filenames'] class TestEnumerationClasses(unittest.TestCase): @@ -250,6 +251,23 @@ def test_set_cpu_and_mem(self): self.assertEqual(self.job_4.submit_script_memory, expected_memory) self.job_4.server = 'local' + capped_memory_gb = servers['server2']['memory'] \ + * default_job_settings['job_max_server_node_memory_allocation'] + original_job_memory_gb = self.job_4.job_memory_gb + self.addCleanup(setattr, self.job_4, 'job_memory_gb', original_job_memory_gb) + self.addCleanup(setattr, self.job_4, 'server', 'local') + self.job_4.server = 'server2' + self.job_4.cpu_cores = None + self.job_4.job_memory_gb = capped_memory_gb + self.job_4.set_cpu_and_mem() + self.assertEqual(self.job_4.job_memory_gb, capped_memory_gb) + self.assertEqual(self.job_4.submit_script_memory_mib, math.ceil(capped_memory_gb * 1024 * 1.05)) + self.assertLess(self.job_4.submit_script_memory_mib, servers['server2']['memory'] * 1024) + self.assertIn('max_total_job_memory', self.job_4.job_status[1]['keywords']) + self.job_4.job_status[1]['keywords'].remove('max_total_job_memory') + self.job_4.job_memory_gb = original_job_memory_gb + self.job_4.server = 'local' + def test_set_file_paths(self): """Test setting up the job's paths""" self.assertEqual(self.job_1.local_path, os.path.join(self.job_1.project_directory, 'calcs', 'Species', diff --git a/arc/job/trsh.py b/arc/job/trsh.py index 4a970dc840..e3ddcbce45 100644 --- a/arc/job/trsh.py +++ b/arc/job/trsh.py @@ -202,7 +202,7 @@ def determine_ess_status(output_path: str, line = '' elif 'malloc failed' in line or 'galloc' in line: keywords = ['Memory'] - error = 'Memory allocation failed (did you ask for too much?)' + error = 'Gaussian could not allocate the memory it was given, it requires more memory.' line = '' elif 'PGFIO/stdio: No such file or directory' in line: keywords = ['Scratch'] @@ -989,14 +989,16 @@ def trsh_ess_job(label: str, else: couldnt_trsh = True output_errors.append( - f'Error: Could not troubleshoot {job_type} for {label}! Gaussian exhausted memory even after ARC ' - f'reached the configured node-memory cap ({max_mem_allocation:.2f} GB total allocation) while ' - f'still reserving scheduler headroom. Use a higher-memory node or lower the job cost; ' + f'Error: Could not troubleshoot {job_type} for {label}! Gaussian could not allocate the memory it ' + f'was given, and this job already requests the configured per-job memory cap on {server}, ' + f'{max_mem_allocation:.2f} GB. It needs more memory than the cap allows. Either raise the cap by ' + f'changing servers[\'{server}\'][\'memory\'] (currently {max_mem} GB) or ' + f'job_max_server_node_memory_allocation, or use a cheaper method or level of theory; ' ) logger.error( - f'Could not troubleshoot {job_type} job in {software} for {label}. ARC already reached the ' - f'configured node-memory cap ({max_mem_allocation:.2f} GB total allocation) and still preserved ' - f'Gaussian headroom.' + f'Could not troubleshoot {job_type} job in {software} for {label}. Gaussian could not allocate ' + f'{memory_gb} GB, which is the configured per-job memory cap on {server} ' + f'({max_mem_allocation:.2f} GB).' ) if attempted_ess_trsh_methods: diff --git a/arc/job/trsh_test.py b/arc/job/trsh_test.py index 80fdf935e4..f9a196f2b7 100644 --- a/arc/job/trsh_test.py +++ b/arc/job/trsh_test.py @@ -433,7 +433,7 @@ def test_trsh_ess_job(self): # Gaussian: test 6 job_status = {'keywords': ['Memory', 'max_total_job_memory'], - 'error': 'Memory allocation failed (did you ask for too much?)'} + 'error': 'Gaussian could not allocate the memory it was given, it requires more memory.'} capped_memory_gb = settings['default_job_settings']['job_max_server_node_memory_allocation'] * \ settings['servers']['server2']['memory'] output_errors, ess_trsh_methods, remove_checkfile, level_of_theory, software, job_type, fine, trsh_keyword, \ @@ -443,7 +443,8 @@ def test_trsh_ess_job(self): self.assertTrue(couldnt_trsh) self.assertEqual(memory, capped_memory_gb) - self.assertIn('Use a higher-memory node or lower the job cost', output_errors[0]) + self.assertIn('job_max_server_node_memory_allocation', output_errors[0]) + self.assertNotIn('higher-memory node', output_errors[0]) # Gaussian: test 7 - part 1 job_status = {'keywords': ['SCF', 'GL502', 'NoSymm']} @@ -832,6 +833,48 @@ def test_trsh_ess_job(self): num_heavy_atoms, cpu_cores, ess_trsh_methods, is_h=True, is_monoatomic=True) + def test_trsh_ess_job_gaussian_galloc_escalates_memory_to_the_cap(self): + """Test that a Gaussian galloc failure escalates the job memory up to the configured cap, then gives up.""" + label, level_of_theory, job_type, software = 'TS0', {'method': 'wb97xd', 'basis': 'def2tzvp'}, 'conf_opt', \ + 'gaussian' + server, num_heavy_atoms, fine = 'server2', 5, False + path = os.path.join(self.base_path['gaussian'], 'galloc.out') + status, keywords, error, _ = trsh.determine_ess_status(output_path=path, + species_label=label, + job_type=job_type, + software=software) + self.assertEqual(status, 'errored') + self.assertEqual(keywords, ['Memory']) + self.assertNotIn('too high', error) + self.assertNotIn('too much', error) + job_status = {'keywords': keywords, 'error': error} + capped_memory_gb = settings['default_job_settings']['job_max_server_node_memory_allocation'] * \ + settings['servers']['server2']['memory'] + + memory, cpu_cores, ess_trsh_methods = 42.0, 12, [] + for expected_memory in [84.0, 168.0, capped_memory_gb]: + output_errors, ess_trsh_methods, _, _, _, _, _, _, memory, _, cpu_cores, couldnt_trsh = \ + trsh.trsh_ess_job(label, level_of_theory, server, job_status, job_type, software, fine, memory, + num_heavy_atoms, cpu_cores, ess_trsh_methods) + self.assertFalse(couldnt_trsh) + self.assertEqual(memory, expected_memory) + self.assertEqual(cpu_cores, 12) + self.assertEqual(ess_trsh_methods[-1], 'memory') + self.assertEqual(output_errors, []) + + output_errors, ess_trsh_methods, _, _, _, _, _, _, memory, _, cpu_cores, couldnt_trsh = \ + trsh.trsh_ess_job(label, level_of_theory, server, job_status, job_type, software, fine, memory, + num_heavy_atoms, cpu_cores, ess_trsh_methods) + self.assertTrue(couldnt_trsh) + self.assertEqual(memory, capped_memory_gb) + self.assertEqual(cpu_cores, 12) + self.assertIn('all_attempted', ess_trsh_methods) + self.assertIn(f'{capped_memory_gb:.2f} GB', output_errors[0]) + self.assertIn("servers['server2']['memory']", output_errors[0]) + self.assertIn('job_max_server_node_memory_allocation', output_errors[0]) + self.assertIn('cheaper method or level of theory', output_errors[0]) + self.assertNotIn('higher-memory node', output_errors[0]) + def test_trsh_ess_job_terachem_trsh_attempt_only(self): """Isolate the terachem trsh_attempt-only case from Gaussian stateful flow.""" label = 'ethanol' diff --git a/arc/testing/trsh/gaussian/galloc.out b/arc/testing/trsh/gaussian/galloc.out new file mode 100644 index 0000000000..c1b13aab6f --- /dev/null +++ b/arc/testing/trsh/gaussian/galloc.out @@ -0,0 +1,109 @@ + Entering Gaussian System, Link 0=g16 + Initial command: + /usr/local/g16/l1.exe "/gtmp/alon/scratch/g16/4393461.zeus-master/Gau-173582.inp" -scrdir="/gtmp/alon/scratch/g16/4393461.zeus-master/" + Entering Link 1 = /usr/local/g16/l1.exe PID= 173583. + + Copyright (c) 1988-2019, Gaussian, Inc. All Rights Reserved. + + This is part of the Gaussian(R) 16 program. It is based on + the Gaussian(R) 09 system (copyright 2009, Gaussian, Inc.), + the Gaussian(R) 03 system (copyright 2003, Gaussian, Inc.), + the Gaussian(R) 98 system (copyright 1998, Gaussian, Inc.), + the Gaussian(R) 94 system (copyright 1995, Gaussian, Inc.), + the Gaussian 92(TM) system (copyright 1992, Gaussian, Inc.), + the Gaussian 90(TM) system (copyright 1990, Gaussian, Inc.), + the Gaussian 88(TM) system (copyright 1988, Gaussian, Inc.), + the Gaussian 86(TM) system (copyright 1986, Carnegie Mellon + University), and the Gaussian 82(TM) system (copyright 1983, + Carnegie Mellon University). Gaussian is a federally registered + trademark of Gaussian, Inc. + + This software contains proprietary and confidential information, + including trade secrets, belonging to Gaussian, Inc. + + This software is provided under written license and may be + used, copied, transmitted, or stored only in accord with that + written license. + + The following legend is applicable only to US Government + contracts under FAR: + + RESTRICTED RIGHTS LEGEND + + Use, reproduction and disclosure by the US Government is + subject to restrictions as set forth in subparagraphs (a) + and (c) of the Commercial Computer Software - Restricted + Rights clause in FAR 52.227-19. + + Gaussian, Inc. + 340 Quinnipiac St., Bldg. 40, Wallingford CT 06492 + + + --------------------------------------------------------------- + Warning -- This program may not be used in any manner that + competes with the business of Gaussian, Inc. or will provide + assistance to any competitor of Gaussian, Inc. The licensee + of this program is prohibited from giving any competitor of + Gaussian, Inc. access to this program. By using this program, + the user acknowledges that Gaussian, Inc. is engaged in the + business of creating and licensing software in the field of + computational chemistry and represents and warrants to the + licensee that it is not a competitor of Gaussian, Inc. and that + it will not use this program in any manner prohibited above. + --------------------------------------------------------------- + + + Cite this work as: + Gaussian 16, Revision C.01, + M. J. Frisch, G. W. Trucks, H. B. Schlegel, G. E. Scuseria, + M. A. Robb, J. R. Cheeseman, G. Scalmani, V. Barone, + G. A. Petersson, H. Nakatsuji, X. Li, M. Caricato, A. V. Marenich, + J. Bloino, B. G. Janesko, R. Gomperts, B. Mennucci, H. P. Hratchian, + J. V. Ortiz, A. F. Izmaylov, J. L. Sonnenberg, D. Williams-Young, + F. Ding, F. Lipparini, F. Egidi, J. Goings, B. Peng, A. Petrone, + T. Henderson, D. Ranasinghe, V. G. Zakrzewski, J. Gao, N. Rega, + G. Zheng, W. Liang, M. Hada, M. Ehara, K. Toyota, R. Fukuda, + J. Hasegawa, M. Ishida, T. Nakajima, Y. Honda, O. Kitao, H. Nakai, + T. Vreven, K. Throssell, J. A. Montgomery, Jr., J. E. Peralta, + F. Ogliaro, M. J. Bearpark, J. J. Heyd, E. N. Brothers, K. N. Kudin, + V. N. Staroverov, T. A. Keith, R. Kobayashi, J. Normand, + K. Raghavachari, A. P. Rendell, J. C. Burant, S. S. Iyengar, + J. Tomasi, M. Cossi, J. M. Millam, M. Klene, C. Adamo, R. Cammi, + J. W. Ochterski, R. L. Martin, K. Morokuma, O. Farkas, + J. B. Foresman, and D. J. Fox, Gaussian, Inc., Wallingford CT, 2019. + + ****************************************** + Gaussian 16: EM64L-G16RevC.01 3-Jul-2019 + 4-Jul-2026 + ****************************************** + %chk=check.chk + %mem=61637mb + %NProcShared=16 + Will use up to 16 processors via shared memory. + -------------------------------------------------------- + #P opt=(calcfc) guess=read wb97xd/def2tzvp IOp(2/9=2000) + -------------------------------------------------------- + 1/10=4,18=20,19=15,26=3,38=1/1,3; + 2/9=2000,12=2,17=6,18=5,40=1/2; + 3/5=44,7=101,11=2,25=1,30=1,71=2,74=-58,116=-2,140=1/1,2,3; + 4/5=1/1; + 5/5=2,38=6/2; + 8/6=4,10=90,11=11/1; + 11/6=1,8=1,9=11,15=111,16=1/1,2,10; + 10/6=1,13=1/2; + 6/7=2,8=2,9=2,10=2,28=1/1; + 7/10=1,25=1/1,2,3,16; + 1/10=4,18=20,19=15,26=3/3(2); + 2/9=2000/2; + 99//99; + 2/9=2000/2; + 3/5=44,7=101,11=2,25=1,30=1,71=1,74=-58/1,2,3; + 4/5=5,16=3,69=1/1; + 5/5=2,38=5/2; + 7//1,2,3,16; + 1/18=20,19=15,26=3/3(-5); + 2/9=2000/2; + 6/7=2,8=2,9=2,10=2,19=2,28=1/1; + 99/9=1/99; + Leave Link 1 at Sat Jul 4 12:46:50 2026, MaxMem= 8078884864 cpu: 0.7 elap: 0.1 +galloc: could not allocate memory.