"""Recovery routing and preservation tests; fixtures remain in the workspace.""" import sys sys.dont_write_bytecode=True import json import os from pathlib import Path import unittest from unittest.mock import patch import rcms,operations,recovery import test_operations class Recovery(unittest.TestCase): setUp = test_operations.Operations.setUp def test_existing_raw_worker_only_postprocesses_and_keeps_raw(self): root=self.root/'run';(root/'wrf').mkdir(parents=True) raw=self.root/'old-output';raw.mkdir();rcms.put(raw/'kept','raw data') man={'cycle':self.cycle,'end':None,'config':self.cfg,'models':['wrf'], 'recovery':{'wrf':{'action':'ADOPT_POST','raw':str(raw)}}} rcms.put(root/'manifest.json',man) def post(man,source,work,r): self.assertEqual(source,raw) dest=work/'products';dest.mkdir();return dest with patch.dict(os.environ,{'SLURM_JOB_ID':'1'}), patch.object(rcms,'preflight'), patch.object(rcms,'preserve_deletions'), patch.object(recovery,'validate_raw',return_value=dict(man,end='2026082106')) as validate, patch.object(rcms,'wrf_post',side_effect=post), patch.object(rcms,'wrf_integrate') as integrate, patch.object(operations,'publish'): rcms.worker(root/'manifest.json','wrf','post','initial') validate.assert_called_once() integrate.assert_not_called() self.assertEqual((raw/'kept').read_text(),'raw data') self.assertTrue((root/'wrf/post-initial/SUCCESS.json').exists()) self.assertFalse((root/'wrf/integration').exists()) def test_resume_worker_keeps_legacy_stage_and_logs_in_new_run(self): root=self.root/'run';(root/'wrf').mkdir(parents=True) source=self.root/'old-work';source.mkdir();rcms.put(source/'old-log','keep') man={'cycle':self.cycle,'end':'2026082106','config':self.cfg,'models':['wrf'],'inputs':[], 'recovery':{'wrf':{'action':'RESUME','work':str(source)}}} rcms.put(root/'manifest.json',man) with patch.dict(os.environ,{'SLURM_JOB_ID':'1'}), patch.object(rcms,'preflight'), patch.object(rcms,'preserve_deletions'), patch.object(recovery,'resume',return_value={'raw':str(source/'output')}) as resume: rcms.worker(root/'manifest.json','wrf','integrate','initial') resume.assert_called_once() self.assertEqual((source/'old-log').read_text(),'keep') self.assertFalse((root/'wrf/integration').is_symlink()) self.assertTrue((root/'wrf/integration/SUCCESS.json').exists()) def test_controller_submits_integration_only_for_unfinished_model(self): config=self.root/'config.json';rcms.put(config,self.cfg) decisions={'wrf':{'action':'RESUME','work':str(self.root/'old')}, 'regcm':{'action':'ADOPT_POST','raw':str(self.root/'raw')}} calls=[] def submit(root,man,model,phase,token,dependency=None): calls.append((model,phase,dependency));return '123' with patch.object(sys,'argv',['rcms.py','submit',self.cycle,'--config',str(config)]), patch.object(operations,'write_preflight'), patch.object(rcms,'validate_config'), patch.object(operations,'doctor'), patch.object(operations,'cycle_lock',return_value=operations.no_lock()), patch.object(operations,'queue',return_value={}), patch.object(operations,'plan',return_value=decisions), patch.object(operations,'new_run',return_value=(self.root,{})), patch.object(rcms,'submit_job',side_effect=submit): operations.main() self.assertEqual(calls,[('wrf','integrate',None),('wrf','post','123'),('regcm','post',None)]) def test_controller_skips_all_finished_models(self): config=self.root/'config.json';rcms.put(config,self.cfg) with patch.object(sys,'argv',['rcms.py','submit',self.cycle,'--config',str(config)]), patch.object(operations,'write_preflight'), patch.object(rcms,'validate_config'), patch.object(operations,'doctor'), patch.object(operations,'cycle_lock',return_value=operations.no_lock()), patch.object(operations,'queue',return_value={}), patch.object(operations,'plan',return_value={m:{'action':'SKIP'} for m in operations.MODEL_NAMES}), patch.object(operations,'new_run') as create, patch.object(rcms,'submit_job') as submit: operations.main() create.assert_not_called();submit.assert_not_called() def test_prepared_cycle_resumes(self): work=self.root/'wrf/wdir'/self.cycle;work.mkdir() rcms.put(work/'met_em.d01.fixture.nc','prepared') self.assertEqual(operations.plan(self.cfg,self.cycle,['wrf'],{})['wrf']['action'],'RESUME') def test_raw_cycle_post_only_without_success_marker(self): work=self.root/'wrf/wdir'/self.cycle/'output';work.mkdir(parents=True) rcms.put(work/'wrfout_d01_2026-08-21_00:00:00','raw') self.assertEqual(operations.plan(self.cfg,self.cycle,['wrf'],{})['wrf']['action'],'ADOPT_POST') def test_finished_cycle_skips_even_with_raw(self): raw=self.root/'wrf/wdir'/self.cycle/'output';raw.mkdir(parents=True) rcms.put(raw/'wrfout_d01_fixture','raw') dest=operations.archive(self.cfg,'wrf',self.cycle);dest.mkdir(parents=True) for name in operations.required_products('wrf'):rcms.put(dest/name,'result') self.assertEqual(operations.plan(self.cfg,self.cycle,['wrf'],{})['wrf']['action'],'SKIP') def test_adopt_manifest_does_not_need_grib(self): self.cfg['slurm']={m:{'nodes':1,'tasks_per_node':1} for m in operations.MODEL_NAMES} decisions={'wrf':{'action':'ADOPT_POST','raw':str(self.root/'raw')}} with patch.object(rcms,'validate_config'),patch.object(rcms,'inventory',side_effect=AssertionError('must not read GRIB')): root,man=operations.new_run(self.cfg,self.cycle,['wrf'],decisions) self.assertEqual(man['inputs'],[]) self.assertTrue((root/'runtime/recovery.py').exists()) def test_resume_keeps_existing_input_and_uses_original_output_path(self): source=self.root/'wrf/wdir'/self.cycle;source.mkdir() rcms.put(source/'input_kept','old data') man={'config':self.cfg,'recovery':{'wrf':{'action':'RESUME','work':str(source)}}} output=recovery.output_dir(man,'wrf',self.root) self.assertEqual(output,source/'output') rcms.put(output/'raw','model output') with self.assertRaisesRegex(RuntimeError,'nonempty'):recovery.output_dir(man,'wrf',self.root) self.assertEqual((source/'input_kept').read_text(),'old data') self.assertEqual((output/'raw').read_text(),'model output') def test_raw_validation_rejects_time_gap(self): raw=self.root/'raw';raw.mkdir() for t in ('00','12'):rcms.put(raw/('wrfout_d01_2026-08-21_'+t+':00:00'),'data') with self.assertRaisesRegex(RuntimeError,'Incomplete WRF'): recovery.validate_raw({'cycle':self.cycle},'wrf',raw,None) def test_icbc_gap_is_rejected(self): source=self.root/'regcm/wdir'/self.cycle;(source/'input').mkdir(parents=True) rcms.put(source/'input/d01_ICBC.fixture.nc','data') class Reader: def axis(self,*args):pass def nc(self,*args):return '' def run(self,*args):return '2026-08-21T00:00:00' with self.assertRaisesRegex(RuntimeError,'missing timestamps'): recovery.check_regcm_input({'cycle':self.cycle,'end':'2026082106'},source,Reader()) if __name__=='__main__': unittest.main(verbosity=2)