Changes between Initial Version and Version 1 of Workshops/JobCheckpointing/Examples/MATLAB

Timestamp:

04/20/2026 03:38:07 PM (3 months ago)

Author:

Carl Baribault

Comment:

Draft for MATLAB

Workshops/JobCheckpointing/Examples/MATLAB

@@ +1 @@
+[[PageOutline]]
+= MATLAB Checkpointing Example =
+
+Here is an example of a checkpointing application implemented in MATLAB script.
+
+By default, the MATLAB checkpointing application behaves as follows.
+
+ * checkpointing every 20 application iterations (one second per iteration) and
+ * running a total of 500 iterations.
+
+== Running the MATLAB checkpointing example with defaults ==
+
+To run the application on Cypress, perform the following.
+
+1. Create your own versions of the following files in your current directory on Cypress. (For file editing with '''nano''' on Cypress, see [[https://wiki.hpc.tulane.edu/trac/wiki/cypress/FileEditingSoftware/Example|File Editing Example]].)
+
+ * [wiki://Workshops/JobCheckpointing/Examples#checkpoint_submitter.sh checkpoint_submitter.sh]
+ * [wiki://Workshops/JobCheckpointing/Examples#checkpoint_runner.sh checkpoint_runner.sh]
+ * [wiki://Workshops/JobCheckpointing/Examples#aggregate_usage.py aggregate_usage.py]
+ * [wiki://Workshops/JobCheckpointing/Examples#plot_progress.py plot_progress.py]
+
+2. Create your own version of the following MATLAB application script file '''checkpoint_signal_iter.m'''.
+
+{{{
+%--------------------------------------------------------------------------
+% checkpoint_signal_iter.m
+%
+% MATLAB R2023a application-level checkpointing example
+% for submitter/runner HPC workflows.
+%
+% Contract:
+%   - Text checkpoint (state_iter.txt): orchestration progress ONLY
+%   - MAT-file checkpoint (state_iter.mat): application state ONLY
+%--------------------------------------------------------------------------
+clc;
+%% ---------------- Global state (required for onCleanup) ----------------
+global iter result TXT_CKPT MAT_CKPT MAX_ITER CHECKPOINT_EVERY
+%% ---------------- Configuration ----------------
+% Text checkpoint (submitter reads this)
+TXT_CKPT = getenv('CKPT_PATH');
+if isempty(TXT_CKPT)
+  TXT_CKPT = 'state_iter.txt';
+end
+% MATLAB binary checkpoint (application state)
+MAT_CKPT = strrep(TXT_CKPT, '.txt', '.mat');
+MAX_ITER = getenv('MAX_ITER');
+if isempty(MAX_ITER)
+  MAX_ITER = 500;
+else
+  MAX_ITER = str2double(MAX_ITER);
+end
+CHECKPOINT_EVERY = getenv('CHECKPOINT_EVERY');
+if isempty(CHECKPOINT_EVERY)
+  CHECKPOINT_EVERY = 20;
+else
+  CHECKPOINT_EVERY = str2double(CHECKPOINT_EVERY);
+end
+fprintf('MATLAB checkpointing application\n');
+fprintf(' Text checkpoint: %s\n', TXT_CKPT);
+fprintf(' State checkpoint: %s\n', MAT_CKPT);
+fprintf(' MAX_ITER=%d, CHECKPOINT_EVERY=%d\n', MAX_ITER, CHECKPOINT_EVERY);
+%% ---------------- Restore checkpoint ----------------
+if isfile(MAT_CKPT)
+  S = load(MAT_CKPT, 'iter', 'result');
+  iter   = S.iter;
+  result = S.result;
+  fprintf('Resuming from checkpoint: iter=%d\n', iter);
+else
+  iter   = 0;
+  result = 0;
+  fprintf('Starting fresh (no checkpoint found)\n');
+end
+%% ---------------- Install cleanup handler ----------------
+cleanupObj = onCleanup(@checkpoint_and_exit);
+%% ---------------- Main computation ----------------
+try
+  while iter < MAX_ITER
+    iter = iter + 1;
+    % ---- Simulated work (replace with real computation) ----
+    pause(1);
+    result = result + sqrt(iter);
+    if mod(iter, CHECKPOINT_EVERY) == 0
+      write_checkpoint();
+      fprintf('[periodic] checkpoint saved at iter=%d\n', iter);
+    end
+  end
+  % Final checkpoint
+  write_checkpoint();
+  fprintf('Completed all iterations\n');
+  % Exit code 0 = workflow complete
+  exit(0);
+catch ME
+  fprintf(2, 'ERROR: %s\n', ME.message);
+  exit(1);
+end
+%% ---------------- Local helper functions ----------------
+function write_checkpoint()
+  global iter result TXT_CKPT MAT_CKPT
+  % Application state
+  save(MAT_CKPT, 'iter', 'result');
+  % Orchestration progress (text)
+  fid = fopen(TXT_CKPT, 'w');
+  fprintf(fid, '%d\n', iter);
+  fclose(fid);
+end
+function checkpoint_and_exit()
+  global iter result TXT_CKPT MAT_CKPT
+  fprintf('MATLAB exiting; saving checkpoint at iter=%d\n', iter);
+  write_checkpoint();
+  fprintf('Exit code 99 (checkpoint written)\n');
+  exit(99);
+end
+}}}
+
+
+3. In order to satisfy the amount of RAM required by MATLAB, modify the requested memory in your local copy of checkpoint_runner.sh according to the following.
+
+{{{
+...
+#SBATCH --mem=4G
+...
+}}}
+
+4. Submit the job via the following command.
+
+{{{
+[tulaneID@cypress1 ~]$MODULE_LIST="matlab/r2023a" \
+APP_CMD="matlab -batch checkpoint_signal_iter" \
+CKPT_PATH=state_iter_mat.txt \
+sbatch checkpoint_submitter.sh
+}}}
+
+5. You can monitor job outputs of both the checkpoint_submitter.sh and the checkpoint_runner.sh jobs as they are updated by executing the following '''tail''' command using the '''-F''' option to retry once per second, then using Ctrl+C to exit the retry process entirely.
+
+{{{
+[tulaneID@cypress1 ~]$ tail -F log*
+}}}
+
+== Overriding the default settings ==
+
+To customize your workflow, see [wiki://Workshops/JobCheckpointing/Examples#Overridingdefaultsettings Overriding default settings] for overriding the default settings in the checkpoint submitter according to your needs.
+
+== Sample run outputs ==
+
+=== List of output files ===
+
+Here is a list of all output files from a sample run sorted by time, latest-to-earliest
+
+{{{
+[tulaneID@cypress2 ~]$ls -lt *.csv *.err *.mat *.out *.png *.txt 
+-rw-r--r-- 1 tulaneID groupID   474 Apr 20 15:09 anomaly_details.csv
+-rw-r--r-- 1 tulaneID groupID 26376 Apr 20 15:09 anomaly_stats_table.png
+-rw-r--r-- 1 tulaneID groupID  1383 Apr 20 15:09 log_submitter_3306584.out
+-rw-r--r-- 1 tulaneID groupID 78666 Apr 20 15:09 resource_trend.png
+-rw-r--r-- 1 tulaneID groupID 60274 Apr 20 15:09 progress_with_anomalies.png
+-rw-r--r-- 1 tulaneID groupID   519 Apr 20 15:09 worker_usage.csv
+-rw-r--r-- 1 tulaneID groupID   491 Apr 20 15:09 log_submitter_3306582.err
+-rw-r--r-- 1 tulaneID groupID   760 Apr 20 15:09 log_submitter_3306582.out
+-rw-r--r-- 1 tulaneID groupID     0 Apr 20 15:09 log_submitter_3306584.err
+-rw-r--r-- 1 tulaneID groupID    18 Apr 20 15:09 submitter_state.txt
+-rw-r--r-- 1 tulaneID groupID  1216 Apr 20 15:08 log_runner_3306583.out
+-rw-r--r-- 1 tulaneID groupID   234 Apr 20 15:08 state_iter_mat.mat
+-rw-r--r-- 1 tulaneID groupID     4 Apr 20 15:08 state_iter_mat.txt
+-rw-r--r-- 1 tulaneID groupID     0 Apr 20 15:06 log_runner_3306583.err
+-rw-r--r-- 1 tulaneID groupID   460 Apr 20 15:06 log_submitter_3306578.err
+-rw-r--r-- 1 tulaneID groupID   760 Apr 20 15:06 log_submitter_3306578.out
+-rw-r--r-- 1 tulaneID groupID    40 Apr 20 15:06 worker_ids.txt
+-rw-r--r-- 1 tulaneID groupID  1111 Apr 20 15:05 log_runner_3306580.out
+-rw-r--r-- 1 tulaneID groupID     0 Apr 20 15:03 log_runner_3306580.err
+-rw-r--r-- 1 tulaneID groupID   613 Apr 20 15:03 log_submitter_3306576.err
+-rw-r--r-- 1 tulaneID groupID   760 Apr 20 15:03 log_submitter_3306576.out
+-rw-r--r-- 1 tulaneID groupID  1111 Apr 20 15:03 log_runner_3306577.out
+-rw-r--r-- 1 tulaneID groupID     0 Apr 20 15:01 log_runner_3306577.err
+-rw-r--r-- 1 tulaneID groupID   460 Apr 20 15:01 log_submitter_3306573.err
+-rw-r--r-- 1 tulaneID groupID   760 Apr 20 15:01 log_submitter_3306573.out
+-rw-r--r-- 1 tulaneID groupID  1111 Apr 20 15:00 log_runner_3306574.out
+-rw-r--r-- 1 tulaneID groupID     0 Apr 20 14:58 log_runner_3306574.err
+-rw-r--r-- 1 tulaneID groupID   491 Apr 20 14:58 log_submitter_3306570.err
+-rw-r--r-- 1 tulaneID groupID   767 Apr 20 14:58 log_submitter_3306570.out
+-rw-r--r-- 1 tulaneID groupID  1107 Apr 20 14:58 log_runner_3306571.out
+-rw-r--r-- 1 tulaneID groupID     0 Apr 20 14:56 log_runner_3306571.err
+}}}
+
+=== Sample checkpoint submitter logs ===
+
+Here are the first and last instances from among the multiple log files generated by the sequence of jobs running the job script [wiki://Workshops/JobCheckpointing/Examples#checkpoint_submitter.sh checkpoint_submitter.sh].
+
+The first job in the sequence was submitted by the user, and all other jobs thereafter in that sequence were submitted by the job script itself via '''self-restarting'''.
+
+==== Listing of the initial checkpoint submitter log - starting workflow ====
+
+Here is the listing of the first log file, '''log_submitter_3306570.out''', generated by [wiki://Workshops/JobCheckpointing/Examples#checkpoint_submitter.sh checkpoint_submitter.sh] ending with the message "Submitter approaching timeout -> restarting self. Submitted batch job 3306573."
+
+{{{
+[tulaneID@cypress2 ~]$cat log_submitter_3306570.out 
+Info[20260420-14:56:04]: Start on cypress01-057; JOBID=3306570
+Info[20260420-14:56:04]: Submitter settings:
+Info[20260420-14:56:04]: CKPT_PATH=state_iter_mat.txt
+Info[20260420-14:56:04]: MAX_ITER=500
+Info[20260420-14:56:04]: RUNNER_SCRIPT=checkpoint_runner.sh
+Info[20260420-14:56:04]: SUBMITTER_MARGIN_SEC=60
+Info[20260420-14:56:04]: SUBMITTER_TIME_LIMIT=3:30
+Info[20260420-14:56:04]: No previous state found; starting fresh.
+Info[20260420-14:56:04]: Submitter cycle: 0
+Info[20260420-14:56:04]: Current checkpoint iteration: 0
+Info[20260420-14:56:04]: Submitted worker: 3306571
+Info[20260420-14:56:04]: Waiting on first of worker 3306571 to finish or self-restart.
+Info[20260420-14:58:44]: Submitter approaching timeout -> restarting self
+Submitted batch job 3306573
+}}}
+
+==== Listing of the final checkpoint submitter log - completed workflow ====
+
+Here is the listing of the final log file, '''log_submitter_3306584.out''', generated by [wiki://Workshops/JobCheckpointing/Examples#checkpoint_submitter.sh checkpoint_submitter.sh] ending with the message "Workflow complete. Exiting submitter."
+
+{{{
+[tulaneID@cypress2 ~]$cat log_submitter_3306584.out 
+Info[20260420-15:09:06]: Start on cypress01-057; JOBID=3306584
+Info[20260420-15:09:06]: Submitter settings:
+Info[20260420-15:09:06]: CKPT_PATH=state_iter_mat.txt
+Info[20260420-15:09:06]: MAX_ITER=500
+Info[20260420-15:09:06]: RUNNER_SCRIPT=checkpoint_runner.sh
+Info[20260420-15:09:06]: SUBMITTER_MARGIN_SEC=60
+Info[20260420-15:09:06]: SUBMITTER_TIME_LIMIT=3:30
+Info[20260420-15:09:06]: Loaded state: submitter_cycle=5
+Info[20260420-15:09:06]: Submitter cycle: 5
+Info[20260420-15:09:06]: Current checkpoint iteration: 500
+Info[20260420-15:09:06]: Reached MAX_ITER=500; running final analytics.
+Info[20260420-15:09:06]: Running analytics aggregate_usage.py ...
+Wrote worker_usage.csv
+     JobID  ElapsedSeconds   TotalCPU  ... ReqMem       NodeList   State
+0  3306571             120  00:39.177  ...    4Gn  cypress01-057  FAILED
+1  3306574             121  00:38.906  ...    4Gn  cypress01-057  FAILED
+2  3306577             120  00:39.133  ...    4Gn  cypress01-057  FAILED
+3  3306580             121  00:38.991  ...    4Gn  cypress01-057  FAILED
+4  3306583             120  00:39.425  ...    4Gn  cypress01-057  FAILED
+
+[5 rows x 9 columns]
+Info[20260420-15:09:08]: Running analytics plot_progress.py ...
+Saved progress_with_anomalies.png
+Saved resource_trend.png
+Saved anomaly_details.csv
+Saved anomaly_stats_table.png
+Info[20260420-15:09:12]: Workflow complete. Exiting submitter.
+}}}
+
+=== Sample checkpoint runner logs ===
+
+Here are the first and last output logs of the sequence of jobs generated by the job script [wiki://Workshops/JobCheckpointing/Examples#checkpoint_runner.sh checkpoint_runner.sh].
+
+All of the jobs in that sequence were submitted by the [wiki://Workshops/JobCheckpointing/Examples#checkpoint_submitter.sh checkpoint_submitter.sh] job script.
+
+==== Listing of the initial checkpoint runner log - starting application ====
+
+Here is the listing of the intial log file, '''log_runner_3306571.out''', generated by the job script [wiki://Workshops/JobCheckpointing/Examples#checkpoint_runner.sh checkpoint_runner.sh] ending with message "Timeout + checkpoint advance detected (0->100)."
+
+{{{
+[tulaneID@cypress2 ~]$cat log_runner_3306571.out 
+Info[20260420-14:56:04]: Start on cypress01-057; JOBID=3306571
+Info[20260420-14:56:04]: Runner settings:
+Info[20260420-14:56:04]: APP_CMD=matlab -batch checkpoint_signal_iter
+Info[20260420-14:56:04]: CHECKPOINT_EVERY=20
+Info[20260420-14:56:04]: CKPT_PATH=state_iter_mat.txt
+Info[20260420-14:56:04]: LAUNCH_MODE=direct
+Info[20260420-14:56:04]: MAX_ITER=500
+Info[20260420-14:56:04]: MODULE_LIST=matlab/r2023a
+Info[20260420-14:56:04]: RUNNER_MARGIN_SEC=60
+Info[20260420-14:56:04]: RUNNER_TIME_LIMIT=00:03:00
+Info[20260420-14:56:04]: SRUN_ARGS=-n 1
+Info[20260420-14:56:04]: Loading modules.
+MATLAB checkpointing application
+ Text checkpoint: state_iter_mat.txt
+ State checkpoint: state_iter_mat.mat
+ MAX_ITER=500, CHECKPOINT_EVERY=20
+Starting fresh (no checkpoint found)
+[periodic] checkpoint saved at iter=20
+[periodic] checkpoint saved at iter=40
+[periodic] checkpoint saved at iter=60
+[periodic] checkpoint saved at iter=80
+[periodic] checkpoint saved at iter=100
+Info[20260420-14:58:04]: Program exit code (from timeout wrapper): 124
+Info[20260420-14:58:04]: Timeout + checkpoint advance detected (0->100).
+}}}
+
+==== Listing of the final checkpoint runner log - completed application ====
+
+Here is the listing of the final log file, '''log_runner_3303171.out''', generated by [wiki://Workshops/JobCheckpointing/Examples#checkpoint_runner.sh checkpoint_runner.sh] ending with message "Timeout + checkpoint advance detected (400->500)."
+
+{{{
+[tulaneID@cypress2 ~]$cat log_runner_3306583.out 
+Info[20260420-15:06:26]: Start on cypress01-057; JOBID=3306583
+Info[20260420-15:06:26]: Runner settings:
+Info[20260420-15:06:26]: APP_CMD=matlab -batch checkpoint_signal_iter
+Info[20260420-15:06:26]: CHECKPOINT_EVERY=20
+Info[20260420-15:06:26]: CKPT_PATH=state_iter_mat.txt
+Info[20260420-15:06:26]: LAUNCH_MODE=direct
+Info[20260420-15:06:26]: MAX_ITER=500
+Info[20260420-15:06:26]: MODULE_LIST=matlab/r2023a
+Info[20260420-15:06:26]: RUNNER_MARGIN_SEC=60
+Info[20260420-15:06:26]: RUNNER_TIME_LIMIT=00:03:00
+Info[20260420-15:06:26]: SRUN_ARGS=-n 1
+Info[20260420-15:06:26]: Loading modules.
+MATLAB checkpointing application
+ Text checkpoint: state_iter_mat.txt
+ State checkpoint: state_iter_mat.mat
+ MAX_ITER=500, CHECKPOINT_EVERY=20
+Resuming from checkpoint: iter=400
+[periodic] checkpoint saved at iter=420
+[periodic] checkpoint saved at iter=440
+[periodic] checkpoint saved at iter=460
+[periodic] checkpoint saved at iter=480
+[periodic] checkpoint saved at iter=500
+Completed all iterations
+MATLAB exiting; saving checkpoint at iter=500
+Exit code 99 (checkpoint written)
+Info[20260420-15:08:26]: Program exit code (from timeout wrapper): 124
+Info[20260420-15:08:26]: Timeout + checkpoint advance detected (400->500).
+}}}
+
+=== Listings of post-workflow analysis output files ===
+
+The following is the complete, annotated listing of all the generated .csv and .txt files
+
+{{{
+[tulaneID@cypress2 ~]$tail -n +1 *.csv *.txt
+==> anomaly_details.csv <==
+JobID,StartTime,ElapsedSeconds,MaxRSS_MiB,TimeZ,MemModZ,AnomalyType
+3306571,2026-04-20 14:56:04,120.0,1141.998291015625,-0.8164965809277377,0.0,none
+3306574,2026-04-20 14:58:44,121.0,1146.6522216796875,1.2247448713915774,0.0,none
+3306577,2026-04-20 15:01:15,120.0,1188.140869140625,-0.8164965809277377,0.0,none
+3306580,2026-04-20 15:03:55,121.0,1143.268585205078,1.2247448713915774,0.0,none
+3306583,2026-04-20 15:06:26,120.0,1155.21240234375,-0.8164965809277377,0.6745,none
+
+==> worker_usage.csv <==
+JobID,ElapsedSeconds,TotalCPU,MaxRSS,MaxRSS_bytes,MaxRSS_MiB,ReqMem,NodeList,State
+3306571,120,00:39.177,1197472K,1197472000.0,1141.998291015625,4Gn,cypress01-057,FAILED
+3306574,121,00:38.906,1202352K,1202352000.0,1146.6522216796875,4Gn,cypress01-057,FAILED
+3306577,120,00:39.133,1245856K,1245856000.0,1188.140869140625,4Gn,cypress01-057,FAILED
+3306580,121,00:38.991,1198804K,1198804000.0,1143.2685852050781,4Gn,cypress01-057,FAILED
+3306583,120,00:39.425,1211328K,1211328000.0,1155.21240234375,4Gn,cypress01-057,FAILED
+
+==> state_iter_mat.txt <==
+500
+
+==> submitter_state.txt <==
+submitter_cycle=5
+
+==> worker_ids.txt <==
+3306571
+3306574
+3306577
+3306580
+3306583
+}}}
+
+=== Post-workflow analysis plots ===
+
+==== Background ====
+
+For background on the following plots, see [wiki://Workshops/JobCheckpointing/Examples#Plottingexecutiontimeandmemoryusage Plotting execution time and memory usage].
+
+==== Table of anomalies, compute time and memory usage ====
+
+Here is the plot, '''anomaly_stats_table.png''', showing the table of anomalies, if any, from among all of the [wiki://Workshops/JobCheckpointing/Examples#checkpoint_runner.sh checkpoint_runner.sh] jobs.
+
+[[Image(MATLAB_anomaly_stats_table.png, width=800px)]]
+
+==== Compute progress ====
+
+Here is the plot, '''progress_with_anomalies.png''', showing '''Checkpoint Progress''' and '''MaxRSS''' (SLURM max job memory usage) vs '''Wall Time'''.
+
+[[Image(MATLAB_progress_with_anomalies.png, width=800px)]]
+
+==== Execution time and memory usage trends ====
+
+Here is the plot, '''resource_trend.png''', showing '''Elapsed (time)''' and '''MaxRSS''' vs. '''Worker job index'''.
+
+[[Image(MATLAB_resource_trend.png, width=800px)]]