Changes between Version 7 and Version 8 of Workshops/cypress/JobArrays

Timestamp:

08/16/25 16:52:42 (3 months ago)

Author:

fuji

Comment:

—

Workshops/cypress/JobArrays

@@ -19 +19 @@
 }}}
 
-There is a python script that is
+There is a Python script that is
 {{{
 [fuji@cypress1 JobArray1]$ cat hello2.py
@@ -126 +126 @@
 === Use Array Task ID to define the script file name ===
 
-Get into '''JobArray2''' directory under '''workshop''',
-{{{
-[fuji@cypress1 ~]$ cd workshop/JobArray2/
+Get into '''JobArray2''' directory under '''hpc-workshop''',
+{{{
+[fuji@cypress1 ~]$ cd hpc-workshop/JobArray2/
 [fuji@cypress1 JobArray2]$ ls
 hello2.py    script01.sh  script03.sh  script05.sh  script07.sh  script09.sh   slurmscript2
@@ -160 +160 @@
 }}}
 
+=== Use Array Task ID to identify the data file ===
+
+Get into '''JobArray3''' directory under '''hpc-workshop''',
+{{{
+[fuji@cypress1 ~]$ cd hpc-workshop/JobArray3/
+[fuji@cypress1 JobArray3]$ ls
+data  slurmscript
+}}}
+
+In '''data''' directory,
+{{{
+[fuji@cypress1 JobArray3]$ ls data
+data_file_10.txt  data_file_2.txt  data_file_4.txt  data_file_6.txt  data_file_8.txt
+data_file_1.txt   data_file_3.txt  data_file_5.txt  data_file_7.txt  data_file_9.txt
+}}}
+'''slurmscript'''
+{{{#!bash
+#!/bin/bash
+#SBATCH --qos=workshop          # Quality of Service
+#SBATCH --partition=workshop    # partition
+#SBATCH --job-name=job_array    # Job Name
+#SBATCH --time=00:01:00         # WallTime
+#SBATCH --nodes=1               # Number of Nodes
+#SBATCH --ntasks-per-node=1     # Number of tasks (MPI processes)
+#SBATCH --cpus-per-task=1       # Number of threads per task (OMP threads)
+#SBATCH --array=1-10            # Array of IDs=1,2,...10
+
+# list all (10) files in a data directory and use a job array to process each file.
+# define the data directory
+DATA_DIRECTORY=./data
+echo Using DATA_DIRECTORY=$DATA_DIRECTORY
+echo Using SLURM_ARRAY_TASK_ID=$SLURM_ARRAY_TASK_ID
+# select the data file from the data directory using the SLURM task ID
+DATA_FILE=$(find $DATA_DIRECTORY -type f | sort -V | sed -n "$SLURM_ARRAY_TASK_ID p")
+echo Using DATA_FILE=$DATA_FILE.
+# define the output directory
+OUTPUT_DIRECTORY=./output
+mkdir -p $OUTPUT_DIRECTORY
+echo Using OUTPUT_DIRECTORY=$OUTPUT_DIRECTORY
+OUTPUT_FILE=$OUTPUT_DIRECTORY/$(basename $DATA_FILE).out
+# if the output file already exists, then bypass and exit
+echo Checking for OUTPUT_FILE=$OUTPUT_FILE...
+if [ -f $OUTPUT_FILE ]; then
+   echo Found. Bypassing processing.
+else
+   echo Not found. Processing.
+   sed -r 's/(.*)/\1 output/' $DATA_FILE >> $OUTPUT_FILE
+   echo Done.
+fi
+}}}
 === Cancel Jobs in Job Array ===
  Look at '''slurmscript12'''