Changes between Version 14 and Version 15 of cypress/XeonPhi

Timestamp:

08/21/15 09:47:52 (9 years ago)

Author:

cmaggio

Comment:

—

cypress/XeonPhi

@@ -255 +255 @@
 }}}
 
+
+
+
+
+
 === Native Programming ===
 
 The native model centers around the notion that each MIC is its own machine with it's own architecture. The first challenge is to compile code to run specifically on the hardware of the MIC.
 
-
-
-SLURM jobscript script is, for example,
-
+Let's start out in interactive mode and then we will look at job submission:
 {{{#!bash
-#!/bin/bash
-#SBATCH --qos=normal            # Quality of Service
-#SBATCH --job-name=nativeTest   # Job Name
-#SBATCH --time=00:10:00         # WallTime
-#SBATCH --nodes=1               # Number of Nodes
-#SBATCH --ntasks-per-node=1     # Number of tasks (MPI presseces)
-#SBATCH --cpus-per-task=1       # Number of processors per task OpenMP threads()
-#SBATCH --gres=mic:1            # Number of Co-Processors
-
-micnativeloadex ./myNativeExecutable -e "OMP_NUM_THREADS=100" -d 0 -v
-}}}
-
-
-
-In the script above we request one MIC device that will be device number 0. 
-"micnativeloadex" command launches MIC native executable. "-e "OMP_NUM_THREADS=100"" option to set the number of threads on the MIC device to 100.
-For more options, see below.
-
-{{{#!bash
-[fuji@cypress01-090 nativeTest]$ micnativeloadex -h
+export MY_PARTITION=workshop
+export MY_QUEUE=workshop
+idev -c 4 --gres=mic:0
+}}}
+
+
+
+* Load the intel modules
+ * intel-psxe
+ * intel/mic/sdk/3.3
+ * intel/mic/runtime/3.3
+
+And let's examine the code helloflops3.c from Jim Jeffers and James Rainers seminal text 
+[[http://lotsofcores.com/|Intels Xeon Phi Coprocessor High - Performance Programming]]
+
+{{{#!c
+//
+//
+// helloflops2
+//
+// A simple example that gets lots of Flops (Floating Point Operations) on 
+// Intel(r) Xeon Phi(tm) co-processors using openmp to scale
+//
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <omp.h>
+#include <sys/time.h>
+
+// dtime 
+// 
+// returns the current wall clock time
+//
+double dtime()
+{
+    double tseconds = 0.0;
+    struct timeval mytime;
+    gettimeofday(&mytime,(struct timezone*)0);
+    tseconds = (double)(mytime.tv_sec + mytime.tv_usec*1.0e-6);
+    return( tseconds );
+}
+
+#define FLOPS_ARRAY_SIZE (1024*1024) 
+#define MAXFLOPS_ITERS 100000000
+#define LOOP_COUNT 128
+
+// number of float pt ops per calculation
+#define FLOPSPERCALC 2     
+// define some arrays - 
+// make sure they are 64 byte aligned
+// for best cache access 
+float fa[FLOPS_ARRAY_SIZE] __attribute__((aligned(64)));
+float fb[FLOPS_ARRAY_SIZE] __attribute__((aligned(64)));
+//
+// Main program - pedal to the metal...calculate using tons o'flops!
+// 
+int main(int argc, char *argv[] ) 
+{
+    int i,j,k;
+    int numthreads;
+    double tstart, tstop, ttime;
+    double gflops = 0.0;
+    float a=1.1;
+
+    //
+    // initialize the compute arrays 
+    //
+    //
+
+#pragma omp parallel
+#pragma omp master
+    numthreads = omp_get_num_threads();
+
+    printf("Initializing\r\n");
+#pragma omp parallel for
+    for(i=0; i<FLOPS_ARRAY_SIZE; i++)
+    {
+        fa[i] = (float)i + 0.1;
+        fb[i] = (float)i + 0.2;
+    }   
+    printf("Starting Compute on %d threads\r\n",numthreads);
+
+    tstart = dtime();
+        
+    // scale the calculation across threads requested 
+    // need to set environment variables OMP_NUM_THREADS and KMP_AFFINITY
+
+#pragma omp parallel for private(j,k)
+    for (i=0; i<numthreads; i++)
+    {
+        // each thread will work it's own array section
+        // calc offset into the right section
+        int offset = i*LOOP_COUNT;
+
+        // loop many times to get lots of calculations
+        for(j=0; j<MAXFLOPS_ITERS; j++)  
+        {
+            // scale 1st array and add in the 2nd array 
+            for(k=0; k<LOOP_COUNT; k++)  
+            {
+                fa[k+offset] = a * fa[k+offset] + fb[k+offset];
+            }
+        }
+    }
+    tstop = dtime();
+    // # of gigaflops we just calculated  
+    gflops = (double)( 1.0e-9*numthreads*LOOP_COUNT*
+                        MAXFLOPS_ITERS*FLOPSPERCALC);    
+
+    //elasped time
+    ttime = tstop - tstart;
+    //
+    // Print the results
+    //
+    if ((ttime) > 0.0)
+    {
+        printf("GFlops = %10.3lf, Secs = %10.3lf, GFlops per sec = %10.3lf\r\n",                   gflops, ttime, gflops/ttime);
+    }
+    return( 0 );
+}
+}}}
+
+
+Lets begin by compiling the code for the host and running it on the host processor
+{{{
+[tuhpc002@cypress01-089 Day2]$ icc -O3 -openmp helloflops3.c -o helloflops3_host
+[tuhpc002@cypress01-089 Day2]$ ./helloflops3_host 
+Initializing
+Starting Compute on 20 threads
+GFlops =    512.000, Secs =      6.349, GFlops per sec =     80.645
+[tuhpc002@cypress01-089 Day2]$ 
+}}}
+
+Not bad. Now lets compile a native version by adding the architecture flag '''-mmic'''.
+{{{
+[tuhpc002@cypress01-089 Day2]$ icc -O3 -openmp -mmic helloflops3.c -o helloflops3_mic
+[tuhpc002@cypress01-089 Day2]$ ./helloflops3_mic 
+-bash: ./helloflops3_mic: cannot execute binary file
+[tuhpc002@cypress01-089 Day2]$ 
+}}}
+
+We could not execute our binary because it we tried to run it on the host architecture. We could ssh onto one of the MICs and manually set our library environment variables to run the code, but a cleaner method (and the recommended method) is to use the intel program '''micnativeloadex'''.
+
+{{{
+[tuhpc002@cypress01-089 Day2]$ micnativeloadex ./helloflops3_mic
+Initializing
+Starting Compute on 240 threads
+GFlops =   6144.000, Secs =      2.630, GFlops per sec =   2335.925
+
+[tuhpc002@cypress01-089 Day2]$ 
+}}}
+
+'''micnativeloadex''' has a number of options which can be seen using the '''-h''' or help flag.
+
+{{{
+[tulaneID@cypress01 $ micnativeloadex -h
 
 Usage:
@@ -304 +443 @@
 }}}
 
+Notice that we can use the '''-d''' flag to select which MIC we want to run on and the '''-e''' flag to set environment variables on the MIC (separated by whitespace). For example, we can choose to run on MIC0 and set the number of threads and their affinity with
+
+{{{
+[tuhpc002@cypress01-089 Day2]$ micnativeloadex ./helloflops3_mic -e "OMP_NUM_THREADS=120 KMP_AFFINITY=scatter" -d 0
+Initializing
+Starting Compute on 120 threads
+GFlops =   3072.000, Secs =      1.500, GFlops per sec =   2048.143
+
+[tuhpc002@cypress01-089 Day2]$ 
+}}}
+
+
+We've been using idev as an instructional tool, but we won't normally be running our MIC native jobs interactively. Rather, we'll be submitting jobs that we want to run in native mode on a compute node. An example SLURM jobscript script for our code would look like
+
+{{{#!bash
+#!/bin/bash
+#SBATCH --qos=workshop          # Quality of Service
+#SBATCH --partition=workshop    #Partition
+#SBATCH --job-name=nativeTest   # Job Name
+#SBATCH --time=00:10:00         # WallTime
+#SBATCH --nodes=1               # Number of Nodes
+#SBATCH --ntasks-per-node=1     # Number of tasks (MPI presseces)
+#SBATCH --cpus-per-task=1       # Number of processors per task OpenMP threads()
+#SBATCH --gres=mic:1            # Number of Co-Processors
+
+module load intel-psxe/2015-update1
+module load intel/mic/sdk/3.3
+module load intel/mic/runtime/3.3
+
+micnativeloadex ./helloflops3_mic -e "OMP_NUM_THREADS=120 KMP_AFFINITY=scatter" -d 0 
+}}}
+
+
+
+
+
+
+
 
 
 
 === Offloading ===
+
+
+
+
+