Example3-Compilation

Compilation of different API features, including compilation.

Function run_migration_equilibrium_simulation, implemented in examples/example_compilation/run.cu, runs a simulation given a mutation rate and outputs result into allele trajectory a. The information from a is then copied to b and a is subsequently deleted. The information from b is then output to a file then to the terminal.

The main purpose of this is to show that simulation parameters can be passed from a standard C++ program to a CUDA program (*.cu), where the GPU-accelerated simulation is run, and then the simulation results in the form of an allele trajectory can be passed back to the C++ program and manipulated. So accelerated simulations can be inserted into any current C/C++ project. go_fish_data_struct.h is included in run.h (which is subsequently included in main.cpp).

/*
 * main.cpp
 *
 *      Author: David Lawrie
 */

#include "run.h"
#include <fstream>

int main(int argc, char **argv){
    GO_Fish::allele_trajectories a;
    a.sim_input_constants.seed1 = 0xbeeff00d;                   //random number seeds
    a.sim_input_constants.seed2 = 0xdecafbad;

    float migration_rate = 0.005;                               //migration proportion (equal between populations)
    run_migration_equilibrium_simulation(a,migration_rate);     //run simulation
    GO_Fish::allele_trajectories b = a;                         //copy-assignment, copies a to b (unneeded, just showing it is possible)
    a.reset();                                                  //frees memory held by a (unneeded, just showing it is possible)

    /* --- output simulation information --- */
    std::ofstream outfile;
    outfile.open("bfile.dat");
    outfile<<b;                                                 //prints full allele_trajectory b to file
    outfile.close();

    std::cout<<std::endl<<"number of time samples: " << b.num_time_samples();
    std::cout<<std::endl<<"mutations in first time sample: " << b.num_mutations_time_sample(0) <<std::endl<<"mutations in final time sample: " << b.maximal_num_mutations() << std::endl; //mutations in final time sample >= mutations in first time sample as all mutations in the latter were preserved by sampling

    //prints the first 10 mutations output from burn-in simulation
    std::cout<<std::endl<<"mutations from burn-in simulation\n";
    int mutation_range_begin = 0; int mutation_range_end = mutation_range_begin+10;
    std::cout<<"ID\tstart frequency pop 1\tstart frequency pop 2\tfinal frequency pop 1\tfinal frequency pop 2"<<std::endl;
    for(int i = mutation_range_begin; i < mutation_range_end; i++){ std::cout<<b.mutation_ID(i)<<"\t"<<b.frequency(0,0,i)<<"\t"<<b.frequency(0,1,i)<<"\t"<<b.frequency(1,0,i)<<"\t"<<b.frequency(1,1,i)<<std::endl; }

    //prints the first 10 mutations output from scenario simulation
    std::cout<<std::endl<<"mutations from scenario simulation\n";
    mutation_range_begin = b.num_mutations_time_sample(0); mutation_range_end = mutation_range_begin+10;
    std::cout<<"ID\tstart frequency pop 1\tstart frequency pop 2\tfinal frequency pop 1\tfinal frequency pop 2"<<std::endl;
    for(int i = mutation_range_begin; i < mutation_range_end; i++){ std::cout<<b.mutation_ID(i)<<"\t"<<b.frequency(0,0,i)<<"\t"<<b.frequency(0,1,i)<<"\t"<<b.frequency(1,0,i)<<"\t"<<b.frequency(1,1,i)<<std::endl; }
    /* --- end output simulation information --- */
}

The simulation starts off blank, running a burn-in simulation until equilibrium is reached, then running the scenario of interest.

/*
 * run_sim.h
 *
 *      Author: dlawrie
 */

#include "go_fish_data_struct.h"

#ifndef RUN_SIM_H_
#define RUN_SIM_H_

void run_migration_equilibrium_simulation(GO_Fish::allele_trajectories & a, float migration_rate);

#endif /* RUN_SIM_H_ */

The below code for example_compilation/run.cu must also include go_fish.cuh in order to run the simulations as including run.h only includes the GO_Fish data structures.

/*
 * run.cu
 *
 *      Author: David Lawrie
 */

#include "go_fish.cuh"
#include "run.h"

//scenario: two populations start in mutation-selection-migration equilibrium with weakly deleterious mutations, then simulate what happens after mutations in population 1 become beneficial (1,000 generations)
//migration set in main.cpp
void run_migration_equilibrium_simulation(GO_Fish::allele_trajectories & a, float migration_rate){
    using namespace Sim_Model;                              //using namespace Sim_Model (avoids Sim_Model::, but compiler can get confused if multiple functions from different namespaces have same name and Sim_Model:: shows ownership)
    using namespace GO_Fish;                                //using namespace GO_Fish (avoids GO_Fish::, but compiler can get confused if multiple functions from different namespaces have same name and GO_Fish:: shows ownership)

    a.sim_input_constants.num_sites = 20*pow(10.f,7);       //number of sites
    a.sim_input_constants.num_populations = 2;              //number of populations

    a.sim_input_constants.init_mse = false;                 //start from blank simulation
    F_mu_h_constant mutation(pow(10.f,-9));                 //per-site mutation rate
    F_mu_h_constant inbreeding(1.f);                        //constant inbreeding
    demography_constant demography(1000);                   //number of individuals in both populations
    migration_constant_equal migration(migration_rate,a.sim_input_constants.num_populations); //constant migration rate
    selection_constant deleterious(-1.f/1000.f);            //constant selection coefficient (weakly deleterious)
    F_mu_h_constant dominance(0.f);                         //constant allele dominance (ignored as population is fully inbred)
    bool_off dont_preserve;                                 //don't preserve mutations
    bool_off dont_sample;                                   //don't sample generations
    a.sim_input_constants.compact_interval = 100;           //interval between compacts

    a.sim_input_constants.num_generations = 2*pow(10.f,4);  //burn-in simulation to achieve migration equilibrium 20,0000 generations
    run_sim(a,mutation,demography,migration,deleterious,inbreeding,dominance,dont_preserve,dont_sample); //only sample final generation

    allele_trajectories c(a);                               //copy constructor, copies a to c (not actually needed for this simulation, just showing it is possible)

    bool_on sample;                                         //sample generation
    bool_pulse<bool_off,bool_on> sample_strategy(dont_sample,sample,0,a.sim_input_constants.num_generations); //sample starting generation of second simulation (i.e. last generation of burn-in simulation)
    a.sim_input_constants.num_generations = pow(10.f,3);    //scenario simulation 1,0000 generations
    a.sim_input_constants.prev_sim_sample = 0;              //start from previous simulation time sample 0
    selection_constant beneficial(20.f/1000.f);             //constant selection coefficient (beneficial)
    selection_population_specific<selection_constant,selection_constant> selection_model(deleterious,beneficial,1); //selection in population 1
    run_sim(a,mutation,demography,migration,selection_model,inbreeding,dominance,dont_preserve,sample_strategy,a); //scenario simulation, start from migration equilibrium, sample both start and final generations
}

Pro Tip: A nice thing about CUDA is the flexibility it gives in compiling mixed C/C++ and CUDA programs. Below is an example where the entire program can alternatively be compiled by NVCC (the NVIDIA CUDA compiler) - e.g. makefile_nvcc - or by g++ & NVCC where NVCC compiles just the GPU-accelerated portions and then hands the linking over g++ - e.g. makefile_gpp. The master makefile controls which is called. This is useful if you are adding the simulation to your own project compiled with a certain compiler that you do not want to change for the code you have written. The CUDA code essentially becomes an accelerated library that you can use in your current project. Master makefile, each line is documented by the top part of the makefile:

# Description of Mac/Linux/Unix Makefile for example_compilation.
# makefile_nvcc is useful when combining GPU-accelerated code into an existing C++ project and using NVCC to compile all files and link objects into executable.
# makefile_gpp is useful when combining GPU-accelerated code into an existing C++ project and user wants to keep non-CUDA code compiled by a different compiler (e.g. g++). 
#
#############################
# build_path := Where to build program and put executable (note: folder must already exist)
# EXEC_FILE := Name of executable 
#
# cpp_object_names := CPP objects required for executable
# cpp_objects := Prepends build path to object names
#
# cuda_object_names := CUDA objects required for executable
# cuda_objects := Prepends build path to object names
#
# nvcc := Command 'make' or 'make nvcc' calls the makefile makefile_nvcc
#
# gpp := Command 'make gpp' calls the makefile makefile_gpp
#
# .PHONY := Defines 'nvvc', 'gpp', and 'clean' as not true targets (i.e. don't remake executable if can't find files called 'all' or 'clean')
#
# clean := Action to perform for command 'make clean'
#  Remove all objects and EXEC_FILE from build_path
#############################

build_path = ../example_compilation
EXEC_FILE = GOFish

cpp_object_names = main.o
cpp_objects = $(addprefix $(build_path)/,$(cpp_object_names))

cuda_object_names = run.o shared.o go_fish_impl.o
cuda_objects = $(addprefix $(build_path)/,$(cuda_object_names))

nvcc: 
   make -f makefile_nvcc
gpp:
   make -f makefile_gpp

.PHONY: nvcc gpp clean

clean:
   rm -f $(cuda_objects) $(cpp_objects) $(build_path)/gpuCode.o $(build_path)/$(EXEC_FILE)

makefile_nvcc, each line is documented by the top part of the makefile:

Tip: The makefile below compiles machine code explicitly for generation 3.0 and 5.2 GPUs and uses just in time (JIT) compilation for everything else (lowest GPU generation which works for 3P is 3.0). Compilation (and program execution) will be faster if compiling for your specific GPU.

e.g. if running a Tesla K20 or Tesla K40, then the corresponding GPU generation is 3.5: all the current --generate-code arch=##,code=## flags can be deleted and replaced with --generate-code arch=compute_35,code=sm_35.

# Description of Mac/Linux/Unix Makefile for example_compilation. 
# NVCC compiles C++ and CUDA code and links all objects into an exectuable.
#
#############################
# build_path := Where to build program and put executable (note: folder must already exist)
# api_path_source := Location of API source folder
# api_path_include := Location of API include folder
# EXEC_FILE := Name of executable 
#
# NVCC := Compiler path, in this case nvcc is in $PATH
# CFLAGS_CUDA := Compiler Flags for CUDA (*.cu) Files: optimize most, fast math, add API include folder to include search path, equivalent to --relocatable-device-code=true --compile
# CFLAGS_CPP := Compiler Flags for CPP (main.cpp) Files: optimize most, add API include folder to include search path, compile
# CODE := GPU types for which to build explicitly (I have a NVIDIA GTX 780M and 980) https://developer.nvidia.com/cuda-gpus, creates machine code for code=sm_30 (780) and code=sm_52 (980) and virtual architectures for all other generations which can be compiled JIT - code=compute_30 for generations between (3.0,5.0) and code=compute_50 for generations (5.0 and up) http://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation
#
# cpp_object_names := CPP objects required for executable
# cpp_objects := Prepends build path to object names
#
# cuda_object_names := CUDA objects required for executable
# cuda_objects := Prepends build path to object names
#
# all := Command 'make' or 'make all' builds executable file $(EXEC_FILE) in location $(build_path)/
#
# ##### Object Dependencies Lists #####
# If one of the files on the right hand side of the : changes, the corresponding object must be recompiled.
# This is a users makefile - it assumes there will be no changes to the GOFish API files, so does not include
# all the non-*.cu (non source file) dependencies except those that the user created (i.e. run.h). If changes 
# to the API .h or .cuh files are expected see Object Dependencies Lists in example_dadi or run 'make clean' 
# before each 'make all'.
# ##### End Object Dependencies Lists #####
#
# $(cpp_objects) := Make target all objects
#  Compile CPP source code into objects, $< := dependencies from Object Dependencies Lists, $@ := object in $(objects)
#
# $(cuda_objects) := Make target all objects
#  Compile CUDA source code into objects, $< := dependencies from Object Dependencies Lists, $@ := object in $(objects)
#
# $(build_path)/$(EXEC_FILE) := Make target executable EXEC_FILE which depends on all objects
#  Link CUDA/CPP objects into executable EXEC_FILE, $@ := $(build_path)/$(EXEC_FILE)
#
# .PHONY := Defines 'all' as not a true target (i.e. don't remake executable if can't find a file called 'all')
#############################

build_path = ../example_compilation
api_path_source = ../../3P/_internal
api_path_include = ../../3P
EXEC_FILE = GOFish

NVCC = nvcc
CFLAGS_CUDA = -O3 --use_fast_math -I $(api_path_include)/ -dc
CFLAGS_CPP = -O3 -I $(api_path_include)/ -c
CODE = --generate-code arch=compute_30,code=sm_30 --generate-code arch=compute_52,code=sm_52 --generate-code arch=compute_30,code=compute_30 --generate-code arch=compute_52,code=compute_52

cpp_object_names = main.o
cpp_objects = $(addprefix $(build_path)/,$(cpp_object_names))

cuda_object_names = run.o shared.o go_fish_impl.o
cuda_objects = $(addprefix $(build_path)/,$(cuda_object_names))

all: $(build_path)/$(EXEC_FILE)

##### OBJECT DEPENDENCIES #####
$(build_path)/main.o: main.cpp run.h
$(build_path)/run.o: run.cu run.h
$(build_path)/shared.o: $(api_path_source)/shared.cu
$(build_path)/go_fish_impl.o: $(api_path_source)/go_fish_impl.cu
##### END OBJECT DEPENDENCIES #####

$(cpp_objects):
   $(NVCC) $(CFLAGS_CPP) $< -o $@

$(cuda_objects):
   $(NVCC) $(CODE) $(CFLAGS_CUDA) $< -o $@

$(build_path)/$(EXEC_FILE): $(cpp_objects) $(cuda_objects)

   $(NVCC) $(CODE) $(cpp_objects) $(cuda_objects)  -o $@

.PHONY: all

makefile_gpp, each line is documented by the top part of the makefile:

Tip: The makefile below compiles machine code explicitly for generation 3.0 and 5.2 GPUs and uses just in time (JIT) compilation for everything else (lowest GPU generation which works for 3P is 3.0). Compilation (and program execution) will be faster if compiling for your specific GPU.

e.g. if running a Tesla K20 or Tesla K40, then the corresponding GPU generation is 3.5: all the current --generate-code arch=##,code=## flags can be deleted and replaced with --generate-code arch=compute_35,code=sm_35.

# Description of Mac/Linux/Unix Makefile for example_compilation using g++ to compile the C++ code and link objects into executable. 
# NVCC compiles CUDA code and links device (GPU) code into an object that can be understood by g++ with the help of the CUDA Runtime Library.
#
#############################
# build_path := Where to build program and put executable (note: folder must already exist)
# api_path_source := Location of API source folder
# api_path_include := Location of API include folder
# EXEC_FILE := Name of executable 
#
# CC := Compiler, in this case g++
# NVCC := Compiler path, in this case nvcc is in $PATH
# CFLAGS_CUDA := Compiler Flags for CUDA (*.cu) Files: optimize most, fast math, add API include folder to include search path, equivalent to --relocatable-device-code=true --compile
# CFLAGS_CPP := Compiler Flags for CPP (main.cpp) Files: optimize most, add API include folder to include search path, compile
# CODE := GPU types for which to build explicitly (I have a NVIDIA GTX 780M and 980) https://developer.nvidia.com/cuda-gpus, creates machine code for code=sm_30 (780) and code=sm_52 (980) and virtual architectures for all other generations which can be compiled JIT - code=compute_30 for generations between (3.0,5.0) and code=compute_50 for generations (5.0 and up) http://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html#options-for-steering-gpu-code-generation
# LIB_CUDART := Location of the CUDA Runtime API
# 
# cpp_object_names := CPP objects required for executable
# cpp_objects := Prepends build path to object names
#
# cuda_object_names := CUDA objects required for executable
# cuda_objects := Prepends build path to object names
#
# all := Command 'make' or 'make all' builds executable file $(EXEC_FILE) in location $(build_path)/
#
# ##### Object Dependencies Lists #####
# If one of the files on the right hand side of the : changes, the corresponding object must be recompiled.
# This is a users makefile - it assumes there will be no changes to the GOFish API files, so does not include
# all the non-*.cu (non source file) dependencies except those that the user created (i.e. run.h). If changes 
# to the API .h or .cuh files are expected see Object Dependencies Lists in example_dadi or run 'make clean' 
# before each 'make all'.
# ##### End Object Dependencies Lists #####
#
# $(cpp_objects) := Make target all objects
#  Use g++ to compile CPP source code into objects, $< := dependencies from Object Dependencies Lists, $@ := object in $(objects)
#
# $(cuda_objects) := Make target all objects
#  Use nvcc to compile CUDA source code into objects, $< := dependencies from Object Dependencies Lists, $@ := object in $(objects)
#
# $(build_path)/gpuCode.o := Make target CUDA object which depends on cuda objects
#  Use nvcc to link device code from CUDA objects into object gpuCode.o, $@ := $(build_path)/gpuCode.o
#
# $(build_path)/$(EXEC_FILE) := Make target executable EXEC_FILE which depends on all objects
#  Use g++ to link CUDA/CPP objects into executable EXEC_FILE reintegrating device code using gpuCode.o and the CUDA runtime library, $@ := $(build_path)/$(EXEC_FILE)
#
# .PHONY := Defines 'all' as not a true target (i.e. don't remake executable if can't find a file called 'all')
#############################

build_path = ../example_compilation
api_path_source = ../../3P/_internal
api_path_include = ../../3P
EXEC_FILE = GOFish

CC = g++
NVCC = nvcc
CFLAGS_CUDA = -O3 --use_fast_math -I $(api_path_include)/ -dc
CFLAGS_CPP = -O3 -I $(api_path_include)/ -c
CODE = --generate-code arch=compute_30,code=sm_30 --generate-code arch=compute_52,code=sm_52 --generate-code arch=compute_30,code=compute_30 --generate-code arch=compute_52,code=compute_52
LIB_CUDART = -L/usr/local/cuda/lib -lcudart

cpp_object_names = main.o
cpp_objects = $(addprefix $(build_path)/,$(cpp_object_names))

cuda_object_names = run.o shared.o go_fish_impl.o
cuda_objects = $(addprefix $(build_path)/,$(cuda_object_names))

all: $(build_path)/$(EXEC_FILE)

##### OBJECT DEPENDENCIES #####
$(build_path)/main.o: main.cpp run.h
$(build_path)/run.o: run.cu run.h
$(build_path)/shared.o: $(api_path_source)/shared.cu
$(build_path)/go_fish_impl.o: $(api_path_source)/go_fish_impl.cu
##### END OBJECT DEPENDENCIES #####

$(cpp_objects):
   $(CC) $(CFLAGS_CPP) $< -o $@

$(cuda_objects):
   $(NVCC) $(CODE) $(CFLAGS_CUDA) $< -o $@

$(build_path)/gpuCode.o: $(cuda_objects)
   $(NVCC) $(CODE) -dlink $(cuda_objects) -o $@

$(build_path)/$(EXEC_FILE): $(cpp_objects) $(cuda_objects) $(build_path)/gpuCode.o

   $(CC) $(cpp_objects) $(build_path)/gpuCode.o $(cuda_objects) $(LIB_CUDART) -o $@

.PHONY: all