NAMD: Node.C Source File

00001 
00007 /*
00008    Toplevel routines for initializing a Node for a simulation
00009    one Node per Pe (processor element).
00010 */
00011 
00012 #if !defined(WIN32) || defined(__CYGWIN__)
00013 #include <unistd.h>
00014 #endif
00015 #include "InfoStream.h"
00016 #include "Node.decl.h"
00017 #include "Node.h"
00018 #ifdef DPMTA
00019 #include <pvm3.h>
00020 #endif
00021 
00022 #include "ProcessorPrivate.h"
00023 
00024 #define MIN_DEBUG_LEVEL 3
00025 //#define DEBUGM
00026 #include "Debug.h"
00027 
00028 #include <stdio.h>
00029 #include <converse.h>
00030 #include "memusage.h"
00031 #include "IMDOutput.h"
00032 #include "Lattice.h"
00033 #include "ComputeMsmMsa.h"  // needed for MsmMsaData definition
00034 #include "ComputeMsm.h"     // needed for MsmInitMsg definition
00035 #include "main.decl.h"
00036 #include "main.h"
00037 #include "WorkDistrib.h"
00038 #include "PatchMgr.h"
00039 #include "Patch.h"
00040 #include "Compute.h"
00041 #include "ComputeMap.h"
00042 #include "ComputeMgr.h"
00043 #include "Molecule.h"
00044 #include "HomePatchList.h"
00045 #include "AtomMap.h"
00046 #include "Sequencer.h"
00047 #include "Controller.h"
00048 #include "NamdState.h"
00049 #include "Output.h"
00050 #include "ProxyMgr.h"
00051 #include "PatchMap.h"
00052 #include "PatchMap.inl"
00053 #include "Parameters.h"
00054 #include "SimParameters.h"
00055 #include "Communicate.h"
00056 #include "LdbCoordinator.h"
00057 #include "ScriptTcl.h"
00058 #include "ComputeMgr.decl.h"
00059 #include "ComputePmeMgr.decl.h"
00060 // #ifdef NAMD_CUDA
00061 #include "ComputeCUDAMgr.decl.h"
00062 #include "ComputeCUDAMgr.h"
00063 #include "ComputePmeCUDAMgr.decl.h"
00064 #include "ComputePmeCUDAMgr.h"
00065 // #endif
00066 #include "ComputeGridForceMgr.decl.h"
00067 #include "Sync.h"
00068 #include "BackEnd.h"
00069 #include "PDB.h"
00070 #include "packmsg.h"
00071 #include "CollectionMgr.decl.h"
00072 #include "ParallelIOMgr.decl.h"
00073 #include "Vector.h"
00074 // BEGIN LA
00075 #include "Random.h"
00076 // END LA
00077 
00078 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00079 extern "C" void CApplicationInit();
00080 #endif
00081 
00082 #include "DumpBench.h"
00083 
00084 class CheckpointMsg : public CMessage_CheckpointMsg {
00085 public:
00086   int task;
00087   int replica;
00088   Controller::checkpoint checkpoint;
00089   char *key;
00090 };
00091 
00092 extern "C" {
00093   void recvCheckpointCReq_handler(envelope*);
00094   void recvCheckpointCAck_handler(envelope*);
00095 }
00096 
00097 #ifdef CMK_BALANCED_INJECTION_API
00098 #include "ckBIconfig.h"
00099 #endif
00100 
00101 #include "CollectionMgr.h"
00102 #include "CollectionMaster.h"
00103 #include "CollectionMgr.decl.h"
00104 #include "CollectionMaster.decl.h"
00105 
00106 #if USE_HPM
00107 extern "C" void HPM_Init(int);
00108 extern "C" void HPM_Start(char *label, int);
00109 extern "C" void HPM_Stop(char *label, int);
00110 extern "C" void HPM_Print(int, int);
00111 #endif
00112 
00113 #if defined(NAMD_MIC)
00114   extern void mic_dumpHostDeviceComputeMap();
00115   extern void mic_initHostDeviceLDB();
00116 #endif
00117 
00118 #ifdef MEASURE_NAMD_WITH_PAPI
00119 #include "papi.h"
00120 #if CMK_SMP
00121 #include <pthread.h>
00122 #endif
00123 #define NUM_PAPI_EVENTS 6
00124 CkpvDeclare(int *, papiEvents);
00125 
00126 #define MEASURE_PAPI_SPP 1
00127 #define MEASURE_PAPI_CACHE 0
00128 #define MEASURE_PAPI_FLOPS 0
00129 
00130 static void namdInitPapiCounters(){
00131         if(CkMyRank()==0){
00132                 //only initialize per OS process (i.e. a charm node)
00133                 int retval = PAPI_library_init(PAPI_VER_CURRENT);
00134                 if(retval != PAPI_VER_CURRENT) {
00135                         if(CkMyPe()==0){
00136                                 NAMD_die("PAPI library is not compatitible!");
00137                         }
00138                 }
00139         #if CMK_SMP
00140                 //now only consider systems that are compatible with POSIX
00141                 if(PAPI_thread_init(pthread_self)!=PAPI_OK) {
00142                         if(CkMyPe()==0){
00143                                 NAMD_die("Multi-thread mode in PAPI could not be initialized!");
00144                         }
00145                 }
00146         #endif
00147         }
00148         CkpvInitialize(int *, papiEvents);
00149         CkpvAccess(papiEvents) = new int[NUM_PAPI_EVENTS+1];
00150 
00151 #if MEASURE_PAPI_CACHE
00152         if(PAPI_query_event(PAPI_L1_DCM)==PAPI_OK) {
00153                 CkpvAccess(papiEvents)[0] = PAPI_L1_DCM;
00154         }else{
00155                 if(CkMyPe()==0){
00156                         CkPrintf("WARNING: PAPI_L1_DCM doesn't exsit on this platform!\n");                     
00157                 }
00158                 //if not default to PAPI_TOT_INS
00159                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00160         }
00161 
00162         if(PAPI_query_event(PAPI_L2_DCM)==PAPI_OK) {
00163                 CkpvAccess(papiEvents)[1] = PAPI_L2_DCM;
00164         }else{
00165                 //if not default to PAPI_TOT_CYC
00166                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00167         }       
00168 #elif MEASURE_PAPI_FLOPS
00169         if(PAPI_query_event(PAPI_FP_INS)==PAPI_OK) {
00170                 CkpvAccess(papiEvents)[0] = PAPI_FP_INS;
00171         }else{
00172                 if(CkMyPe()==0){
00173                         CkPrintf("WARNING: PAPI_FP_INS doesn't exsit on this platform!\n");
00174                 }
00175                 //if not default to PAPI_TOT_INS
00176                 CkpvAccess(papiEvents)[0] = PAPI_TOT_INS;
00177         }
00178 
00179         if(PAPI_query_event(PAPI_FMA_INS)==PAPI_OK) {
00180                 CkpvAccess(papiEvents)[1] = PAPI_FMA_INS;
00181         }else{
00182                 //if not default to PAPI_TOT_CYC
00183                 CkpvAccess(papiEvents)[1] = PAPI_TOT_CYC;
00184         }
00185 #elif MEASURE_PAPI_SPP
00186 /* for SPP we record these
00187 1) PAPI_FP_OPS
00188 2) PAPI_TOT_INS
00189 3) perf::PERF_COUNT_HW_CACHE_LL:MISS
00190 4) DATA_PREFETCHER:ALL
00191 5) PAPI_L1_DCA
00192 6) INSTRUCTION_FETCH_STALL
00193 7) PAPI_TOT_CYC, and 
00194 8) real (wall) time
00195 */
00196         int papiEventSet = PAPI_NULL; 
00197         if (PAPI_create_eventset(&papiEventSet) != PAPI_OK) {
00198           CmiAbort("PAPI failed to create event set!\n");
00199         }
00200 
00201         if(PAPI_query_event(PAPI_FP_OPS)==PAPI_OK) {
00202                 CkpvAccess(papiEvents)[0] = PAPI_FP_OPS;
00203         }else{
00204                 if(CkMyPe()==0){
00205                         CkAbort("WARNING: PAPI_FP_OPS doesn't exist on this platform!");
00206                 }
00207         }
00208         if(PAPI_query_event(PAPI_TOT_INS)==PAPI_OK) {
00209                 CkpvAccess(papiEvents)[1] = PAPI_TOT_INS;
00210         }else{
00211                 if(CkMyPe()==0){
00212                         CkAbort("WARNING: PAPI_TOT_INS doesn't exist on this platform!");
00213                 }
00214         }
00215         int EventCode;
00216         int ret;
00217         ret=PAPI_event_name_to_code("perf::PERF_COUNT_HW_CACHE_LL:MISS",&EventCode);
00218         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00219           CkpvAccess(papiEvents)[2] = EventCode;
00220         }else{
00221                 if(CkMyPe()==0){
00222                         CkAbort("WARNING: perf::PERF_COUNT_HW_CACHE_LL:MISS doesn't exist on this platform!");
00223                 }
00224         }
00225         ret=PAPI_event_name_to_code("DATA_PREFETCHER:ALL",&EventCode);
00226         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00227           CkpvAccess(papiEvents)[3] = EventCode;
00228         }else{
00229                 if(CkMyPe()==0){
00230                         CkAbort("WARNING: DATA_PREFETCHER:ALL doesn't exist on this platform!");
00231                 }
00232         }
00233         if(PAPI_query_event(PAPI_L1_DCA)==PAPI_OK) {
00234                 CkpvAccess(papiEvents)[4] = PAPI_L1_DCA;
00235         }else{
00236                 if(CkMyPe()==0){
00237                         CkAbort("WARNING: PAPI_L1_DCA doesn't exist on this platform!");
00238                 }
00239         }
00240         /*      ret=PAPI_event_name_to_code("INSTRUCTION_FETCH_STALL",&EventCode);
00241         if(ret==PAPI_OK && PAPI_query_event(EventCode)==PAPI_OK) {
00242           CkpvAccess(papiEvents)[5] = EventCode;
00243         }else{
00244                 if(CkMyPe()==0){
00245                         CkAbort("WARNING: INSTRUCTION_FETCH_STALL doesn't exist on this platform!");
00246                 }
00247         }
00248         */
00249         if(PAPI_query_event(PAPI_TOT_CYC)==PAPI_OK) {
00250                 CkpvAccess(papiEvents)[5] = PAPI_TOT_CYC;
00251         }else{
00252                 if(CkMyPe()==0){
00253                         CkAbort("WARNING: PAPI_TOT_CYC doesn't exist on this platform!");
00254                 }
00255         }
00256         for(int i=0;i<NUM_PAPI_EVENTS;i++)
00257           {
00258             int papiRetValue=PAPI_add_events(papiEventSet, &CkpvAccess(papiEvents)[i],1);
00259             if (papiRetValue != PAPI_OK) {
00260               CkPrintf("failure for event %d\n",i);
00261               if (papiRetValue == PAPI_ECNFLCT) {
00262                 CmiAbort("PAPI events conflict! Please re-assign event types!\n");
00263               } else {
00264                 CmiAbort("PAPI failed to add designated events!\n");
00265               }
00266             }
00267             
00268           }
00269 #endif
00270 }
00271 #endif
00272 
00273 #ifdef OPENATOM_VERSION
00274 static void startOA(){(char inDriverFile[1024], char inPhysicsFile[1024], CkCallback doneCB)
00275 {
00276   CProxy_oaSetup moaInstance = CProxy_oaSetup::ckNew(inDriverFile, inPhysicsFile, doneCB);
00277 }
00278 #endif //OPENATOM_VERSION
00279 
00280 //======================================================================
00281 // Public Functions
00282 
00283 //----------------------------------------------------------------------
00284 
00285 int eventEndOfTimeStep;
00286 double startupTime;
00287 
00288 //----------------------------------------------------------------------
00289 // BOC constructor
00290 Node::Node(GroupInitMsg *msg)
00291 {    
00292   DebugM(4,"Creating Node\n");
00293 #if(CMK_CCS_AVAILABLE && CMK_WEB_MODE)
00294   CApplicationInit();
00295 #endif
00296   if (CkpvAccess(Node_instance) == 0) {
00297     CkpvAccess(Node_instance) = this;
00298     eventEndOfTimeStep = traceRegisterUserEvent("EndOfTimeStep", 135);
00299   } else {
00300     NAMD_bug("Node::Node() - another instance of Node exists!");
00301   }
00302 
00303   CkpvAccess(BOCclass_group) = msg->group;
00304   delete msg;
00305 
00306   CkpvAccess(BOCclass_group).node = thisgroup;
00307 
00308   recvCheckpointCReq_index = CmiRegisterHandler((CmiHandler)recvCheckpointCReq_handler);
00309   recvCheckpointCAck_index = CmiRegisterHandler((CmiHandler)recvCheckpointCAck_handler);
00310 
00311   startupPhase = 0;
00312 
00313   molecule = NULL;
00314   parameters = NULL;
00315   simParameters = NULL;
00316   configList = NULL;
00317   pdb = NULL;
00318   state = NULL;
00319   output = NULL;
00320   imd = new IMDOutput;
00321   colvars = 0;
00322 
00323 #if USE_HPM
00324   // assumes that this will be done only on BG/P
00325   TopoManager *tmgr = new TopoManager();
00326   int x, y, z;
00327   tmgr->rankToCoordinates(CkMyPe(), x, y, z, localRankOnNode);
00328   delete tmgr;
00329 #endif
00330 
00331   specialTracing = traceAvailable() && (traceIsOn()==0);
00332 
00333   DebugM(4,"Creating PatchMap, AtomMap, ComputeMap\n");
00334   patchMap = PatchMap::Instance();
00335   atomMap = AtomMap::Instance();
00336   if ( CkMyRank() == 0 ) ComputeMap::Instance();
00337 
00338   //Note: Binding BOC vars such as workDistrib has been moved
00339   //to the 1st phase of startup because the in-order message delivery
00340   //is not always guaranteed --Chao Mei
00341 #ifdef CMK_BALANCED_INJECTION_API
00342   if(CkMyRank() == 0){
00343     balancedInjectionLevel=ck_get_GNI_BIConfig();
00344     // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00345     ck_set_GNI_BIConfig(20);
00346     // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00347   }
00348 #endif
00349 
00350 }
00351 
00352 //----------------------------------------------------------------------
00353 // ~Node(void) needs to clean up everything.
00354 
00355 Node::~Node(void)
00356 {
00357   delete output;
00358   delete computeMap;
00359   delete atomMap;
00360   delete patchMap;
00361   delete CkpvAccess(comm);
00362   // BEGIN LA
00363   delete rand;
00364   // END LA
00365 #ifdef MEASURE_NAMD_WITH_PAPI
00366   delete CkpvAccess(papiEvents);
00367 #endif
00368 }
00369 
00370 void Node::bindBocVars(){
00371     DebugM(4,"Binding to BOC's\n");
00372     CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00373     patchMgr = pm.ckLocalBranch();
00374     CProxy_ProxyMgr prm(CkpvAccess(BOCclass_group).proxyMgr);
00375     proxyMgr = prm.ckLocalBranch();
00376     CProxy_WorkDistrib wd(CkpvAccess(BOCclass_group).workDistrib);
00377     workDistrib = wd.ckLocalBranch();
00378     CProxy_ComputeMgr cm(CkpvAccess(BOCclass_group).computeMgr);
00379     computeMgr = cm.ckLocalBranch();
00380     CProxy_LdbCoordinator lc(CkpvAccess(BOCclass_group).ldbCoordinator);
00381     ldbCoordinator = lc.ckLocalBranch();
00382   #ifdef MEM_OPT_VERSION      
00383     CProxy_ParallelIOMgr io(CkpvAccess(BOCclass_group).ioMgr);
00384     ioMgr = io.ckLocalBranch();
00385   #endif
00386 
00387 }
00388 
00389 //----------------------------------------------------------------------
00390 // Malloc Test Sequence
00391 void Node::mallocTest(int step) {
00392   int MB = 1024*1024;
00393   int size = 100;
00394   char* foo = (char*) malloc(size*MB);
00395   if ( ! foo ) {
00396     char buf[256];
00397     sprintf(buf,"Malloc fails on Pe %d at %d MB.\n",CkMyPe(),step*size);
00398     NAMD_die(buf);
00399   }
00400   memset(foo,0,size*MB*sizeof(char));
00401 }
00402 
00403 void Node::mallocTestQd() {
00404   if ( mallocTest_size ) {
00405     CkPrintf("All PEs successfully allocated %d MB.\n", 100*mallocTest_size);
00406   } else {
00407     CkPrintf("Starting malloc test on all PEs.\n");
00408   }
00409   fflush(stdout);
00410   ++mallocTest_size;
00411   CkStartQD(CkIndex_Node::mallocTestQd(), &thishandle);
00412   (CProxy_Node(CkpvAccess(BOCclass_group).node)).mallocTest(mallocTest_size);
00413 }
00414 
00415 //----------------------------------------------------------------------
00416 // Startup Sequence
00417 
00418 void Node::messageStartUp() {
00419   (CProxy_Node(CkpvAccess(BOCclass_group).node)).startup();
00420 }
00421 
00422 SimParameters *node_simParameters;
00423 Parameters *node_parameters;
00424 Molecule *node_molecule;
00425 
00426 extern void registerUserEventsForAllComputeObjs(void);
00427 
00428 void Node::startup() {
00429   int gotoRun = false;
00430   double newTime;
00431 
00432   if (!CkMyPe()) {
00433     if (!startupPhase) {
00434       iout << iINFO << "\n";
00435       startupTime = CmiWallTimer();
00436       iout << iINFO << "Entering startup at " << startupTime << " s, ";
00437     } else {
00438       newTime = CmiWallTimer();
00439       iout << iINFO << "Startup phase " << startupPhase-1 << " took "
00440            << newTime - startupTime << " s, ";
00441       startupTime = newTime;
00442     }
00443     iout << memusage_MB() << " MB of memory in use\n" << endi;
00444     fflush(stdout);
00445   }
00446   switch (startupPhase) {
00447 
00448   case 0:
00449     computeMap = ComputeMap::Object();
00450     namdOneCommInit(); // Namd1.X style
00451   break;
00452 
00453   case 1:
00454       bindBocVars();
00455 
00456     // send & receive molecule, simparameters... (Namd1.X style)
00457     if (CkMyPe()) {
00458       namdOneRecv();
00459     } else {
00460       namdOneSend();
00461     }
00462   break;
00463 
00464   case 2:
00465     // fix up one-per-node objects (for SMP version)
00466     simParameters = node_simParameters;
00467     parameters = node_parameters;
00468     molecule = node_molecule;
00469 
00470     SimParameters::nonbonded_select();
00471     if ( simParameters->PMEOn ) SimParameters::pme_select();   
00472  
00473     #if !CMK_SMP || ! USE_CKLOOP
00474     //the CkLoop library should be only used in SMP mode
00475     simParameters->useCkLoop = 0;
00476     #else
00477     if ( CkNumPes() < 2 * CkNumNodes() ) simParameters->useCkLoop = 0;
00478     #endif
00479 
00480 
00481     if ( simParameters->mallocTest ) {
00482       if (!CkMyPe()) {
00483         mallocTest_size = 0;
00484         CkStartQD(CkIndex_Node::mallocTestQd(), &thishandle);
00485       }
00486       return;
00487     }
00488 
00489       
00490         #ifdef MEASURE_NAMD_WITH_PAPI
00491         if(simParameters->papiMeasure) namdInitPapiCounters();  
00492         #endif
00493     
00494     #ifdef MEM_OPT_VERSION
00495     //At this point, each Node object has received the simParameters,
00496     //parameters and the atom signatures info from the master Node
00497     //(proc 0). It's time to initialize the parallel IO manager and
00498     //read the binary per-atom file --Chao Mei
00499 
00500     //Step 1: initialize the parallel IO manager per Node
00501     ioMgr->initialize(this);
00502     #endif
00503 
00504   break;
00505 
00506   case 3:
00507 
00508     #ifdef MEM_OPT_VERSION
00509     //Step 2: read the binary per-atom files (signater index, coordinates etc.)
00510     ioMgr->readPerAtomInfo();
00511     #endif
00512 
00513   break;
00514 
00515   case 4:
00516 
00517     #ifdef MEM_OPT_VERSION
00518     //Step 3: update counters of tuples and exclusions inside Molecule object
00519     ioMgr->updateMolInfo();
00520 
00521     //Step 4: prepare distributing the atoms to neighboring procs if necessary
00522     ioMgr->migrateAtomsMGrp();
00523 
00524     //step 5: initialize patchMap and send it to every other processors
00525     //to decide atoms to patch distribution on every input processor
00526     if(!CkMyPe()) {
00527         workDistrib->patchMapInit(); // create space division
00528         workDistrib->sendPatchMap();
00529     }
00530     #endif
00531 
00532     #if USE_HPM
00533     HPM_Init(localRankOnNode);
00534     #endif    
00535 
00536     // take care of inital thread setting
00537     threadInit();
00538 
00539     // create blank AtomMap
00540     AtomMap::Object()->allocateMap(molecule->numAtoms);
00541 
00542     if (!CkMyPe()) {
00543 #ifdef NAMD_CUDA
00544       if (simParameters->usePMECUDA) {
00545         // computePmeCUDAMgr was created in BackEnd.C
00546         // This empty branch is to avoid initializing ComputePmeMgr
00547       } else
00548 #endif
00549       if (simParameters->PMEOn) {
00550         CkpvAccess(BOCclass_group).computePmeMgr = CProxy_ComputePmeMgr::ckNew();
00551       }
00552         #ifdef OPENATOM_VERSION
00553         if ( simParameters->openatomOn ) { 
00554           CkpvAccess(BOCclass_group).computeMoaMgr = CProxy_ComputeMoaMgr::ckNew();
00555         }
00556         #endif // OPENATOM_VERSION
00557 
00558     }
00559     
00560     #ifdef OPENATOM_VERSION
00561     if ( simParameters->openatomOn ) {
00562       // if ( ! CkMyPe() ) { 
00563         CkCallback doneMoaStart(CkIndexmain::doneMoaSetup(), thishandle); 
00564         startOA(simParameters->moaDriverFile, simParameters->moaPhysicsFile, doneMoaStart);
00565       // }
00566     }
00567     #endif // OPENATOM_VERSION
00568   
00569     // BEGIN LA
00570     rand = new Random(simParameters->randomSeed);
00571     rand->split(CkMyPe(), CkNumPes());
00572     // END LA
00573 
00574   break;
00575 
00576   case 5:
00577     #ifdef MEM_OPT_VERSION
00578     //Now, every input proc has received all the atoms necessary
00579     //to decide the patches those atoms belong to
00580     
00581     //step 1: integrate the migrated atoms into the atom list that
00582     //contains the initally distributed atoms, and sort the atoms
00583     //based on hydrogenList value
00584     ioMgr->integrateMigratedAtoms();
00585 
00586     //step 2: integrate the cluster size of each atom on each output proc
00587     ioMgr->integrateClusterSize();
00588 
00589     //step 3: calculate the number of atoms in each patch on every
00590     //input procs (atoms belonging to a patch may lie on different
00591     //procs), and reduce such info on proc 0. Such info is required
00592     //for determing which node a particular patch is assigned to.
00593     ioMgr->calcAtomsInEachPatch();
00594 
00595     //set to false to re-send PatchMap later
00596     workDistrib->setPatchMapArrived(false);
00597     #endif
00598     break;
00599   case 6:     
00600     if(simParameters->isSendSpanningTreeOn()) {                         
00601                         ProxyMgr::Object()->setSendSpanning();
00602     }
00603     if(simParameters->isRecvSpanningTreeOn()) {                         
00604                         ProxyMgr::Object()->setRecvSpanning();
00605     }
00606     if(simParameters->proxyTreeBranchFactor) {
00607                         ProxyMgr::Object()->setProxyTreeBranchFactor(simParameters->proxyTreeBranchFactor);
00608     }
00609     #ifdef PROCTRACE_DEBUG
00610     DebugFileTrace::Instance("procTrace");
00611     #endif
00612 
00613     if (!CkMyPe()) {
00614       output = new Output; // create output object just on PE(0)
00615 
00616       #ifndef MEM_OPT_VERSION
00617       workDistrib->patchMapInit(); // create space division
00618       workDistrib->createHomePatches(); // load atoms into HomePatch(es)
00619       #endif
00620       
00621       workDistrib->assignNodeToPatch();
00622       workDistrib->mapComputes();         
00623       //ComputeMap::Object()->printComputeMap();
00624 
00625       // For MIC runs, take the additional step after the compute map has been created to
00626       //   assign the various computes to either the host or the device.  This info will
00627       //   be distributed across the PEs.
00628       #if defined(NAMD_MIC)
00629         mic_initHostDeviceLDB();
00630       #endif
00631           
00632           if(simParameters->simulateInitialMapping) {
00633           iout << iINFO << "Simulating initial mapping with " << simParameters->simulatedPEs
00634                           << " PEs with " << simParameters->simulatedNodeSize << " PEs per node\n" << endi;
00635                   outputPatchComputeMaps("init_mapping", 0);
00636                   iout << iINFO << "Simulating initial mapping is done, now NAMD exits\n" << endi;
00637                   BackEnd::exit();
00638           }
00639 
00640       registerUserEventsForAllComputeObjs();
00641 
00642       //in MEM_OPT_VERSION, patchMap is resent
00643       //because they have been updated since creation including
00644       //#atoms per patch, the proc a patch should stay etc. --Chao Mei
00645       workDistrib->sendPatchMap();
00646       #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00647       CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00648       //a node broadcast
00649       npm.createProxyInfo(PatchMap::Object()->numPatches());
00650       #endif
00651     }
00652     {
00653         #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
00654         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00655         if(CkMyRank()==0) {
00656             //just need to register once
00657             npm[CkMyNode()].ckLocalBranch()->registerLocalProxyMgr(CkpvAccess(BOCclass_group).proxyMgr);
00658         }
00659         npm[CkMyNode()].ckLocalBranch()->registerLocalPatchMap(CkMyRank(), PatchMap::Object());
00660         #endif
00661     }
00662   break;
00663 
00664   case 7:
00665 #ifdef CHARM_HAS_MSA
00666     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00667       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00668       msm[CkMyPe()].initialize(new CkQdMsg);
00669     }
00670 #else
00671     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00672       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00673       MsmInitMsg *msg = new MsmInitMsg;
00674       Lattice lattice = simParameters->lattice;  // system lattice vectors
00675       ScaledPosition smin=0, smax=0;
00676       if (lattice.a_p() && lattice.b_p() && lattice.c_p()) {
00677         msg->smin = smin;
00678         msg->smax = smax;
00679         msm[CkMyPe()].initialize(msg);  // call from my own PE
00680       }
00681       else if ( ! CkMyPe() ) {
00682         pdb->get_extremes(smin, smax);  // only available on PE 0
00683         msg->smin = smin;
00684         msg->smax = smax;
00685         msm.initialize(msg);  // broadcast to chare group
00686       }
00687 
00688       /*
00689       CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00690       Node *node = nd.ckLocalBranch();
00691       ScaledPosition smin, smax;
00692       node->pdb->get_extremes(smin, smax);
00693       msg->smin = smin;                       // extreme positions in system
00694       msg->smax = smax;
00695       msm[CkMyPe()].initialize(msg);
00696       */
00697     }
00698 #endif
00699 
00700     if ( simParameters->PMEOn ) {
00701       #ifdef OPENATOM_VERSION
00702       if ( simParameters->openatomOn ) { 
00703         CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00704         moa[CkMyPe()].initialize(new CkQdMsg);
00705       }
00706       #endif // OPENATOM_VERSION
00707 #ifdef NAMD_CUDA
00708       if ( simParameters->usePMECUDA ) {
00709         if(CkMyRank()==0) {
00710           CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00711           pme.ckLocalBranch()->initialize(new CkQdMsg);  // must run on pe 0 to call ckNew
00712         }
00713       } else 
00714 #endif
00715       {
00716         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00717         pme[CkMyPe()].initialize(new CkQdMsg);          
00718       }
00719     }
00720     break;
00721 
00722   case 8:
00723 #ifdef NAMD_CUDA
00724     if ( (simParameters->useCUDA2 || simParameters->bondedCUDA) && CkMyRank()==0 ) {
00725       CProxy_ComputeCUDAMgr nb(CkpvAccess(BOCclass_group).computeCUDAMgr);
00726       nb.ckLocalBranch()->initialize(new CkQdMsg);
00727     }
00728 #endif
00729     break;
00730 
00731   case 9:
00732     workDistrib->sendComputeMap();
00733     break;
00734 
00735   case 10:
00736     #ifdef MEM_OPT_VERSION
00737     //migrate atoms to HomePatch processors
00738     ioMgr->sendAtomsToHomePatchProcs();
00739     #endif
00740     break;
00741     
00742   case 11:
00743     // part 2 of MSM init
00744     if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00745       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00746       msm[CkMyPe()].initialize_create();  // call from my own PE
00747     }
00748 
00749     if ( simParameters->PMEOn ) {
00750       #ifdef OPENATOM_VERSION
00751       if ( simParameters->openatomOn ) { 
00752         CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00753         moa[CkMyPe()].initWorkers(new CkQdMsg);
00754       }
00755       #endif // OPENATOM_VERSION
00756 #ifdef NAMD_CUDA
00757       if ( simParameters->usePMECUDA ) {
00758         if(CkMyRank()==0) {
00759           CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00760           pme[CkMyNode()].initialize_pencils(new CkQdMsg);
00761         }
00762       } else
00763 #endif
00764       {
00765         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00766         pme[CkMyPe()].initialize_pencils(new CkQdMsg);          
00767       }
00768     }
00769 #ifdef CHARM_HAS_MSA
00770     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00771       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00772       msm[CkMyPe()].initWorkers(new CkQdMsg);
00773     }
00774 #else
00775     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00776       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00777       msm[CkMyPe()].update(new CkQdMsg);
00778     }
00779 #endif
00780 
00781     #ifdef MEM_OPT_VERSION
00782     //Now every processor has all the atoms it needs to create the HomePatches.
00783     //The HomePatches are created in parallel on every home patch procs.
00784     ioMgr->createHomePatches();
00785     #else
00786     if (!CkMyPe()) {
00787       workDistrib->distributeHomePatches();          
00788     }
00789     #endif
00790   break;
00791 
00792   case 12:
00793     if ( simParameters->PMEOn ) {
00794       #ifdef OPENATOM_VERSION
00795       if ( simParameters->openatomOn ) { 
00796         CProxy_ComputeMoaMgr moa(CkpvAccess(BOCclass_group).computeMoaMgr); 
00797         moa[CkMyPe()].startWorkers(new CkQdMsg);
00798       }
00799       #endif // OPENATOM_VERSION
00800 #ifdef NAMD_CUDA
00801       if ( simParameters->usePMECUDA ) {
00802         if(CkMyRank()==0) {
00803           CProxy_ComputePmeCUDAMgr pme(CkpvAccess(BOCclass_group).computePmeCUDAMgr);
00804           pme[CkMyNode()].activate_pencils(new CkQdMsg);
00805         }
00806       } else
00807 #endif
00808       {
00809         CProxy_ComputePmeMgr pme(CkpvAccess(BOCclass_group).computePmeMgr);
00810         pme[CkMyPe()].activate_pencils(new CkQdMsg);          
00811       }
00812     }
00813 #ifdef CHARM_HAS_MSA
00814     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00815       CProxy_ComputeMsmMsaMgr msm(CkpvAccess(BOCclass_group).computeMsmMsaMgr);
00816       msm[CkMyPe()].startWorkers(new CkQdMsg);
00817     }
00818 #else
00819     /*
00820     else if ( simParameters->MSMOn && ! simParameters->MsmSerialOn ) {
00821       CProxy_ComputeMsmMgr msm(CkpvAccess(BOCclass_group).computeMsmMgr);
00822       //msm[CkMyPe()].startWorkers(new CkQdMsg);
00823     }
00824     */
00825 #endif
00826 
00827     proxyMgr->createProxies();  // need Home patches before this
00828     if (!CkMyPe()) LdbCoordinator::Object()->createLoadBalancer();
00829 
00830 #ifdef NAMD_TCL
00831     // TclInitSubsystems() has a race condition so we create one interp per node here
00832     if (CkMyPe() && CkMyNodeSize() > 1 && ! CkMyRank()) Tcl_DeleteInterp(Tcl_CreateInterp());
00833 #endif
00834 
00835 #ifdef USE_NODEPATCHMGR
00836         //at this point, PatchMap info has been recved on PEs. It is time to create
00837         //the home patch spanning tree for receiving proxy list info
00838         if(proxyMgr->getSendSpanning() || proxyMgr->getRecvSpanning()) {
00839                 if(CkMyRank()==0) {
00840                         CProxy_NodeProxyMgr npm(CkpvAccess(BOCclass_group).nodeProxyMgr);
00841                         npm[CkMyNode()].ckLocalBranch()->createSTForHomePatches(PatchMap::Object());
00842                 }
00843         }
00844 #endif
00845 
00846   break;
00847 
00848   case 13:
00849 
00850     // DMK - DEBUG - If, in MIC runs, the debug option to dump all the compute maps to files
00851     //   for debugging/verification purposes has been enabled, have each PE do so now.
00852     #if defined(NAMD_MIC)
00853       mic_dumpHostDeviceComputeMap();
00854     #endif
00855 
00856     if (!CkMyPe()) {
00857       iout << iINFO << "CREATING " << ComputeMap::Object()->numComputes()
00858            << " COMPUTE OBJECTS\n" << endi;
00859     }
00860     DebugM(4,"Creating Computes\n");
00861     computeMgr->createComputes(ComputeMap::Object());
00862     DebugM(4,"Building Sequencers\n");
00863     buildSequencers();
00864     DebugM(4,"Initializing LDB\n");
00865     LdbCoordinator::Object()->initialize(PatchMap::Object(),ComputeMap::Object());
00866   break;
00867 
00868   case 14:
00869     // computes may create proxies on the fly so put these in separate phase
00870     Sync::Object()->openSync();  // decide if to open local Sync 
00871     if (proxySendSpanning || proxyRecvSpanning ) proxyMgr->buildProxySpanningTree();
00872 #ifdef CMK_BALANCED_INJECTION_API
00873     if(CkMyRank() == 0){
00874       // CkPrintf("[%d] get retrieved BI=%d\n",CkMyPe(),balancedInjectionLevel);
00875       ck_set_GNI_BIConfig(balancedInjectionLevel);
00876       // CkPrintf("[%d] set retrieved BI=%d\n",CkMyPe(),ck_get_GNI_BIConfig());
00877     }
00878 #endif
00879 
00880   break;
00881 
00882   case 15:
00883     {
00884         //For debugging
00885         /*if(!CkMyPe()){
00886         FILE *dumpFile = fopen("/tmp/NAMD_Bench.dump", "w");
00887         dumpbench(dumpFile);
00888         NAMD_die("Normal execution\n");
00889         }*/
00890     }
00891     #ifdef MEM_OPT_VERSION
00892     //free space in the Molecule object that are not used anymore
00893     ioMgr->freeMolSpace();
00894     #endif
00895     gotoRun = true;
00896   break;
00897 
00898   default:
00899     NAMD_bug("Startup Phase has a bug - check case statement");
00900   break;
00901 
00902   }
00903 
00904   startupPhase++;
00905   if (!CkMyPe()) {
00906     if (!gotoRun) {
00907       CkStartQD(CkCallback(CkIndex_Node::startup(), thisgroup));
00908     } else {
00909       Node::messageRun();
00910     }
00911   }
00912 }
00913 
00914 #ifdef OPENATOM_VERSION
00915 void Node::doneMoaStart()
00916 {
00917 #ifdef OPENATOM_VERSION_DEBUG
00918   CkPrintf("doneMoaStart executed on processor %d.\n", CkMyPe() );
00919 #endif //OPENATOM_VERSION_DEBUG
00920 }
00921 #endif //OPENATOM_VERSION
00922 
00923 void Node::namdOneCommInit()
00924 {
00925   if (CkpvAccess(comm) == NULL) {
00926     CkpvAccess(comm) = new Communicate();
00927 #ifdef DPMTA
00928     pvmc_init();
00929 #endif
00930   }
00931 }
00932 
00933 // Namd 1.X style Send/Recv of simulation information
00934 
00935 void Node::namdOneRecv() {
00936   if ( CmiMyRank() ) return;
00937 
00938   MIStream *conv_msg;
00939 
00940   // Receive molecule and simulation parameter information
00941   simParameters = node_simParameters = new SimParameters;
00942   //****** BEGIN CHARMM/XPLOR type changes
00943   parameters = node_parameters = new Parameters();
00944   //****** END CHARMM/XPLOR type changes
00945   molecule = node_molecule = new Molecule(simParameters,parameters);
00946 
00947   DebugM(4, "Getting SimParameters\n");
00948   conv_msg = CkpvAccess(comm)->newInputStream(0, SIMPARAMSTAG);
00949   simParameters->receive_SimParameters(conv_msg);
00950 
00951   DebugM(4, "Getting Parameters\n");
00952   conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
00953   parameters->receive_Parameters(conv_msg);
00954 
00955   DebugM(4, "Getting Molecule\n");
00956   conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00957   // Modified by JLai -- 10.21.11
00958   molecule->receive_Molecule(conv_msg);
00959   if(simParameters->goForcesOn) {
00960     iout << iINFO << "Compute Nodes receiving GoMolecule Information" << "\n" << endi;
00961     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
00962     molecule->receive_GoMolecule(conv_msg);
00963   } 
00964   // End of modification
00965   DebugM(4, "Done Receiving\n");
00966 }
00967 
00968 void Node::namdOneSend() {
00969   node_simParameters = simParameters;
00970   node_parameters = parameters;
00971   node_molecule = molecule;
00972 
00973   MOStream *conv_msg;
00974   // I'm Pe(0) so I send what I know
00975   DebugM(4, "Sending SimParameters\n");  
00976   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, SIMPARAMSTAG, BUFSIZE);
00977   simParameters->send_SimParameters(conv_msg);
00978 
00979   DebugM(4, "Sending Parameters\n");
00980   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, BUFSIZE);
00981   parameters->send_Parameters(conv_msg);
00982 
00983   DebugM(4, "Sending Molecule\n");
00984   int bufSize = BUFSIZE;
00985   if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
00986   conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
00987   // Modified by JLai -- 10.21.11
00988   molecule->send_Molecule(conv_msg);
00989   
00990   if(simParameters->goForcesOn) {
00991     iout << iINFO <<  "Master Node sending GoMolecule Information" << "\n" << endi;
00992     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
00993     molecule->send_GoMolecule(conv_msg);
00994   } // End of modification
00995 }
00996 
00997 
00998 void Node::reloadStructure(const char *fname, const char *pdbname) {
00999   delete molecule;
01000   molecule = state->molecule = 0;
01001   delete pdb;
01002   pdb = state->pdb = 0;
01003   state->loadStructure(fname,pdbname,1);
01004   this->molecule = state->molecule;
01005   this->pdb = state->pdb;
01006   CProxy_Node nodeProxy(thisgroup);
01007   nodeProxy.resendMolecule();
01008 }
01009 
01010 
01011 void Node::resendMolecule() {
01012   if ( CmiMyRank() ) {
01013     return;
01014   }
01015   if ( CmiMyPe() == 0 ) {
01016     int bufSize = BUFSIZE;
01017     MOStream *conv_msg;
01018     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, STATICPARAMSTAG, bufSize);
01019     parameters->send_Parameters(conv_msg);
01020     if(molecule->numAtoms>=1000000) bufSize = 16*BUFSIZE;
01021     conv_msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, MOLECULETAG, bufSize);
01022     molecule->send_Molecule(conv_msg);
01023   } else {
01024     MIStream *conv_msg;
01025     delete parameters;
01026     parameters = new Parameters;
01027     conv_msg = CkpvAccess(comm)->newInputStream(0, STATICPARAMSTAG);
01028     parameters->receive_Parameters(conv_msg);
01029     delete molecule;
01030     molecule = new Molecule(simParameters,parameters);
01031     conv_msg = CkpvAccess(comm)->newInputStream(0, MOLECULETAG);
01032     molecule->receive_Molecule(conv_msg);
01033   }
01034   node_parameters = parameters;
01035   node_molecule = molecule;
01036   SimParameters::nonbonded_select();
01037   if ( simParameters->PMEOn ) SimParameters::pme_select();
01038   computeMgr->sendBuildCudaExclusions();
01039   CProxy_Node nodeProxy(thisgroup);
01040   for ( int i=0; i<CmiMyNodeSize(); ++i ) {
01041     nodeProxy[CmiMyPe()+i].resendMolecule2();
01042   }
01043 }
01044 
01045 void Node::resendMolecule2() {
01046   parameters = node_parameters;
01047   molecule = node_molecule;
01048   AtomMap::Object()->allocateMap(molecule->numAtoms);
01049 }
01050 
01051 
01052 // Initial thread setup
01053 
01054 void Node::threadInit() {
01055   // Thread initialization
01056   if (CthImplemented()) {
01057     CthSetStrategyDefault(CthSelf());
01058   } else {
01059     NAMD_bug("Node::startup() Oh no, tiny elvis, threads not implemented");
01060   }
01061 }
01062 
01063 //
01064 void Node::buildSequencers() {
01065   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01066   ResizeArrayIter<HomePatchElem> ai(*hpl);
01067 
01068   // Controller object is only on Pe(0)
01069   if ( ! CkMyPe() ) {
01070     Controller *controller = new Controller(state);
01071     state->useController(controller);
01072   }
01073 
01074   // Assign Sequencer to all HomePatch(es)
01075   for (ai=ai.begin(); ai != ai.end(); ai++) {
01076     HomePatch *patch = (*ai).patch;
01077     Sequencer *sequencer = new Sequencer(patch);
01078     patch->useSequencer(sequencer);
01079   }
01080 }
01081 
01082 
01083 
01084 //-----------------------------------------------------------------------
01085 // Node run() - broadcast to all nodes
01086 //-----------------------------------------------------------------------
01087 void Node::messageRun() {
01088   (CProxy_Node(CkpvAccess(BOCclass_group).node)).run();
01089 }
01090 
01091 
01092 //-----------------------------------------------------------------------
01093 // run(void) runs the specified simulation for the specified number of
01094 // steps, overriding the contents of the configuration file
01095 //-----------------------------------------------------------------------
01096 void Node::run()
01097 {
01098   // Start Controller (aka scalar Sequencer) on Pe(0)
01099 //  printf("\n\n I am in Node.C in run method about to call  state->runController\n\n");
01100   if ( ! CkMyPe() ) {
01101     state->runController();
01102   }
01103 
01104   DebugM(4, "Starting Sequencers\n");
01105   // Run Sequencer on each HomePatch - i.e. start simulation
01106   HomePatchList *hpl = PatchMap::Object()->homePatchList();
01107   ResizeArrayIter<HomePatchElem> ai(*hpl);
01108   for (ai=ai.begin(); ai != ai.end(); ai++) {
01109     HomePatch *patch = (*ai).patch;
01110 //CkPrintf("Proc#%d in Node calling Sequencer ",CkMyPe());
01111     patch->runSequencer();
01112   }
01113 
01114   if (!CkMyPe()) {
01115     double newTime = CmiWallTimer();
01116     iout << iINFO << "Startup phase " << startupPhase-1 << " took "
01117          << newTime - startupTime << " s, "
01118          << memusage_MB() << " MB of memory in use\n";
01119     iout << iINFO << "Finished startup at " << newTime << " s, "
01120          << memusage_MB() << " MB of memory in use\n\n" << endi;
01121     fflush(stdout);
01122   }
01123   
01124 }
01125 
01126 
01127 //-----------------------------------------------------------------------
01128 // Node scriptBarrier() - twiddle parameters with simulation halted
01129 //-----------------------------------------------------------------------
01130 
01131 void Node::enableScriptBarrier() {
01132   CkStartQD(CkIndex_Node::scriptBarrier(), &thishandle);
01133 }
01134 
01135 void Node::scriptBarrier() {
01136   //script->awaken();
01137 }
01138 
01139 void Node::scriptParam(ScriptParamMsg *msg) {
01140   simParameters->scriptSet(msg->param,msg->value);
01141   delete msg;
01142 }
01143 
01144 void Node::reloadCharges(const char *filename) {
01145   FILE *file = fopen(filename,"r");
01146   if ( ! file ) NAMD_die("node::reloadCharges():Error opening charge file.");
01147 
01148   int n = molecule->numAtoms;
01149   float *charge = new float[n];
01150 
01151   for ( int i = 0; i < n; ++i ) {
01152     if ( ! fscanf(file,"%f",&charge[i]) )
01153       NAMD_die("Node::reloadCharges():Not enough numbers in charge file.");
01154   }
01155 
01156   fclose(file);
01157   CProxy_Node(thisgroup).reloadCharges(charge,n);
01158   delete [] charge;
01159 }
01160 
01161 void Node::reloadCharges(float charge[], int n) {
01162   molecule->reloadCharges(charge,n);
01163 }
01164 
01165 
01166 // BEGIN gf
01167 void Node::reloadGridforceGrid(const char * key) {
01168     DebugM(4, "reloadGridforceGrid(const char*) called on node " << CkMyPe() << "\n" << endi);
01169     
01170     int gridnum;
01171     MGridforceParams *mgridParams;
01172     if (key == NULL) {
01173         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01174         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01175     } else {
01176         gridnum = simParameters->mgridforcelist.index_for_key(key);
01177         mgridParams = simParameters->mgridforcelist.find_key(key);
01178     }
01179     
01180     if (gridnum < 0 || mgridParams == NULL) {
01181         NAMD_die("Node::reloadGridforceGrid(const char*):Could not find grid.");
01182     }
01183     
01184     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01185     if (grid == NULL) {
01186         NAMD_bug("Node::reloadGridforceGrid(const char*):grid not found");
01187     }
01188     grid->reinitialize(simParameters, mgridParams);
01189     
01190     CProxy_Node(thisgroup).reloadGridforceGrid(gridnum);
01191     
01192     DebugM(4, "reloadGridforceGrid(const char*) finished\n" << endi);
01193 }
01194 
01195 void Node::updateGridScale(const char* key, Vector scale) {
01196     DebugM(4, "updateGridScale(char*, Vector) called on node " << CkMyPe() << "\n" << endi);
01197     
01198     int gridnum;
01199     MGridforceParams* mgridParams;
01200     if (key == NULL) {
01201         gridnum = simParameters->mgridforcelist.index_for_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01202         mgridParams = simParameters->mgridforcelist.find_key(MGRIDFORCEPARAMS_DEFAULTKEY);
01203     } else {
01204         gridnum = simParameters->mgridforcelist.index_for_key(key);
01205         mgridParams = simParameters->mgridforcelist.find_key(key);
01206     }
01207 
01208     if (gridnum < 0 || mgridParams == NULL) {
01209         NAMD_die("Node::updateGridScale(char*, Vector): Could not find grid.");
01210     }
01211     
01212     GridforceGrid* grid = molecule->get_gridfrc_grid(gridnum);
01213     if (grid == NULL) {
01214         NAMD_bug("Node::updateGridScale(char*, Vector): grid not found");
01215     }
01216     CProxy_Node(thisgroup).updateGridScale(gridnum, scale.x, scale.y, scale.z);
01217     
01218     DebugM(4, "updateGridScale(char*, Vector) finished\n" << endi);
01219 }
01220 void Node::updateGridScale(int gridnum, float sx, float sy, float sz) {
01221     if (CmiMyRank()) return;
01222     DebugM(4, "updateGridScale(char*, int, float, float, float) called on node " << CkMyPe() << "\n" << endi);
01223        
01224     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01225     if (grid == NULL) {
01226         NAMD_bug("Node::updateGridScale(char*, int, float, float, float):grid not found");
01227     }
01228     
01229     Vector scale(sx,sy,sz);
01230     simParameters->mgridforcelist.at_index(gridnum)->gridforceScale = scale;
01231     grid->set_scale( scale );
01232 
01233     DebugM(4, "updateGridScale(char*, int, float, float, float) finished\n" << endi);
01234 }
01235 
01236 void Node::reloadGridforceGrid(int gridnum) {
01237     if (CmiMyRank()) return;
01238     DebugM(4, "reloadGridforceGrid(int) called on node " << CkMyPe() << "\n" << endi);
01239     
01240     GridforceGrid *grid = molecule->get_gridfrc_grid(gridnum);
01241     if (grid == NULL) {
01242         NAMD_bug("Node::reloadGridforceGrid(int):grid not found");
01243     }
01244     
01245     if (CkMyPe()) {
01246         // not node 0 -> receive grid
01247         DebugM(4, "Receiving grid\n");
01248         
01249         delete grid;
01250         
01251         MIStream *msg = CkpvAccess(comm)->newInputStream(0, GRIDFORCEGRIDTAG);
01252         grid = GridforceGrid::unpack_grid(gridnum, msg);
01253         molecule->set_gridfrc_grid(gridnum, grid);
01254         delete msg;
01255     } else {
01256         // node 0 -> send grid
01257         DebugM(4, "Sending grid\n");
01258         
01259         MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, GRIDFORCEGRIDTAG, BUFSIZE);
01260         GridforceGrid::pack_grid(grid, msg);
01261         msg->end();
01262         delete msg;
01263     }
01264     
01265     DebugM(4, "reloadGridforceGrid(int) finished\n" << endi);
01266 }
01267 // END gf
01268 
01269 
01270 // initiating replica
01271 void Node::sendCheckpointReq(int remote, const char *key, int task, Lattice &lat, ControllerState &cs) {
01272   CheckpointMsg *msg = new (1+strlen(key),0) CheckpointMsg;
01273   msg->replica = CmiMyPartition();
01274   msg->task = task;
01275   msg->checkpoint.lattice = lat;
01276   msg->checkpoint.state = cs;
01277   strcpy(msg->key,key);
01278   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01279   CmiSetHandler(env,recvCheckpointCReq_index);
01280 #if CMK_HAS_PARTITION
01281   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01282 #else
01283   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01284 #endif
01285 }
01286 
01287 // responding replica
01288 extern "C" {
01289   void recvCheckpointCReq_handler(envelope *env) {
01290     Node::Object()->recvCheckpointReq(CheckpointMsg::unpack(EnvToUsr(env)));
01291   }
01292 }
01293 
01294 // responding replica
01295 void Node::recvCheckpointReq(CheckpointMsg *msg) {
01296   state->controller->recvCheckpointReq(msg->key,msg->task,msg->checkpoint);
01297 
01298   int remote = msg->replica;
01299   msg->replica = CmiMyPartition();
01300   envelope *env = UsrToEnv(CheckpointMsg::pack(msg));
01301   CmiSetHandler(env,recvCheckpointCAck_index);
01302 #if CMK_HAS_PARTITION
01303   CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
01304 #else
01305   CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
01306 #endif
01307 }
01308 
01309 // initiating replica
01310 extern "C" {
01311   void recvCheckpointCAck_handler(envelope *env) {
01312     Node::Object()->recvCheckpointAck(CheckpointMsg::unpack(EnvToUsr(env)));
01313   }
01314 }
01315 
01316 // initiating replica
01317 void Node::recvCheckpointAck(CheckpointMsg *msg) {
01318   state->controller->recvCheckpointAck(msg->checkpoint);
01319   delete msg;
01320 }
01321 
01322 
01323 void Node::sendEnableExitScheduler(void) {
01324   //CmiPrintf("sendEnableExitScheduler\n");
01325   CProxy_Node nodeProxy(thisgroup);
01326   nodeProxy[0].recvEnableExitScheduler();
01327 }
01328 
01329 void Node::recvEnableExitScheduler(void) {
01330   //CmiPrintf("recvEnableExitScheduler\n");
01331   enableExitScheduler();
01332 }
01333 
01334 void Node::enableExitScheduler(void) {
01335   if ( CkMyPe() ) {
01336     sendEnableExitScheduler();
01337   } else {
01338     CkStartQD(CkIndex_Node::exitScheduler(), &thishandle);
01339   }
01340 }
01341 
01342 void Node::exitScheduler(void) {
01343   //CmiPrintf("exitScheduler %d\n",CkMyPe());
01344   CsdExitScheduler();
01345 }
01346 
01347 void Node::sendEnableEarlyExit(void) {
01348   CProxy_Node nodeProxy(thisgroup);
01349   nodeProxy[0].recvEnableEarlyExit();
01350 }
01351 
01352 void Node::recvEnableEarlyExit() {
01353   enableEarlyExit();
01354 }
01355 
01356 void Node::enableEarlyExit(void) {
01357   if ( CkMyPe() ) {
01358     sendEnableEarlyExit();
01359   } else {
01360     CkStartQD(CkIndex_Node::earlyExit(),&thishandle);
01361   }
01362 }
01363 
01364 void Node::earlyExit(void) {
01365   NAMD_die("Exiting prematurely; see error messages above.");
01366 }
01367 
01368 
01369 //------------------------------------------------------------------------
01370 // Some odd utilities
01371 //------------------------------------------------------------------------
01372 void Node::saveMolDataPointers(NamdState *state)
01373 {
01374   this->molecule = state->molecule;
01375   this->parameters = state->parameters;
01376   this->simParameters = state->simParameters;
01377   this->configList = state->configList;
01378   this->pdb = state->pdb;
01379   this->state = state;
01380 }
01381 
01382 // entry methods for BG/P HPM (performance counters) library
01383 void Node::startHPM() {
01384 #if USE_HPM
01385   HPM_Start("500 steps", localRankOnNode);
01386 #endif
01387 }
01388 
01389 void Node::stopHPM() {
01390 #if USE_HPM
01391   HPM_Stop("500 steps", localRankOnNode);
01392   HPM_Print(CkMyPe(), localRankOnNode);
01393 #endif
01394 }
01395 
01396 void Node::traceBarrier(int turnOnTrace, int step){
01397         curTimeStep = step;
01398         if(turnOnTrace) traceBegin();
01399         else traceEnd();
01400 
01401     if(turnOnTrace) CmiTurnOnStats();
01402     else CmiTurnOffStats();
01403 
01404         //CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnTrace, step, CkMyPe());    
01405         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01406         CkCallback cb(CkIndex_Node::resumeAfterTraceBarrier(NULL), nd[0]);
01407         contribute(0, NULL, CkReduction::sum_int, cb);
01408         
01409 }
01410 
01411 void Node::resumeAfterTraceBarrier(CkReductionMsg *msg){
01412         CmiAssert(CmiMyPe()==0);
01413         delete msg;     
01414         state->controller->resumeAfterTraceBarrier(curTimeStep);
01415 }
01416 
01417 void Node::papiMeasureBarrier(int turnOnMeasure, int step){
01418 #ifdef MEASURE_NAMD_WITH_PAPI
01419         curMFlopStep = step;
01420         double results[NUM_PAPI_EVENTS+1];
01421 
01422         if(turnOnMeasure){              
01423           CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]=CmiWallTimer();
01424 
01425           long long counters[NUM_PAPI_EVENTS+1];
01426           int ret=PAPI_start_counters(CkpvAccess(papiEvents), NUM_PAPI_EVENTS);
01427           if(ret==PAPI_OK)
01428             {
01429               //              CkPrintf("traceBarrier start counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01430             }
01431           else
01432             {
01433               CkPrintf("error PAPI_start_counters (%d) at step %d called on proc %d\n",ret , step, CkMyPe());
01434             }
01435           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)!=PAPI_OK)
01436             {
01437               CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01438             };
01439         }else{
01440           long long counters[NUM_PAPI_EVENTS+1];
01441           for(int i=0;i<NUM_PAPI_EVENTS;i++)  counters[i]=0LL;
01442           if(PAPI_read_counters(counters, NUM_PAPI_EVENTS)==PAPI_OK)
01443             {
01444 #if !MEASURE_PAPI_SPP
01445               results[0] = (double)counters[0]/1e6;
01446               results[1] = (double)counters[1]/1e6;
01447 #else
01448               for(int i=0;i<NUM_PAPI_EVENTS;i++)  results[i] = counters[i]/1e6;
01449 #endif
01450               //              for(int i=0;i<NUM_PAPI_EVENTS;i++) CkPrintf("[%d] counter %d is %ld\n",CkMyPe(),i,counters[i]);
01451             }
01452           else
01453             {
01454               //              CkPrintf("error PAPI_read_counters %d\n",PAPI_read_counters(counters, NUM_PAPI_EVENTS));
01455             }
01456           //      CkPrintf("traceBarrier stop counters (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01457                 
01458           PAPI_stop_counters(counters, NUM_PAPI_EVENTS);        
01459         }
01460         if(CkMyPe()==0)
01461           //        CkPrintf("traceBarrier (%d) at step %d called on proc %d\n", turnOnMeasure, step, CkMyPe());
01462         results[NUM_PAPI_EVENTS]=CkpvAccess(papiEvents)[NUM_PAPI_EVENTS]; //starttime
01463         CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01464         CkCallback cb(CkIndex_Node::resumeAfterPapiMeasureBarrier(NULL), nd[0]);
01465         contribute(sizeof(double)*(NUM_PAPI_EVENTS+1), &results, CkReduction::sum_double, cb);  
01466 #endif
01467 }
01468 
01469 void Node::resumeAfterPapiMeasureBarrier(CkReductionMsg *msg){
01470 #ifdef MEASURE_NAMD_WITH_PAPI
01471   
01472         if(simParameters->papiMeasureStartStep != curMFlopStep) {
01473                 double *results = (double *)msg->getData();
01474                 double endtime=CmiWallTimer();
01475                 int bstep = simParameters->papiMeasureStartStep;
01476                 int estep = bstep + simParameters->numPapiMeasureSteps;
01477 #if MEASURE_PAPI_SPP
01478                 CkPrintf("SPP INFO: PAPI_FP_OPS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[0]);
01479                 CkPrintf("SPP INFO: PAPI_TOT_INS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[1]);
01480                 CkPrintf("SPP INFO: perf::PERF_COUNT_HW_CACHE_LL:MISS timestep %d to %d is %lf(1e6)\n", bstep,estep,results[2]);
01481                 CkPrintf("SPP INFO: DATA_PREFETCHER:ALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[3]);
01482                 CkPrintf("SPP INFO: PAPI_L1_DCA timestep %d to %d is %lf(1e6)\n", bstep,estep,results[4]);
01483                 CkPrintf("SPP INFO: PAPI_TOT_CYC timestep %d to % is %lf(1e6)\n", bstep,estep,results[5]);
01484                 //              CkPrintf("SPP INFO: INSTRUCTION_FETCH_STALL timestep %d to %d is %lf(1e6)\n", bstep,estep,results[6]);
01485                 //              CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]/CkNumPes());
01486                 CkPrintf("SPP INFO: WALLtime timestep %d to %d is %lf\n", bstep,estep,endtime-results[NUM_PAPI_EVENTS]);
01487                 CkPrintf("SPP INFO: endtime %lf avgtime %lf tottime %lf\n", endtime,results[NUM_PAPI_EVENTS]/CkNumPes(),results[NUM_PAPI_EVENTS] );
01488 #else
01489                 if(CkpvAccess(papiEvents)[0] == PAPI_FP_INS){
01490                         double totalFPIns = results[0];
01491                         if(CkpvAccess(papiEvents)[1] == PAPI_FMA_INS) totalFPIns += (results[1]*2);
01492                         CkPrintf("FLOPS INFO: from timestep %d to %d, the total FP instruction of NAMD is %lf(x1e6) per processor\n", 
01493                                          bstep, estep, totalFPIns/CkNumPes());
01494                 }else{
01495                         char nameBuf[PAPI_MAX_STR_LEN];
01496                         CkPrintf("PAPI COUNTERS INFO: from timestep %d to %d, ", 
01497                                          bstep, estep);
01498                         for(int i=0; i<NUM_PAPI_EVENTS; i++) {
01499                                 PAPI_event_code_to_name(CkpvAccess(papiEvents)[i], nameBuf);
01500                                 CkPrintf("%s is %lf(x1e6), ", nameBuf, results[i]/CkNumPes());
01501                         }
01502                         CkPrintf("per processor\n");
01503                 }               
01504 #endif
01505         }
01506         delete msg;     
01507         state->controller->resumeAfterPapiMeasureBarrier(curMFlopStep);
01508 #endif
01509 }
01510 
01511 extern char *gNAMDBinaryName;
01512 void Node::outputPatchComputeMaps(const char *filename, int tag){
01513         if(!simParameters->outputMaps && !simParameters->simulateInitialMapping) return;
01514 
01515         int numpes = CkNumPes();
01516         int nodesize = CkMyNodeSize();
01517         if(simParameters->simulateInitialMapping) {
01518                 numpes = simParameters->simulatedPEs;
01519                 nodesize = simParameters->simulatedNodeSize;
01520         }
01521 
01522         char fname[128];
01523         sprintf(fname, "mapdump_%s.%d_%d_%d_%s", filename, numpes, nodesize, tag, gNAMDBinaryName);
01524 
01525         FILE *fp = fopen(fname, "w");
01526         if(fp == NULL) {
01527                 NAMD_die("Error in outputing PatchMap and ComputeMap info!\n");
01528                 return;
01529         }
01530         PatchMap *pMap = PatchMap::Object();
01531         ComputeMap *cMap = ComputeMap::Object();
01532         int numPatches = pMap->numPatches();
01533         int numComputes = cMap->numComputes();
01534         fprintf(fp, "%d %d %d %d %d %d %d\n", numpes, nodesize, numPatches, numComputes, 
01535                         pMap->gridsize_a(), pMap->gridsize_b(), pMap->gridsize_c());
01536         //output PatchMap info
01537         for(int i=0; i<numPatches; i++) {
01538         #ifdef MEM_OPT_VERSION
01539                 fprintf(fp, "%d %d\n", pMap->numAtoms(i), pMap->node(i));
01540         #else
01541                 fprintf(fp, "%d %d\n", pMap->patch(i)->getNumAtoms(), pMap->node(i));
01542         #endif
01543         }
01544 
01545         //output ComputeMap info
01546         for(int i=0; i<numComputes; i++) {              
01547                 fprintf(fp, "%d %d %d %d\n", cMap->node(i), cMap->type(i), cMap->pid(i,0), cMap->pid(i,1));             
01548         }
01549 }
01550 
01551 
01552 //======================================================================
01553 // Private functions
01554 
01555 #include "Node.def.h"
01556