WorkDistrib Class Reference

#include <WorkDistrib.h>

List of all members.

Classes

struct  pe_sortop_compact
struct  pe_sortop_diffuse

Public Member Functions

 WorkDistrib ()
 ~WorkDistrib (void)
void enqueueWork (LocalWorkMsg *msg)
void enqueueExcls (LocalWorkMsg *msg)
void enqueueBonds (LocalWorkMsg *msg)
void enqueueAngles (LocalWorkMsg *msg)
void enqueueDihedrals (LocalWorkMsg *msg)
void enqueueImpropers (LocalWorkMsg *msg)
void enqueueThole (LocalWorkMsg *msg)
void enqueueAniso (LocalWorkMsg *msg)
void enqueueCrossterms (LocalWorkMsg *msg)
void enqueueGromacsPair (LocalWorkMsg *msg)
void enqueuePme (LocalWorkMsg *msg)
void enqueueSelfA1 (LocalWorkMsg *msg)
void enqueueSelfA2 (LocalWorkMsg *msg)
void enqueueSelfA3 (LocalWorkMsg *msg)
void enqueueSelfB1 (LocalWorkMsg *msg)
void enqueueSelfB2 (LocalWorkMsg *msg)
void enqueueSelfB3 (LocalWorkMsg *msg)
void enqueueWorkA1 (LocalWorkMsg *msg)
void enqueueWorkA2 (LocalWorkMsg *msg)
void enqueueWorkA3 (LocalWorkMsg *msg)
void enqueueWorkB1 (LocalWorkMsg *msg)
void enqueueWorkB2 (LocalWorkMsg *msg)
void enqueueWorkB3 (LocalWorkMsg *msg)
void enqueueWorkC (LocalWorkMsg *msg)
void enqueueCUDA (LocalWorkMsg *msg)
void enqueueCUDAP2 (LocalWorkMsg *msg)
void enqueueCUDAP3 (LocalWorkMsg *msg)
void finishCUDAPatch (FinishWorkMsg *msg)
void finishCUDA (LocalWorkMsg *msg)
void finishCUDAP2 (LocalWorkMsg *msg)
void finishCUDAP3 (LocalWorkMsg *msg)
void enqueueMIC (LocalWorkMsg *msg)
void finishMIC (LocalWorkMsg *msg)
void enqueueLCPO (LocalWorkMsg *msg)
void mapComputes (void)
void sendPatchMap (void)
void sendComputeMap (void)
void saveComputeMapChanges (int, CkGroupID)
void recvComputeMapChanges (ComputeMapChangeMsg *)
void doneSaveComputeMap (CkReductionMsg *)
FullAtomListcreateAtomLists (const char *basename=0)
void createHomePatches (void)
void distributeHomePatches (void)
void reinitAtoms (const char *basename=0)
void patchMapInit (void)
void assignNodeToPatch (void)
void savePatchMap (PatchMapMsg *msg)
void setPatchMapArrived (bool s)
void initHostDeviceLDB ()
void contributeHostDeviceLDB (int peSetLen, int *peSet)
void setDeviceLDBParams (int dt, int hs, int sp1, int pp1, int pp2)

Static Public Member Functions

static void messageEnqueueWork (Compute *)
static void messageFinishCUDA (Compute *)
static void messageFinishMIC (Compute *)
static void sortPmePes (int *pmepes, int xdim, int ydim)
static void peOrderingReady ()
static void send_initHostDeviceLDB ()
static void send_contributeHostDeviceLDB (int peSetLen, int *peSet)
static void send_setDeviceLDBParams (int dt, int hs, int sp1, int pp1, int pp2)
static void buildNodeAwarePeOrdering (void)

Static Public Attributes

static int peOrderingInit
static int * peDiffuseOrdering
static int * peDiffuseOrderingIndex
static int * peCompactOrdering
static int * peCompactOrderingIndex

Detailed Description

Definition at line 42 of file WorkDistrib.h.


Constructor & Destructor Documentation

WorkDistrib::WorkDistrib (  ) 

Definition at line 105 of file WorkDistrib.C.

References eventMachineProgress, and NAMD_bug().

00106 {
00107   CkpvAccess(BOCclass_group).workDistrib = thisgroup;
00108   patchMapArrived = false;
00109   computeMapArrived = false;
00110 
00111 #if CMK_SMP
00112 #define MACHINE_PROGRESS
00113 #else
00114 #define MACHINE_PROGRESS { traceUserEvent(eventMachineProgress);  CmiMachineProgressImpl(); }
00115   if ( CkMyNodeSize() > 1 ) NAMD_bug("CkMyNodeSize() > 1 for non-smp build");
00116   eventMachineProgress = traceRegisterUserEvent("CmiMachineProgressImpl",233);
00117 #endif
00118 }

WorkDistrib::~WorkDistrib ( void   ) 

Definition at line 121 of file WorkDistrib.C.

00122 { }


Member Function Documentation

void WorkDistrib::assignNodeToPatch ( void   ) 

Definition at line 1323 of file WorkDistrib.C.

References endi(), Patch::getNumAtoms(), iINFO(), iout, Node::molecule, NAMD_die(), PatchMap::node(), Molecule::numAtoms, Node::numNodes(), PatchMap::numPatches(), Node::Object(), PatchMap::Object(), PatchMap::patch(), Node::simParameters, SimParameters::simulatedPEs, and SimParameters::simulateInitialMapping.

Referenced by Node::startup().

01324 {
01325   PatchMap *patchMap = PatchMap::Object();
01326   int nNodes = Node::Object()->numNodes();
01327   SimParameters *simparam = Node::Object()->simParameters;
01328   if(simparam->simulateInitialMapping) {
01329           nNodes = simparam->simulatedPEs;
01330   }
01331 
01332 #if (CMK_BLUEGENEP | CMK_BLUEGENEL) && USE_TOPOMAP 
01333   TopoManager tmgr;
01334   int numPes = tmgr.getDimNX() * tmgr.getDimNY() * tmgr.getDimNZ();
01335   if (numPes > patchMap->numPatches() && (assignPatchesTopoGridRecBisection() > 0)) {
01336     CkPrintf ("Blue Gene/L topology partitioner finished successfully \n");
01337   }
01338   else
01339 #endif
01340   assignPatchesSpaceFillingCurve();       
01341   
01342   int *nAtoms = new int[nNodes];
01343   int numAtoms=0;
01344   int i;
01345   for(i=0; i < nNodes; i++)
01346     nAtoms[i] = 0;
01347 
01348   for(i=0; i < patchMap->numPatches(); i++)
01349   {
01350     //    iout << iINFO << "Patch " << i << " has " 
01351     //   << patchMap->patch(i)->getNumAtoms() << " atoms and "
01352     //   << patchMap->patch(i)->getNumAtoms() * 
01353     //            patchMap->patch(i)->getNumAtoms() 
01354     //   << " pairs.\n" << endi;         
01355 #ifdef MEM_OPT_VERSION
01356       numAtoms += patchMap->numAtoms(i);
01357       nAtoms[patchMap->node(i)] += patchMap->numAtoms(i);         
01358 #else
01359     if (patchMap->patch(i)) {
01360       numAtoms += patchMap->patch(i)->getNumAtoms();
01361       nAtoms[patchMap->node(i)] += patchMap->patch(i)->getNumAtoms();     
01362     }
01363 #endif
01364   }
01365 
01366   if ( numAtoms != Node::Object()->molecule->numAtoms ) {
01367     for(i=0; i < nNodes; i++)
01368       iout << iINFO << nAtoms[i] << " atoms assigned to node " << i << "\n" << endi;
01369     iout << iINFO << "Assigned " << numAtoms << " atoms but expected " << Node::Object()->molecule->numAtoms << "\n" << endi;
01370     NAMD_die("Incorrect atom count in WorkDistrib::assignNodeToPatch\n");
01371   }
01372 
01373   delete [] nAtoms;
01374  
01375   //  PatchMap::Object()->printPatchMap();
01376 }

void WorkDistrib::buildNodeAwarePeOrdering ( void   )  [static]

Definition at line 180 of file WorkDistrib.C.

References ResizeArray< Elem >::begin(), endi(), iout, iWARN(), j, peCompactOrdering, peCompactOrderingIndex, peDiffuseOrdering, peDiffuseOrderingIndex, peOrderingInit, peOrderingReady(), randtopo, and sort.

Referenced by build_ordering().

00180                                            {
00181 
00182  CmiMemLock();
00183  if ( ! peOrderingInit ) {
00184   //CkPrintf("WorkDistrib::buildNodeAwarePeOrdering on pe %d\n", CkMyPe());
00185 
00186   const int numPhys = CmiNumPhysicalNodes();
00187   const int numNode = CmiNumNodes();
00188   const int numPe = CmiNumPes();
00189   ResizeArray<int> numNodeInPhys(numPhys);
00190   ResizeArray<int> rankInPhysOfNode(numNode);
00191 
00192   peDiffuseOrdering = new int[numPe];
00193   peDiffuseOrderingIndex = new int[numPe];
00194   peCompactOrdering = new int[numPe];
00195   peCompactOrderingIndex = new int[numPe];
00196 
00197   int k = 0;
00198   for ( int ph=0; ph<numPhys; ++ph ) {
00199     int *pes, npes;
00200     CmiGetPesOnPhysicalNode(ph, &pes, &npes);
00201     for ( int i=0; i<npes; ++i, ++k ) {
00202       peCompactOrdering[k] = pes[i];
00203     }
00204     numNodeInPhys[ph] = 0;
00205     for ( int i=0, j=0; i<npes; i += CmiNodeSize(CmiNodeOf(pes[i])), ++j ) {
00206       rankInPhysOfNode[CmiNodeOf(pes[i])] = j;
00207       numNodeInPhys[ph] += 1;
00208     }
00209   }
00210 
00211   if ( randtopo && numPhys > 2 ) {
00212     if ( ! CkMyNode() ) {
00213       iout << iWARN << "RANDOMIZING PHYSICAL NODE ORDERING\n" << endi;
00214     }
00215     ResizeArray<int> randPhysOrder(numPhys);
00216     for ( int j=0; j<numPhys; ++j ) {
00217       randPhysOrder[j] = j;
00218     }
00219     Random(314159265).reorder(randPhysOrder.begin()+2, numPhys-2);
00220     for ( int j=0, k=0; j<numPhys; ++j ) {
00221       const int ph = randPhysOrder[j];
00222       int *pes, npes;
00223       CmiGetPesOnPhysicalNode(ph, &pes, &npes);
00224       for ( int i=0; i<npes; ++i, ++k ) {
00225         peCompactOrdering[k] = pes[i];
00226       }
00227     }
00228   }
00229 
00230   for ( int i=0; i<numPe; ++i ) {
00231     peDiffuseOrdering[i] = i;
00232   }
00233   std::sort(peDiffuseOrdering, peDiffuseOrdering+numPe,
00234     pe_sortop_bit_reversed(rankInPhysOfNode.begin()));
00235 
00236   for ( int i=0; i<numPe; ++i ) {
00237     peDiffuseOrderingIndex[peDiffuseOrdering[i]] = i;
00238     peCompactOrderingIndex[peCompactOrdering[i]] = i;
00239   }
00240 
00241   if ( 0 && CmiMyNode() == 0 ) for ( int i=0; i<numPe; ++i ) {
00242     CkPrintf("order %5d %5d %5d %5d %5d\n", i,
00243       peDiffuseOrdering[i],
00244       peDiffuseOrderingIndex[i],
00245       peCompactOrdering[i],
00246       peCompactOrderingIndex[i]);
00247   }
00248 
00249   peOrderingInit = 1;
00250  }
00251  CmiMemUnlock();
00252  peOrderingReady();
00253 
00254 }

void WorkDistrib::contributeHostDeviceLDB ( int  peSetLen,
int *  peSet 
)

Definition at line 3402 of file WorkDistrib.C.

03402                                                                    {
03403   #if defined(NAMD_MIC)
03404     mic_contributeHostDeviceLDB(peSetLen, peSet);
03405   #endif
03406 }

FullAtomList * WorkDistrib::createAtomLists ( const char *  basename = 0  ) 

Definition at line 622 of file WorkDistrib.C.

References ResizeArray< Elem >::add(), SimParameters::alchOn, Lattice::apply_transform(), PatchMap::assignToPatch(), Molecule::atomcharge(), CompAtomExt::atomFixed, HydrogenGroupID::atomID, Molecule::atommass(), atoms, HydrogenGroupID::atomsInGroup, HydrogenGroupID::atomsInMigrationGroup, Molecule::atomvdwtype(), ResizeArray< Elem >::begin(), SimParameters::binaryOutput, CompAtom::charge, SimParameters::comMove, Node::configList, StringList::data, endi(), FALSE, ConfigList::find(), FullAtom::fixedPosition, PDB::get_all_positions(), Molecule::get_fep_type(), Molecule::getAtoms(), PatchMap::gridsize_a(), PatchMap::gridsize_b(), PatchMap::gridsize_c(), CompAtomExt::groupFixed, Molecule::hydrogenGroup, CompAtom::hydrogenGroupSize, CompAtomExt::id, PatchMap::index_a(), PatchMap::index_b(), PatchMap::index_c(), SimParameters::initialTemp, iout, Molecule::is_atom_fixed(), HydrogenGroupID::isGP, HydrogenGroupID::isMP, Molecule::langevin_param(), FullAtom::langevinParam, SimParameters::lattice, SimParameters::lesOn, FullAtom::mass, PatchMap::max_a(), PatchMap::max_b(), PatchMap::max_c(), FullAtom::migrationGroupSize, PatchMap::min_a(), PatchMap::min_b(), PatchMap::min_c(), Node::molecule, NAMD_bug(), NAMD_die(), Lattice::nearest(), CompAtom::nonbondedGroupSize, PDB::num_atoms(), PatchMap::numPatches(), numPatches, PatchMap::Object(), order, SimParameters::outputPatchDetails, SimParameters::pairInteractionOn, CompAtom::partition, Node::pdb, CompAtom::position, SimParameters::pressureProfileAtomTypes, read_binary_file(), FullAtom::recipMass, Molecule::rigid_bond_length(), RIGID_NONE, FullAtom::rigidBondLength, SimParameters::rigidBonds, Node::simParameters, ResizeArray< Elem >::size(), sortAtomsForPatches(), SimParameters::staticAtomAssignment, atom_constants::status, FullAtom::status, FullAtom::transform, TRUE, CompAtom::vdwType, and FullAtom::velocity.

Referenced by createHomePatches(), and reinitAtoms().

00623 {
00624   int i;
00625   StringList *current;  //  Pointer used to retrieve configuration items
00626   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00627   Node *node = nd.ckLocalBranch();
00628   PatchMap *patchMap = PatchMap::Object();
00629   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00630   PatchMgr *patchMgr = pm.ckLocalBranch();
00631   SimParameters *params = node->simParameters;
00632   Molecule *molecule = node->molecule;
00633   PDB *pdb = node->pdb;
00634 
00635   int numPatches = patchMap->numPatches();
00636   int numAtoms = pdb->num_atoms();
00637 
00638   Vector *positions = new Position[numAtoms];
00639   Vector *velocities = new Velocity[numAtoms];
00640 
00641  if ( basename ) {
00642   if ( params->binaryOutput ) {
00643     read_binary_file((std::string(basename)+".coor").c_str(), positions, numAtoms);
00644     read_binary_file((std::string(basename)+".vel").c_str(), velocities, numAtoms);
00645   } else {
00646     PDB coorpdb((std::string(basename)+".coor").c_str());
00647     if ( coorpdb.num_atoms() != numAtoms ) {
00648       NAMD_die("Incorrect atom count in coordinate pdb file");
00649     }
00650     coorpdb.get_all_positions(positions);
00651     velocities_from_PDB((std::string(basename)+".vel").c_str(), velocities, numAtoms);
00652   }
00653  } else {
00654   pdb->get_all_positions(positions);
00655 
00656   if ( params->initialTemp < 0.0 ) {
00657     Bool binvels=FALSE;
00658 
00659     //  Reading the veolcities from a PDB
00660     current = node->configList->find("velocities");
00661 
00662     if (current == NULL) {
00663       current = node->configList->find("binvelocities");
00664       binvels = TRUE;
00665     }
00666 
00667     if (!binvels) {
00668       velocities_from_PDB(current->data, velocities, numAtoms);
00669     }
00670     else {
00671       velocities_from_binfile(current->data, velocities, numAtoms);
00672     }
00673   }
00674   else {
00675     // Random velocities for a given temperature
00676     random_velocities(params->initialTemp, molecule, velocities, numAtoms);
00677   }
00678  }
00679 
00680   //  If COMMotion == no, remove center of mass motion
00681   if (!(params->comMove)) {
00682     remove_com_motion(velocities, molecule, numAtoms);
00683   }
00684 
00685   FullAtomList *atoms = new FullAtomList[numPatches];
00686 
00687   const Lattice lattice = params->lattice;
00688 
00689     if ( params->staticAtomAssignment ) {
00690       FullAtomList sortAtoms;
00691       for ( i=0; i < numAtoms; i++ ) {
00692         HydrogenGroupID &h = molecule->hydrogenGroup[i];
00693         if ( ! h.isMP ) continue;
00694         FullAtom a;
00695         a.id = i;
00696         a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
00697         a.position = positions[h.atomID];
00698         sortAtoms.add(a);
00699       } 
00700       int *order = new int[sortAtoms.size()];
00701       for ( i=0; i < sortAtoms.size(); i++ ) {
00702         order[i] = i;
00703       } 
00704       int *breaks = new int[numPatches];
00705       sortAtomsForPatches(order,breaks,sortAtoms.begin(),
00706                         sortAtoms.size(),numAtoms,
00707                         patchMap->gridsize_c(),
00708                         patchMap->gridsize_b(),
00709                         patchMap->gridsize_a());
00710 
00711       i = 0;
00712       for ( int pid = 0; pid < numPatches; ++pid ) {
00713         int iend = breaks[pid];
00714         for ( ; i<iend; ++i ) {
00715           FullAtom &sa = sortAtoms[order[i]];
00716           int mgs = sa.migrationGroupSize;
00717 /*
00718 CkPrintf("patch %d (%d %d %d) has group %d atom %d size %d at %.2f %.2f %.2f\n",
00719           pid, patchMap->index_a(pid), patchMap->index_b(pid), 
00720           patchMap->index_c(pid), order[i], sa.id, mgs,
00721           sa.position.x, sa.position.y, sa.position.z);
00722 */
00723           for ( int k=0; k<mgs; ++k ) {
00724             HydrogenGroupID &h = molecule->hydrogenGroup[sa.id + k];
00725             int aid = h.atomID;
00726             FullAtom a;
00727             a.id = aid;
00728             a.position = positions[aid];
00729             a.velocity = velocities[aid];
00730             a.vdwType = molecule->atomvdwtype(aid);
00731             a.status = molecule->getAtoms()[aid].status;
00732             a.langevinParam = molecule->langevin_param(aid);
00733             a.hydrogenGroupSize = h.isGP ? h.atomsInGroup : 0;
00734             a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
00735             if(params->rigidBonds != RIGID_NONE) {
00736               a.rigidBondLength = molecule->rigid_bond_length(aid);
00737             }else{
00738               a.rigidBondLength = 0.0;
00739             }
00740             atoms[pid].add(a);
00741           }
00742         }
00743 CkPrintf("patch %d (%d %d %d) has %d atoms\n",
00744           pid, patchMap->index_a(pid), patchMap->index_b(pid), 
00745           patchMap->index_c(pid), atoms[pid].size());
00746       }
00747       delete [] order;
00748       delete [] breaks;
00749     } else
00750     {
00751     // split atoms into patches based on migration group and position
00752     int aid, pid=0;
00753     for(i=0; i < numAtoms; i++)
00754       {
00755       // Assign atoms to patches without splitting hydrogen groups.
00756       // We know that the hydrogenGroup array is sorted with group parents
00757       // listed first.  Thus, only change the pid if an atom is a group parent.
00758       HydrogenGroupID &h = molecule->hydrogenGroup[i];
00759       aid = h.atomID;
00760       FullAtom a;
00761       a.id = aid;
00762       a.position = positions[aid];
00763       a.velocity = velocities[aid];
00764       a.vdwType = molecule->atomvdwtype(aid);
00765       a.status = molecule->getAtoms()[aid].status;
00766       a.langevinParam = molecule->langevin_param(aid);
00767       a.hydrogenGroupSize = h.isGP ? h.atomsInGroup : 0;
00768       a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
00769       if(params->rigidBonds != RIGID_NONE) {
00770         a.rigidBondLength = molecule->rigid_bond_length(aid);
00771       }else{
00772         a.rigidBondLength = 0.0;
00773       }
00774       if (h.isMP) {
00775         pid = patchMap->assignToPatch(positions[aid],lattice);
00776       } // else: don't change pid
00777       atoms[pid].add(a);
00778       }
00779     }
00780 
00781   delete [] positions;
00782   delete [] velocities;
00783 
00784   for(i=0; i < numPatches; i++)
00785   {
00786     ScaledPosition center(0.5*(patchMap->min_a(i)+patchMap->max_a(i)),
00787                           0.5*(patchMap->min_b(i)+patchMap->max_b(i)),
00788                           0.5*(patchMap->min_c(i)+patchMap->max_c(i)));
00789 
00790     int n = atoms[i].size();
00791     FullAtom *a = atoms[i].begin();
00792     int j;
00793 //Modifications for alchemical fep
00794     Bool alchOn = params->alchOn;
00795 //fepe
00796     Bool lesOn = params->lesOn;
00797   
00798     Bool pairInteractionOn = params->pairInteractionOn;
00799 
00800     Bool pressureProfileTypes = (params->pressureProfileAtomTypes > 1);
00801 
00802     Transform mother_transform;
00803     for(j=0; j < n; j++)
00804     {
00805       int aid = a[j].id;
00806 
00807       a[j].nonbondedGroupSize = 0;  // must be set based on coordinates
00808 
00809       a[j].atomFixed = molecule->is_atom_fixed(aid) ? 1 : 0;
00810       a[j].fixedPosition = a[j].position;
00811 
00812       if ( a[j].migrationGroupSize ) {
00813        if ( a[j].migrationGroupSize != a[j].hydrogenGroupSize ) {
00814             Position pos = a[j].position;
00815             int mgs = a[j].migrationGroupSize;
00816             int c = 1;
00817             for ( int k=a[j].hydrogenGroupSize; k<mgs;
00818                                 k+=a[j+k].hydrogenGroupSize ) {
00819               pos += a[j+k].position;
00820               ++c;
00821             }
00822             pos *= 1./c;
00823             mother_transform = a[j].transform;  // should be 0,0,0
00824             pos = lattice.nearest(pos,center,&mother_transform);
00825             a[j].position = lattice.apply_transform(a[j].position,mother_transform);
00826             a[j].transform = mother_transform;
00827        } else {
00828         a[j].position = lattice.nearest(
00829                 a[j].position, center, &(a[j].transform));
00830         mother_transform = a[j].transform;
00831        }
00832       } else {
00833         a[j].position = lattice.apply_transform(a[j].position,mother_transform);
00834         a[j].transform = mother_transform;
00835       }
00836 
00837       a[j].mass = molecule->atommass(aid);
00838       // Using double precision division for reciprocal mass.
00839       a[j].recipMass = ( a[j].mass > 0 ? (1. / a[j].mass) : 0 );
00840       a[j].charge = molecule->atomcharge(aid);
00841 
00842 //Modifications for alchemical fep
00843       if ( alchOn || lesOn || pairInteractionOn || pressureProfileTypes) {
00844         a[j].partition = molecule->get_fep_type(aid);
00845       } 
00846       else {
00847         a[j].partition = 0;
00848       }
00849 //fepe
00850 
00851     }
00852 
00853     int size, allfixed, k;
00854     for(j=0; j < n; j+=size) {
00855       size = a[j].hydrogenGroupSize;
00856       if ( ! size ) {
00857         NAMD_bug("Mother atom with hydrogenGroupSize of 0!");
00858       }
00859       allfixed = 1;
00860       for ( k = 0; k < size; ++k ) {
00861         allfixed = ( allfixed && (a[j+k].atomFixed) );
00862       }
00863       for ( k = 0; k < size; ++k ) {
00864         a[j+k].groupFixed = allfixed ? 1 : 0;
00865       }
00866     }
00867 
00868     if ( params->outputPatchDetails ) {
00869       int patchId = i;
00870       int numAtomsInPatch = n;
00871       int numFixedAtomsInPatch = 0;
00872       int numAtomsInFixedGroupsInPatch = 0;
00873       for(j=0; j < n; j++) {
00874         numFixedAtomsInPatch += ( a[j].atomFixed ? 1 : 0 );
00875         numAtomsInFixedGroupsInPatch += ( a[j].groupFixed ? 1 : 0 );
00876       }
00877       iout << "PATCH_DETAILS:"
00878            << " patch " << patchId
00879            << " atoms " << numAtomsInPatch
00880            << " fixed_atoms " << numFixedAtomsInPatch
00881            << " fixed_groups " << numAtomsInFixedGroupsInPatch
00882            << "\n" << endi;
00883     }
00884   }
00885 
00886   return atoms;
00887 
00888 }

void WorkDistrib::createHomePatches ( void   ) 

Definition at line 893 of file WorkDistrib.C.

References atoms, createAtomLists(), PatchMgr::createHomePatch(), DebugM, endi(), iINFO(), iout, PatchMap::numPatches(), numPatches, PatchMap::Object(), and ResizeArray< Elem >::size().

Referenced by Node::startup().

00894 {
00895   int i;
00896   PatchMap *patchMap = PatchMap::Object();
00897   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00898   PatchMgr *patchMgr = pm.ckLocalBranch();
00899 
00900   int numPatches = patchMap->numPatches();
00901 
00902   FullAtomList *atoms = createAtomLists();
00903     
00904 #ifdef MEM_OPT_VERSION
00905 /*  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00906   Node *node = nd.ckLocalBranch();
00907   node->molecule->delEachAtomSigs();
00908   node->molecule->delMassChargeSpace();
00909 */
00910 #endif
00911 
00912   int maxAtoms = -1;
00913   int maxPatch = -1;
00914   for(i=0; i < numPatches; i++) {
00915     int numAtoms = atoms[i].size();
00916     if ( numAtoms > maxAtoms ) { maxAtoms = numAtoms; maxPatch = i; }
00917   }
00918   iout << iINFO << "LARGEST PATCH (" << maxPatch <<
00919         ") HAS " << maxAtoms << " ATOMS\n" << endi;
00920 
00921   for(i=0; i < numPatches; i++)
00922   {
00923     if ( ! ( i % 100 ) )
00924     {
00925       DebugM(3,"Created " << i << " patches so far.\n");
00926     }
00927 
00928     patchMgr->createHomePatch(i,atoms[i]);
00929   }
00930 
00931   delete [] atoms;
00932 }

void WorkDistrib::distributeHomePatches ( void   ) 

Definition at line 934 of file WorkDistrib.C.

References DebugM, PatchMgr::movePatch(), Node::myid(), PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), and PatchMgr::sendMovePatches().

Referenced by Node::startup().

00934                                         {
00935   // ref BOC
00936   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00937   Node *node = nd.ckLocalBranch();
00938   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00939   PatchMgr *patchMgr = pm.ckLocalBranch();
00940   // ref singleton
00941   PatchMap *patchMap = PatchMap::Object();
00942 
00943   // Move patches to the proper node
00944   for(int i=0;i < patchMap->numPatches(); i++)
00945   {
00946     if (patchMap->node(i) != node->myid() )
00947     {
00948       DebugM(3,"patchMgr->movePatch("
00949         << i << "," << patchMap->node(i) << ")\n");
00950       patchMgr->movePatch(i,patchMap->node(i));
00951     }
00952   }
00953   patchMgr->sendMovePatches();
00954 }

void WorkDistrib::doneSaveComputeMap ( CkReductionMsg *  msg  ) 

Definition at line 427 of file WorkDistrib.C.

00427                                                         {
00428   delete msg;
00429 
00430   CkSendMsgBranch(saveComputeMapReturnEP, CkAllocMsg(0,0,0), 0, saveComputeMapReturnChareID);
00431 }

void WorkDistrib::enqueueAngles ( LocalWorkMsg msg  ) 

Definition at line 2973 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02973                                                  {
02974   msg->compute->doWork();  MACHINE_PROGRESS
02975   if ( msg->compute->localWorkMsg != msg )
02976     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
02977 }

void WorkDistrib::enqueueAniso ( LocalWorkMsg msg  ) 

Definition at line 2997 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02997                                                 {
02998   msg->compute->doWork();  MACHINE_PROGRESS
02999   if ( msg->compute->localWorkMsg != msg )
03000     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03001 }

void WorkDistrib::enqueueBonds ( LocalWorkMsg msg  ) 

Definition at line 2967 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02967                                                 {
02968   msg->compute->doWork();  MACHINE_PROGRESS
02969   if ( msg->compute->localWorkMsg != msg )
02970     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
02971 }

void WorkDistrib::enqueueCrossterms ( LocalWorkMsg msg  ) 

Definition at line 3003 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03003                                                      {
03004   msg->compute->doWork();  MACHINE_PROGRESS
03005   if ( msg->compute->localWorkMsg != msg )
03006     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03007 }

void WorkDistrib::enqueueCUDA ( LocalWorkMsg msg  ) 

Definition at line 3101 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03101                                                {
03102   msg->compute->doWork();  MACHINE_PROGRESS
03103   // ComputeNonbondedCUDA *c = msg->compute;
03104   // if ( c->localWorkMsg != msg && c->localWorkMsg2 != msg )
03105   //   NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03106 }

void WorkDistrib::enqueueCUDAP2 ( LocalWorkMsg msg  ) 

Definition at line 3107 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03107                                                  {
03108   msg->compute->doWork();  MACHINE_PROGRESS
03109 }

void WorkDistrib::enqueueCUDAP3 ( LocalWorkMsg msg  ) 

Definition at line 3110 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03110                                                  {
03111   msg->compute->doWork();  MACHINE_PROGRESS
03112 }

void WorkDistrib::enqueueDihedrals ( LocalWorkMsg msg  ) 

Definition at line 2979 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02979                                                     {
02980   msg->compute->doWork();  MACHINE_PROGRESS
02981   if ( msg->compute->localWorkMsg != msg )
02982     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
02983 }

void WorkDistrib::enqueueExcls ( LocalWorkMsg msg  ) 

Definition at line 2961 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02961                                                 {
02962   msg->compute->doWork();  MACHINE_PROGRESS
02963   if ( msg->compute->localWorkMsg != msg )
02964     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
02965 }

void WorkDistrib::enqueueGromacsPair ( LocalWorkMsg msg  ) 

Definition at line 3010 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03010                                                       {
03011   msg->compute->doWork();
03012   MACHINE_PROGRESS
03013   if ( msg->compute->localWorkMsg != msg )
03014     NAMD_bug("\nWorkDistrib LocalWorkMsg recycling failed! Check enqueueGromacsPair from WorkDistrib.C\n");
03015 }

void WorkDistrib::enqueueImpropers ( LocalWorkMsg msg  ) 

Definition at line 2985 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02985                                                     {
02986   msg->compute->doWork();  MACHINE_PROGRESS
02987   if ( msg->compute->localWorkMsg != msg )
02988     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
02989 }

void WorkDistrib::enqueueLCPO ( LocalWorkMsg msg  ) 

Definition at line 3024 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, and NAMD_bug().

03024                                                {
03025   msg->compute->doWork();
03026   if ( msg->compute->localWorkMsg != msg )
03027     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03028 }

void WorkDistrib::enqueueMIC ( LocalWorkMsg msg  ) 

Definition at line 3131 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03131                                               {
03132   msg->compute->doWork();  MACHINE_PROGRESS
03133 }

void WorkDistrib::enqueuePme ( LocalWorkMsg msg  ) 

Definition at line 3018 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03018                                               {
03019   msg->compute->doWork();  MACHINE_PROGRESS
03020   if ( msg->compute->localWorkMsg != msg )
03021     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03022 }

void WorkDistrib::enqueueSelfA1 ( LocalWorkMsg msg  ) 

Definition at line 3029 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03029                                                  {
03030   msg->compute->doWork();  MACHINE_PROGRESS
03031   if ( msg->compute->localWorkMsg != msg )
03032     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03033 }

void WorkDistrib::enqueueSelfA2 ( LocalWorkMsg msg  ) 

Definition at line 3034 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03034                                                  {
03035   msg->compute->doWork();  MACHINE_PROGRESS
03036   if ( msg->compute->localWorkMsg != msg )
03037     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03038 }

void WorkDistrib::enqueueSelfA3 ( LocalWorkMsg msg  ) 

Definition at line 3039 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03039                                                  {
03040   msg->compute->doWork();  MACHINE_PROGRESS
03041   if ( msg->compute->localWorkMsg != msg )
03042     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03043 }

void WorkDistrib::enqueueSelfB1 ( LocalWorkMsg msg  ) 

Definition at line 3045 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03045                                                  {
03046   msg->compute->doWork();  MACHINE_PROGRESS
03047   if ( msg->compute->localWorkMsg != msg )
03048     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03049 }

void WorkDistrib::enqueueSelfB2 ( LocalWorkMsg msg  ) 

Definition at line 3050 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03050                                                  {
03051   msg->compute->doWork();  MACHINE_PROGRESS
03052   if ( msg->compute->localWorkMsg != msg )
03053     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03054 }

void WorkDistrib::enqueueSelfB3 ( LocalWorkMsg msg  ) 

Definition at line 3055 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03055                                                  {
03056   msg->compute->doWork();  MACHINE_PROGRESS
03057   if ( msg->compute->localWorkMsg != msg )
03058     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03059 }

void WorkDistrib::enqueueThole ( LocalWorkMsg msg  ) 

Definition at line 2991 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02991                                                 {
02992   msg->compute->doWork();  MACHINE_PROGRESS
02993   if ( msg->compute->localWorkMsg != msg )
02994     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
02995 }

void WorkDistrib::enqueueWork ( LocalWorkMsg msg  ) 

Definition at line 2955 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

02955                                                {
02956   msg->compute->doWork();  MACHINE_PROGRESS
02957   if ( msg->compute->localWorkMsg != msg )
02958     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
02959 }

void WorkDistrib::enqueueWorkA1 ( LocalWorkMsg msg  ) 

Definition at line 3061 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03061                                                  {
03062   msg->compute->doWork();  MACHINE_PROGRESS
03063   if ( msg->compute->localWorkMsg != msg )
03064     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03065 }

void WorkDistrib::enqueueWorkA2 ( LocalWorkMsg msg  ) 

Definition at line 3066 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03066                                                  {
03067   msg->compute->doWork();  MACHINE_PROGRESS
03068   if ( msg->compute->localWorkMsg != msg )
03069     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03070 }

void WorkDistrib::enqueueWorkA3 ( LocalWorkMsg msg  ) 

Definition at line 3071 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03071                                                  {
03072   msg->compute->doWork();  MACHINE_PROGRESS
03073   if ( msg->compute->localWorkMsg != msg )
03074     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03075 }

void WorkDistrib::enqueueWorkB1 ( LocalWorkMsg msg  ) 

Definition at line 3077 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03077                                                  {
03078   msg->compute->doWork();  MACHINE_PROGRESS
03079   if ( msg->compute->localWorkMsg != msg )
03080     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03081 }

void WorkDistrib::enqueueWorkB2 ( LocalWorkMsg msg  ) 

Definition at line 3082 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03082                                                  {
03083   msg->compute->doWork();  MACHINE_PROGRESS
03084   if ( msg->compute->localWorkMsg != msg )
03085     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03086 }

void WorkDistrib::enqueueWorkB3 ( LocalWorkMsg msg  ) 

Definition at line 3087 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03087                                                  {
03088   msg->compute->doWork();  MACHINE_PROGRESS
03089   if ( msg->compute->localWorkMsg != msg )
03090     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03091 }

void WorkDistrib::enqueueWorkC ( LocalWorkMsg msg  ) 

Definition at line 3095 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

03095                                                 {
03096   msg->compute->doWork();  MACHINE_PROGRESS
03097   if ( msg->compute->localWorkMsg != msg )
03098     NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03099 }

void WorkDistrib::finishCUDA ( LocalWorkMsg msg  ) 

Definition at line 3118 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03118                                               {
03119   msg->compute->doWork();  MACHINE_PROGRESS
03120   // ComputeNonbondedCUDA *c = msg->compute;
03121   // if ( c->localWorkMsg != msg && c->localWorkMsg2 != msg )
03122   //   NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
03123 }

void WorkDistrib::finishCUDAP2 ( LocalWorkMsg msg  ) 

Definition at line 3124 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03124                                                 {
03125   msg->compute->doWork();  MACHINE_PROGRESS
03126 }

void WorkDistrib::finishCUDAP3 ( LocalWorkMsg msg  ) 

Definition at line 3127 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03127                                                 {
03128   msg->compute->doWork();  MACHINE_PROGRESS
03129 }

void WorkDistrib::finishCUDAPatch ( FinishWorkMsg msg  ) 

Definition at line 3114 of file WorkDistrib.C.

References FinishWorkMsg::compute, FinishWorkMsg::data, and Compute::finishPatch().

03114                                                     {
03115   msg->compute->finishPatch(msg->data);
03116 }

void WorkDistrib::finishMIC ( LocalWorkMsg msg  ) 

Definition at line 3134 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

03134                                              {
03135   msg->compute->doWork();  MACHINE_PROGRESS
03136 }

void WorkDistrib::initHostDeviceLDB (  ) 

Definition at line 3389 of file WorkDistrib.C.

03389                                     {
03390   #if defined(NAMD_MIC)
03391     mic_hostDeviceLDB();
03392   #endif
03393 }

void WorkDistrib::mapComputes ( void   ) 

Definition at line 2274 of file WorkDistrib.C.

References ComputeMap::allocateCids(), SimParameters::bondedCUDA, computeAnglesType, computeAnisoType, computeBondsType, computeConsForceType, computeConsTorqueType, computeCrosstermsType, computeCylindricalBCType, computeDihedralsType, computeEFieldType, computeEwaldType, computeExclsType, computeExtType, computeFmmType, computeFullDirectType, computeGBISserType, computeGlobalType, computeGridForceType, computeGromacsPairType, computeImpropersType, SimParameters::computeMapFilename, computeMsmMsaType, computeMsmSerialType, computeMsmType, computeNonbondedCUDA2Type, computeNonbondedCUDAType, computeNonbondedMICType, computePmeCUDAType, computePmeType, computeQMType, computeRestraintsType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfGromacsPairType, computeSelfImpropersType, computeSelfTholeType, computeSphericalBCType, computeStirType, computeTclBCType, computeTholeType, SimParameters::consForceOn, SimParameters::consTorqueOn, SimParameters::constraintsOn, SimParameters::cylindricalBCOn, DebugM, SimParameters::drudeOn, SimParameters::eFieldOn, SimParameters::extForcesOn, SimParameters::FMAOn, SimParameters::FMMOn, SimParameters::fullDirectOn, SimParameters::GBISserOn, SimParameters::globalForcesOn, SimParameters::goGroPair, SimParameters::LCPOOn, ComputeMap::loadComputeMap(), SimParameters::loadComputeMap, SimParameters::mgridforceOn, SimParameters::MSMOn, SimParameters::MsmSerialOn, NAMD_die(), Node::Object(), ComputeMap::Object(), PatchMap::Object(), SimParameters::pairInteractionOn, SimParameters::pairInteractionSelf, SimParameters::PMEOn, SimParameters::pressureProfileEwaldOn, SimParameters::qmForcesOn, ComputeMap::saveComputeMap(), Node::simParameters, SimParameters::sphericalBCOn, SimParameters::stirOn, SimParameters::storeComputeMap, SimParameters::tclBCOn, SimParameters::useCUDA2, SimParameters::useDPME, and SimParameters::usePMECUDA.

Referenced by Node::startup().

02275 {
02276   PatchMap *patchMap = PatchMap::Object();
02277   ComputeMap *computeMap = ComputeMap::Object();
02278   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
02279   Node *node = nd.ckLocalBranch();
02280 
02281   DebugM(3,"Mapping computes\n");
02282 
02283   computeMap->allocateCids();
02284 
02285   // Handle full electrostatics
02286   if ( node->simParameters->fullDirectOn )
02287     mapComputeHomePatches(computeFullDirectType);
02288   if ( node->simParameters->FMAOn )
02289 #ifdef DPMTA
02290     mapComputeHomePatches(computeDPMTAType);
02291 #else
02292     NAMD_die("This binary does not include DPMTA (FMA).");
02293 #endif
02294   if ( node->simParameters->PMEOn ) {
02295 #ifdef DPME
02296     if ( node->simParameters->useDPME )
02297       mapComputeHomePatches(computeDPMEType);
02298     else {
02299       mapComputeHomePatches(computePmeType);
02300       if ( node->simParameters->pressureProfileEwaldOn )
02301         mapComputeHomePatches(computeEwaldType);
02302     }
02303 #else
02304 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
02305     if (node->simParameters->usePMECUDA) {
02306       mapComputePatch(computePmeCUDAType);
02307     } else
02308 #endif
02309     {
02310       mapComputePatch(computePmeType);
02311     }
02312     if ( node->simParameters->pressureProfileEwaldOn )
02313       mapComputeHomePatches(computeEwaldType);
02314 #endif
02315   }
02316 
02317   if ( node->simParameters->globalForcesOn ) {
02318     DebugM(2,"adding ComputeGlobal\n");
02319     mapComputeHomePatches(computeGlobalType);
02320   }
02321 
02322   if ( node->simParameters->extForcesOn )
02323     mapComputeHomePatches(computeExtType);
02324 
02325   if ( node->simParameters->qmForcesOn )
02326     mapComputeHomePatches(computeQMType);
02327 
02328   if ( node->simParameters->GBISserOn )
02329     mapComputeHomePatches(computeGBISserType);
02330 
02331   if ( node->simParameters->MsmSerialOn )
02332     mapComputeHomePatches(computeMsmSerialType);
02333 #ifdef CHARM_HAS_MSA
02334   else if ( node->simParameters->MSMOn )
02335     mapComputeHomePatches(computeMsmMsaType);
02336 #else
02337   else if ( node->simParameters->MSMOn )
02338     mapComputeHomePatches(computeMsmType);
02339 #endif
02340 
02341   if ( node->simParameters->FMMOn )
02342     mapComputeHomePatches(computeFmmType);
02343 
02344 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
02345 #ifdef BONDED_CUDA
02346   if (node->simParameters->bondedCUDA) {
02347     mapComputeNode(computeBondedCUDAType);
02348   }
02349 #endif
02350 #endif
02351 
02352 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
02353   if (node->simParameters->useCUDA2) {
02354     mapComputeNode(computeNonbondedCUDA2Type);
02355   } else {
02356     mapComputeNode(computeNonbondedCUDAType);
02357   }
02358   mapComputeHomeTuples(computeExclsType);
02359   mapComputePatch(computeSelfExclsType);
02360 #endif
02361 
02362 #ifdef NAMD_MIC
02363   mapComputeNode(computeNonbondedMICType);
02364 #endif
02365 
02366   mapComputeNonbonded();
02367 
02368   if ( node->simParameters->LCPOOn ) {
02369     mapComputeLCPO();
02370   }
02371 
02372   // If we're doing true pair interactions, no need for bonded terms.
02373   // But if we're doing within-group interactions, we do need them.
02374   if ( !node->simParameters->pairInteractionOn || 
02375       node->simParameters->pairInteractionSelf) { 
02376     mapComputeHomeTuples(computeBondsType);
02377     mapComputeHomeTuples(computeAnglesType);
02378     mapComputeHomeTuples(computeDihedralsType);
02379     mapComputeHomeTuples(computeImpropersType);
02380     mapComputeHomeTuples(computeCrosstermsType);
02381     mapComputePatch(computeSelfBondsType);
02382     mapComputePatch(computeSelfAnglesType);
02383     mapComputePatch(computeSelfDihedralsType);
02384     mapComputePatch(computeSelfImpropersType);
02385     mapComputePatch(computeSelfCrosstermsType);
02386   }
02387 
02388   if ( node->simParameters->goGroPair ) {
02389       // JLai
02390       mapComputeHomeTuples(computeGromacsPairType);
02391       mapComputePatch(computeSelfGromacsPairType);
02392     // End of JLai
02393   }
02394 
02395   if ( node->simParameters->drudeOn ) {
02396     mapComputeHomeTuples(computeTholeType);
02397     mapComputePatch(computeSelfTholeType);
02398     mapComputeHomeTuples(computeAnisoType);
02399     mapComputePatch(computeSelfAnisoType);
02400   }
02401 
02402   if ( node->simParameters->eFieldOn )
02403     mapComputePatch(computeEFieldType);
02404   /* BEGIN gf */
02405   if ( node->simParameters->mgridforceOn )
02406     mapComputePatch(computeGridForceType);
02407   /* END gf */
02408   if ( node->simParameters->stirOn )
02409     mapComputePatch(computeStirType);
02410   if ( node->simParameters->sphericalBCOn )
02411     mapComputePatch(computeSphericalBCType);
02412   if ( node->simParameters->cylindricalBCOn )
02413     mapComputePatch(computeCylindricalBCType);
02414   if ( node->simParameters->tclBCOn ) {
02415     mapComputeHomePatches(computeTclBCType);
02416   }
02417   if ( node->simParameters->constraintsOn )
02418     mapComputePatch(computeRestraintsType);
02419   if ( node->simParameters->consForceOn )
02420     mapComputePatch(computeConsForceType);
02421   if ( node->simParameters->consTorqueOn )
02422     mapComputePatch(computeConsTorqueType);
02423 
02424     // store the latest compute map
02425   SimParameters *simParams = Node::Object()->simParameters;
02426   if (simParams->storeComputeMap) {
02427     computeMap->saveComputeMap(simParams->computeMapFilename);
02428   }
02429     // override mapping decision
02430   if (simParams->loadComputeMap) {
02431     computeMap->loadComputeMap(simParams->computeMapFilename);
02432     CkPrintf("ComputeMap has been loaded from %s.\n", simParams->computeMapFilename);
02433   }
02434 }

void WorkDistrib::messageEnqueueWork ( Compute compute  )  [static]

Definition at line 2732 of file WorkDistrib.C.

References Compute::cid, LocalWorkMsg::compute, computeAnglesType, computeAnisoType, computeBondsType, computeCrosstermsType, computeDihedralsType, computeExclsType, computeGromacsPairType, computeImpropersType, computeNonbondedCUDA2Type, computeNonbondedCUDAType, computeNonbondedMICType, computeNonbondedPairType, computeNonbondedSelfType, computePmeCUDAType, computePmeType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfGromacsPairType, computeSelfImpropersType, computeSelfTholeType, computeTholeType, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), SET_PRIORITY, and Compute::type().

Referenced by cuda_check_pme_forces(), Compute::enqueueWork(), ComputePmeMgr::submitReductions(), and ComputePmeMgr::ungridCalc().

02732                                                      {
02733   LocalWorkMsg *msg = compute->localWorkMsg;
02734   int seq = compute->sequence();
02735   int gbisPhase = compute->getGBISPhase();
02736 
02737   if ( seq < 0 ) {
02738     NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageEnqueueWork");
02739   } else {
02740     SET_PRIORITY(msg,seq,compute->priority());
02741   }
02742 
02743   msg->compute = compute; // pointer is valid since send is to local Pe
02744   int type = compute->type();
02745   int cid = compute->cid;
02746 
02747   CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
02748   switch ( type ) {
02749   case computeExclsType:
02750   case computeSelfExclsType:
02751     wdProxy[CkMyPe()].enqueueExcls(msg);
02752     break;
02753   case computeBondsType:
02754   case computeSelfBondsType:
02755     wdProxy[CkMyPe()].enqueueBonds(msg);
02756     break;
02757   case computeAnglesType:
02758   case computeSelfAnglesType:
02759     wdProxy[CkMyPe()].enqueueAngles(msg);
02760     break;
02761   case computeDihedralsType:
02762   case computeSelfDihedralsType:
02763     wdProxy[CkMyPe()].enqueueDihedrals(msg);
02764     break;
02765   case computeImpropersType:
02766   case computeSelfImpropersType:
02767     wdProxy[CkMyPe()].enqueueImpropers(msg);
02768     break;
02769   case computeTholeType:
02770   case computeSelfTholeType:
02771     wdProxy[CkMyPe()].enqueueThole(msg);
02772     break;
02773   case computeAnisoType:
02774   case computeSelfAnisoType:
02775     wdProxy[CkMyPe()].enqueueAniso(msg);
02776     break;
02777   case computeCrosstermsType:
02778   case computeSelfCrosstermsType:
02779     wdProxy[CkMyPe()].enqueueCrossterms(msg);
02780     break;
02781   // JLai
02782   case computeGromacsPairType:
02783   case computeSelfGromacsPairType:
02784     wdProxy[CkMyPe()].enqueueGromacsPair(msg);
02785     break;    
02786   // End of JLai
02787   case computeLCPOType:
02788     wdProxy[CkMyPe()].enqueueLCPO(msg);
02789     break;
02790   case computeNonbondedSelfType:
02791     switch ( seq % 2 ) {
02792     case 0:
02793       //wdProxy[CkMyPe()].enqueueSelfA(msg);
02794       switch ( gbisPhase ) {
02795          case 1:
02796            wdProxy[CkMyPe()].enqueueSelfA1(msg);
02797            break;
02798          case 2:
02799            wdProxy[CkMyPe()].enqueueSelfA2(msg);
02800            break;
02801          case 3:
02802            wdProxy[CkMyPe()].enqueueSelfA3(msg);
02803            break;
02804       }
02805       break;
02806     case 1:
02807       //wdProxy[CkMyPe()].enqueueSelfB(msg);
02808       switch ( gbisPhase ) {
02809          case 1:
02810            wdProxy[CkMyPe()].enqueueSelfB1(msg);
02811            break;
02812          case 2:
02813            wdProxy[CkMyPe()].enqueueSelfB2(msg);
02814            break;
02815          case 3:
02816            wdProxy[CkMyPe()].enqueueSelfB3(msg);
02817            break;
02818       }
02819       break;
02820     default:
02821       NAMD_bug("WorkDistrib::messageEnqueueSelf case statement error!");
02822     }
02823     break;
02824   case computeNonbondedPairType:
02825     switch ( seq % 2 ) {
02826     case 0:
02827       //wdProxy[CkMyPe()].enqueueWorkA(msg);
02828       switch ( gbisPhase ) {
02829          case 1:
02830            wdProxy[CkMyPe()].enqueueWorkA1(msg);
02831            break;
02832          case 2:
02833            wdProxy[CkMyPe()].enqueueWorkA2(msg);
02834            break;
02835          case 3:
02836            wdProxy[CkMyPe()].enqueueWorkA3(msg);
02837            break;
02838       }
02839       break;
02840     case 1:
02841       //wdProxy[CkMyPe()].enqueueWorkB(msg);
02842       switch ( gbisPhase ) {
02843          case 1:
02844            wdProxy[CkMyPe()].enqueueWorkB1(msg);
02845            break;
02846          case 2:
02847            wdProxy[CkMyPe()].enqueueWorkB2(msg);
02848            break;
02849          case 3:
02850            wdProxy[CkMyPe()].enqueueWorkB3(msg);
02851            break;
02852       }
02853       break;
02854     case 2:
02855       wdProxy[CkMyPe()].enqueueWorkC(msg);
02856       break;
02857     default:
02858       NAMD_bug("WorkDistrib::messageEnqueueWork case statement error!");
02859     }
02860     break;
02861   case computeNonbondedCUDAType:
02862 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
02863   case computeNonbondedCUDA2Type:
02864 //     CkPrintf("WorkDistrib[%d]::CUDA seq=%d phase=%d\n", CkMyPe(), seq, gbisPhase);
02865     //wdProxy[CkMyPe()].enqueueCUDA(msg);
02866     switch ( gbisPhase ) {
02867        case 1:
02868          wdProxy[CkMyPe()].enqueueCUDA(msg);
02869          break;
02870        case 2:
02871          wdProxy[CkMyPe()].enqueueCUDAP2(msg);
02872          break;
02873        case 3:
02874          wdProxy[CkMyPe()].enqueueCUDAP3(msg);
02875          break;
02876     }
02877 #else
02878     msg->compute->doWork();  MACHINE_PROGRESS
02879 #endif
02880     break;
02881   case computeNonbondedMICType:
02882 #ifdef NAMD_MIC
02883     wdProxy[CkMyPe()].enqueueMIC(msg);
02884 #endif
02885     break;
02886   case computePmeType:
02887     // CkPrintf("PME %d %d %x\n", CkMyPe(), seq, compute->priority());
02888     wdProxy[CkMyPe()].enqueuePme(msg);
02889     break;
02890 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
02891   case computePmeCUDAType:
02892     wdProxy[CkMyPe()].enqueuePme(msg);
02893     break;
02894 #endif
02895   default:
02896     wdProxy[CkMyPe()].enqueueWork(msg);
02897   }
02898 }

void WorkDistrib::messageFinishCUDA ( Compute compute  )  [static]

Definition at line 2901 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), and SET_PRIORITY.

Referenced by cuda_check_local_progress(), and cuda_check_remote_progress().

02901                                                     {
02902   LocalWorkMsg *msg = compute->localWorkMsg;
02903   int seq = compute->sequence();
02904   int gbisPhase = compute->getGBISPhase();
02905 
02906   if ( seq < 0 ) {
02907     NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageEnqueueWork");
02908   } else {
02909     SET_PRIORITY(msg,seq,compute->priority());
02910   }
02911 
02912   msg->compute = compute; // pointer is valid since send is to local Pe
02913   CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
02914 
02915 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
02916     //wdProxy[CkMyPe()].finishCUDA(msg);
02917     switch ( gbisPhase ) {
02918        case 1:
02919          wdProxy[CkMyPe()].finishCUDA(msg);
02920          break;
02921        case 2:
02922          wdProxy[CkMyPe()].finishCUDAP2(msg);
02923          break;
02924        case 3:
02925          wdProxy[CkMyPe()].finishCUDAP3(msg);
02926          break;
02927     }
02928 #else
02929     msg->compute->doWork();  MACHINE_PROGRESS
02930 #endif
02931 }

void WorkDistrib::messageFinishMIC ( Compute compute  )  [static]

Definition at line 2934 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), and SET_PRIORITY.

02934                                                    {
02935   LocalWorkMsg *msg = compute->localWorkMsg;
02936   int seq = compute->sequence();
02937   int gbisPhase = compute->getGBISPhase();
02938 
02939   if ( seq < 0 ) {
02940     NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageFinishMIC");
02941   } else {
02942     SET_PRIORITY(msg,seq,compute->priority());
02943   }
02944 
02945   msg->compute = compute; // pointer is valid since send is to local Pe
02946   CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
02947 
02948 #ifdef NAMD_MIC
02949     wdProxy[CkMyPe()].finishMIC(msg);
02950 #else
02951     msg->compute->doWork();  MACHINE_PROGRESS
02952 #endif
02953 }

void WorkDistrib::patchMapInit ( void   ) 

Definition at line 1109 of file WorkDistrib.C.

References Lattice::a_p(), Lattice::a_r(), Lattice::b_p(), Lattice::b_r(), Lattice::c_p(), Lattice::c_r(), DebugM, endi(), PDB::find_extremes(), SimParameters::FMAOn, SimParameters::FMMOn, PDB::get_extremes(), iINFO(), iout, isOutputProcessor(), iWARN(), SimParameters::lattice, SimParameters::LCPOOn, PatchMap::makePatches(), SimParameters::maxPatches, SimParameters::minAtomsPerPatch, Node::molecule, SimParameters::MSMOn, NAMD_die(), SimParameters::noPatchesOnOne, SimParameters::noPatchesOnOutputPEs, SimParameters::noPatchesOnZero, PDB::num_atoms(), Molecule::numAtoms, SimParameters::numoutputprocs, numPatches, Node::Object(), PatchMap::Object(), Lattice::origin(), SimParameters::patchDimension, Node::pdb, SimParameters::replicaUniformPatchGrids, Node::simParameters, SimParameters::simulatedPEs, SimParameters::simulateInitialMapping, PatchMap::sizeGrid(), SimParameters::staticAtomAssignment, SimParameters::twoAwayX, SimParameters::twoAwayY, SimParameters::twoAwayZ, Vector::x, Vector::y, and Vector::z.

Referenced by Node::startup().

01110 {
01111   PatchMap *patchMap = PatchMap::Object();
01112   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01113   Node *node = nd.ckLocalBranch();
01114   SimParameters *params = node->simParameters;
01115   Lattice lattice = params->lattice;
01116 
01117   BigReal patchSize = params->patchDimension;
01118 
01119 #ifndef MEM_OPT_VERSION
01120   const int totalAtoms = node->pdb->num_atoms();
01121 #else
01122   const int totalAtoms = node->molecule->numAtoms;
01123 #endif
01124 
01125   ScaledPosition xmin, xmax;
01126 
01127   double maxNumPatches = 1.e9;  // need to adjust fractional values
01128   if ( params->minAtomsPerPatch > 0 )
01129     maxNumPatches = totalAtoms / params->minAtomsPerPatch;
01130 
01131   DebugM(3,"Mapping patches\n");
01132   if ( lattice.a_p() && lattice.b_p() && lattice.c_p() ) {
01133     xmin = 0.;  xmax = 0.;
01134   }
01135   else if ( params->FMAOn || params->MSMOn || params->FMMOn ) {
01136   // Need to use full box for FMA to match NAMD 1.X results.
01137 #if 0
01138     node->pdb->find_extremes(&(xmin.x),&(xmax.x),lattice.a_r());
01139     node->pdb->find_extremes(&(xmin.y),&(xmax.y),lattice.b_r());
01140     node->pdb->find_extremes(&(xmin.z),&(xmax.z),lattice.c_r());
01141 #endif
01142     node->pdb->find_extremes(lattice);
01143     node->pdb->get_extremes(xmin, xmax);
01144 #if 0
01145     printf("+++ center=%.4f %.4f %.4f\n",
01146         lattice.origin().x, lattice.origin().y, lattice.origin().z);
01147     printf("+++ xmin=%.4f  xmax=%.4f\n", xmin.x, xmax.x);
01148     printf("+++ ymin=%.4f  ymax=%.4f\n", xmin.y, xmax.y);
01149     printf("+++ zmin=%.4f  zmax=%.4f\n", xmin.z, xmax.z);
01150 #endif
01151   // Otherwise, this allows a small number of stray atoms.
01152   }
01153   else {
01154 #if 0
01155     node->pdb->find_extremes(&(xmin.x),&(xmax.x),lattice.a_r(),0.9);
01156     node->pdb->find_extremes(&(xmin.y),&(xmax.y),lattice.b_r(),0.9);
01157     node->pdb->find_extremes(&(xmin.z),&(xmax.z),lattice.c_r(),0.9);
01158 #endif
01159     node->pdb->find_extremes(lattice, 1.0);
01160     node->pdb->get_extremes(xmin, xmax);
01161     iout << iINFO << "ORIGINAL ATOMS MINMAX IS " << xmin << "  " << xmax << "\n" << endi;
01162     double frac = ( (double)totalAtoms - 10000. ) / (double)totalAtoms;
01163     if ( frac < 0.9 ) { frac = 0.9; }
01164     node->pdb->find_extremes(lattice, frac);
01165     node->pdb->get_extremes(xmin, xmax);
01166     iout << iINFO << "ADJUSTED ATOMS MINMAX IS " << xmin << "  " << xmax << "\n" << endi;
01167   }
01168 
01169 #if 0
01170   BigReal origin_shift;
01171   origin_shift = lattice.a_r() * lattice.origin();
01172   xmin.x -= origin_shift;
01173   xmax.x -= origin_shift;
01174   origin_shift = lattice.b_r() * lattice.origin();
01175   xmin.y -= origin_shift;
01176   xmax.y -= origin_shift;
01177   origin_shift = lattice.c_r() * lattice.origin();
01178   xmin.z -= origin_shift;
01179   xmax.z -= origin_shift;
01180 #endif
01181 
01182   // SimParameters default is -1 for unset
01183   int twoAwayX = params->twoAwayX;
01184   int twoAwayY = params->twoAwayY;
01185   int twoAwayZ = params->twoAwayZ;
01186 
01187   // SASA implementation is not compatible with twoAway patches
01188   if (params->LCPOOn && patchSize < 32.4) {
01189     if ( twoAwayX > 0 || twoAwayY > 0 || twoAwayZ > 0 ) {
01190       iout << iWARN << "Ignoring twoAway[XYZ] due to LCPO SASA implementation.\n" << endi;
01191     }
01192     twoAwayX = twoAwayY = twoAwayZ = 0;
01193   }
01194 
01195   // if you think you know what you're doing go right ahead
01196   if ( twoAwayX > 0 ) maxNumPatches = 1.e9;
01197   if ( twoAwayY > 0 ) maxNumPatches = 1.e9;
01198   if ( twoAwayZ > 0 ) maxNumPatches = 1.e9;
01199   if ( params->maxPatches > 0 ) {
01200       maxNumPatches = params->maxPatches;
01201       iout << iINFO << "LIMITING NUMBER OF PATCHES TO " <<
01202                                 maxNumPatches << "\n" << endi;
01203   }
01204 
01205   int numpes = CkNumPes();
01206   SimParameters *simparam = Node::Object()->simParameters;
01207   if(simparam->simulateInitialMapping) {
01208     numpes = simparam->simulatedPEs;
01209     delete [] patchMap->nPatchesOnNode;
01210     patchMap->nPatchesOnNode = new int[numpes];
01211     memset(patchMap->nPatchesOnNode, 0, numpes*sizeof(int));    
01212   }
01213 
01214 #if defined(NAMD_CUDA) || defined(NAMD_HIP) || defined(NAMD_MIC)
01215   // for CUDA be sure there are more patches than pes
01216 
01217   int numPatches = patchMap->sizeGrid(
01218         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
01219         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
01220   if ( numPatches < numpes && twoAwayX < 0 ) {
01221     twoAwayX = 1;
01222     numPatches = patchMap->sizeGrid(
01223         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
01224         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
01225   }
01226   if ( numPatches < numpes && twoAwayY < 0 ) {
01227     twoAwayY = 1;
01228     numPatches = patchMap->sizeGrid(
01229         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
01230         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
01231   }
01232   if ( numPatches < numpes && twoAwayZ < 0 ) {
01233     twoAwayZ = 1;
01234     numPatches = patchMap->sizeGrid(
01235         xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
01236         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
01237   }
01238   if ( numPatches < numpes ) {
01239     #if defined(NAMD_MIC)
01240     NAMD_die("MIC-enabled NAMD requires at least one patch per thread.");
01241     #endif
01242   }
01243   if ( numPatches % numpes && numPatches <= 1.4 * numpes ) {
01244     int exactFit = numPatches - numPatches % numpes;
01245     int newNumPatches = patchMap->sizeGrid(
01246         xmin,xmax,lattice,patchSize,exactFit,params->staticAtomAssignment,
01247         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
01248     if ( newNumPatches == exactFit ) {
01249       iout << iINFO << "REDUCING NUMBER OF PATCHES TO IMPROVE LOAD BALANCE\n" << endi;
01250       maxNumPatches = exactFit;
01251     }
01252   }
01253 
01254   patchMap->makePatches(xmin,xmax,lattice,patchSize,maxNumPatches,
01255         params->staticAtomAssignment, params->replicaUniformPatchGrids, params->LCPOOn,
01256         twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
01257 
01258 #else
01259 
01260   int availPes = numpes;
01261   if ( params->noPatchesOnZero && numpes > 1 ) {
01262       availPes -= 1;
01263       if(params->noPatchesOnOne && numpes > 2)
01264         availPes -= 1;
01265   }
01266 #ifdef MEM_OPT_VERSION
01267   if(params->noPatchesOnOutputPEs && numpes - params->numoutputprocs >2)
01268     {
01269       availPes -= params->numoutputprocs;
01270       if ( params->noPatchesOnZero && numpes > 1 && isOutputProcessor(0)){
01271         availPes++;
01272       }
01273       if ( params->noPatchesOnOne && numpes > 2 && isOutputProcessor(1)){
01274         availPes++;
01275       }
01276     }
01277 #endif
01278 
01279   int numPatches = patchMap->sizeGrid(
01280         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
01281         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
01282   if ( ( numPatches > (0.3*availPes) || numPatches > maxNumPatches
01283        ) && twoAwayZ < 0 ) {
01284     twoAwayZ = 0;
01285     numPatches = patchMap->sizeGrid(
01286         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
01287         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
01288   }
01289   if ( ( numPatches > (0.6*availPes) || numPatches > maxNumPatches
01290        ) && twoAwayY < 0 ) {
01291     twoAwayY = 0;
01292     numPatches = patchMap->sizeGrid(
01293         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
01294         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
01295   }
01296   if ( ( numPatches > availPes || numPatches > maxNumPatches
01297        ) && twoAwayX < 0 ) {
01298     twoAwayX = 0;
01299     numPatches = patchMap->sizeGrid(
01300         xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
01301         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
01302   }
01303   if ( numPatches > availPes && numPatches <= (1.4*availPes) && availPes <= maxNumPatches ) {
01304     int newNumPatches = patchMap->sizeGrid(
01305         xmin,xmax,lattice,patchSize,availPes,params->staticAtomAssignment,
01306         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
01307     if ( newNumPatches <= availPes && numPatches <= (1.4*newNumPatches) ) {
01308       iout << iINFO << "REDUCING NUMBER OF PATCHES TO IMPROVE LOAD BALANCE\n" << endi;
01309       maxNumPatches = availPes;
01310     }
01311   }
01312 
01313   patchMap->makePatches(xmin,xmax,lattice,patchSize,maxNumPatches,
01314         params->staticAtomAssignment, params->replicaUniformPatchGrids, params->LCPOOn,
01315         twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
01316 
01317 #endif
01318 
01319 }

void WorkDistrib::peOrderingReady (  )  [static]

Definition at line 170 of file WorkDistrib.C.

References cuda_initialize(), and mic_initialize().

Referenced by buildNodeAwarePeOrdering().

00170                                   {
00171   //CkPrintf("WorkDistrib::peOrderingReady on %d\n", CkMyPe());
00172 #if defined(NAMD_CUDA) || defined(NAMD_HIP)
00173   cuda_initialize();
00174 #endif
00175 #ifdef NAMD_MIC
00176   mic_initialize();
00177 #endif
00178 }

void WorkDistrib::recvComputeMapChanges ( ComputeMapChangeMsg msg  ) 

Definition at line 374 of file WorkDistrib.C.

References ALLBUTME, BUFSIZE, COMPUTEMAPTAG, MOStream::end(), MIStream::get(), NAMD_bug(), ComputeMap::newNode(), ComputeMap::newNumPartitions(), ComputeMap::numComputes(), ComputeMap::Object(), MOStream::put(), ComputeMap::setNewNode(), and ComputeMap::setNewNumPartitions().

00374                                                                 {
00375 
00376   delete msg;
00377 
00378   ComputeMap *computeMap = ComputeMap::Object();
00379 
00380   int i;
00381   int nc = computeMap->numComputes();
00382   
00383   if ( ! CkMyPe() ) { // send
00384     // CkPrintf("At %f on %d WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
00385     MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, COMPUTEMAPTAG, BUFSIZE);
00386     msg->put(nc);
00387     for (i=0; i<nc; i++) {
00388       int data = computeMap->newNode(i);
00389       msg->put(data);
00390     }
00391     msg->put(nc);
00392     for (i=0; i<nc; i++) {
00393       char data = computeMap->newNumPartitions(i);
00394       msg->put(data);
00395     }
00396     msg->put(nc);
00397     msg->end();
00398     delete msg;
00399     // CkPrintf("At %f on %d done WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
00400   } else if ( ! CkMyRank() ) { // receive
00401     // if ( CkMyNode() == 1 ) CkPrintf("At %f on %d WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
00402     MIStream *msg = CkpvAccess(comm)->newInputStream(0, COMPUTEMAPTAG);
00403     msg->get(i);
00404     if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 1 failed\n");
00405     for (i=0; i<nc; i++) {
00406       int data;
00407       msg->get(data);
00408       computeMap->setNewNode(i,data);
00409     }
00410     msg->get(i);
00411     if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 2 failed\n");
00412     for (i=0; i<nc; i++) {
00413       char data;
00414       msg->get(data);
00415       computeMap->setNewNumPartitions(i,data);
00416     }
00417     msg->get(i);
00418     if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 3 failed\n");
00419     delete msg;
00420     // if ( CkMyNode() == 1 ) CkPrintf("At %f on %d done WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
00421   }
00422 
00423   CkCallback cb(CkIndex_WorkDistrib::doneSaveComputeMap(NULL), 0, thisgroup);
00424   contribute(0, NULL, CkReduction::random, cb);
00425 }

void WorkDistrib::reinitAtoms ( const char *  basename = 0  ) 

Definition at line 956 of file WorkDistrib.C.

References atoms, createAtomLists(), PatchMap::numPatches(), numPatches, PatchMap::Object(), and PatchMgr::sendAtoms().

00956                                                   {
00957 
00958   PatchMap *patchMap = PatchMap::Object();
00959   CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
00960   PatchMgr *patchMgr = pm.ckLocalBranch();
00961 
00962   int numPatches = patchMap->numPatches();
00963 
00964   FullAtomList *atoms = createAtomLists(basename);
00965 
00966   for(int i=0; i < numPatches; i++) {
00967     patchMgr->sendAtoms(i,atoms[i]);
00968   }
00969 
00970   delete [] atoms;
00971 
00972 }

void WorkDistrib::saveComputeMapChanges ( int  ep,
CkGroupID  chareID 
)

Definition at line 356 of file WorkDistrib.C.

Referenced by ComputeMgr::updateComputes2().

00357 {
00358   saveComputeMapReturnEP = ep;
00359   saveComputeMapReturnChareID = chareID;
00360 
00361   ComputeMapChangeMsg *mapMsg = new (0, 0, 0) ComputeMapChangeMsg;
00362   CProxy_WorkDistrib(thisgroup).recvComputeMapChanges(mapMsg);
00363 
00364 /*
00365     // store the latest compute map
00366   SimParameters *simParams = Node::Object()->simParameters;
00367   if (simParams->storeComputeMap) {
00368     computeMap->saveComputeMap(simParams->computeMapFilename);
00369     CkPrintf("ComputeMap has been stored in %s.\n", simParams->computeMapFilename);
00370   }
00371 */
00372 }

void WorkDistrib::savePatchMap ( PatchMapMsg msg  ) 

Definition at line 1018 of file WorkDistrib.C.

References SimParameters::isRecvSpanningTreeUnset(), SimParameters::isSendSpanningTreeUnset(), numPatches, ProxyMgr::Object(), PatchMap::Object(), PatchMapMsg::patchMapData, ProxyMgr::setRecvSpanning(), ProxyMgr::setSendSpanning(), Node::simParameters, split(), and PatchMap::unpack().

01019 {
01020   // Use a resend to forward messages before processing.  Otherwise the
01021   // map distribution is slow on many CPUs.  We need to use a tree
01022   // rather than a broadcast because some implementations of broadcast
01023   // generate a copy of the message on the sender for each recipient.
01024   // This is because MPI doesn't allow re-use of an outstanding buffer.
01025 
01026   if ( CkMyRank() ) patchMapArrived = true;
01027 
01028   if ( patchMapArrived && CkMyPe() ) {
01029     PatchMap::Object()->unpack(msg->patchMapData);
01030 
01031     //Automatically enable spanning tree
01032     CProxy_Node nd(CkpvAccess(BOCclass_group).node);
01033     Node *node = nd.ckLocalBranch();
01034     SimParameters *params = node->simParameters;
01035     if( ( PatchMap::Object()->numPatches() <= CkNumPes()/4
01036 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
01037         || CkNumPes() > CkNumNodes()
01038         ) && ( CkNumNodes() > 1
01039 #endif
01040       ) && params->isSendSpanningTreeUnset() )
01041       ProxyMgr::Object()->setSendSpanning();
01042 
01043 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
01044     if ( CkNumPes() > CkNumNodes() && CkNumNodes() > 1
01045           && params->isRecvSpanningTreeUnset() )
01046       ProxyMgr::Object()->setRecvSpanning();
01047 #endif
01048   }
01049 
01050   if ( patchMapArrived ) {
01051     if ( CkMyRank() + 1 < CkNodeSize(CkMyNode()) ) {
01052       ((CProxy_WorkDistrib(thisgroup))[CkMyPe()+1]).savePatchMap(msg);
01053     } else {
01054       delete msg;
01055     }
01056     return;
01057   }
01058 
01059   patchMapArrived = true;
01060 
01061   int self = CkMyNode();
01062   int range_begin = 0;
01063   int range_end = CkNumNodes();
01064   while ( self != range_begin ) {
01065     ++range_begin;
01066     int split = range_begin + ( range_end - range_begin ) / 2;
01067     if ( self < split ) { range_end = split; }
01068     else { range_begin = split; }
01069   }
01070   int send_near = self + 1;
01071   int send_far = send_near + ( range_end - send_near ) / 2;
01072 
01073   int pids[3];
01074   int npid = 0;
01075   if ( send_far < range_end ) pids[npid++] = CkNodeFirst(send_far);
01076   if ( send_near < send_far ) pids[npid++] = CkNodeFirst(send_near);
01077   pids[npid++] = CkMyPe();  // always send the message to ourselves
01078   CProxy_WorkDistrib(thisgroup).savePatchMap(msg,npid,pids);
01079 }

void WorkDistrib::send_contributeHostDeviceLDB ( int  peSetLen,
int *  peSet 
) [static]

Definition at line 3395 of file WorkDistrib.C.

03395                                                                         {
03396   #if defined(NAMD_MIC)
03397     CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
03398     wdProxy[0].contributeHostDeviceLDB(peSetLen, peSet);
03399   #endif
03400 }

void WorkDistrib::send_initHostDeviceLDB (  )  [static]

Definition at line 3382 of file WorkDistrib.C.

03382                                          {
03383   #if defined(NAMD_MIC)
03384     CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
03385     wdProxy.initHostDeviceLDB();
03386   #endif
03387 }

void WorkDistrib::send_setDeviceLDBParams ( int  dt,
int  hs,
int  sp1,
int  pp1,
int  pp2 
) [static]

Definition at line 3408 of file WorkDistrib.C.

03408                                                                                    {
03409   #if defined(NAMD_MIC)
03410     CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
03411     wdProxy.setDeviceLDBParams(dt, hs, sp1, pp1, pp2);
03412   #endif
03413 }

void WorkDistrib::sendComputeMap ( void   ) 

Definition at line 1082 of file WorkDistrib.C.

References ALLBUTME, BUFSIZE, COMPUTEMAPTAG, MOStream::end(), ComputeMap::initPtrs(), ComputeMap::Object(), ComputeMap::pack(), and ComputeMap::unpack().

Referenced by Node::startup().

01083 {
01084   if ( CkMyRank() ) return;
01085 
01086   if ( CkNumNodes() == 1 ) {
01087     computeMapArrived = true;
01088     ComputeMap::Object()->initPtrs();
01089     return;
01090   }
01091 
01092   if ( ! CkMyPe() ) { // send
01093     MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, COMPUTEMAPTAG, BUFSIZE);
01094     ComputeMap::Object()->pack(msg);
01095     msg->end();
01096     delete msg;
01097   } else if ( ! CkMyRank() ) { // receive
01098     MIStream *msg = CkpvAccess(comm)->newInputStream(0, COMPUTEMAPTAG);
01099     ComputeMap::Object()->unpack(msg);
01100     delete msg;
01101   }
01102 
01103   computeMapArrived = true;
01104   ComputeMap::Object()->initPtrs();
01105 }

void WorkDistrib::sendPatchMap ( void   ) 

Definition at line 982 of file WorkDistrib.C.

References SimParameters::isRecvSpanningTreeUnset(), SimParameters::isSendSpanningTreeUnset(), numPatches, ProxyMgr::Object(), PatchMap::Object(), PatchMap::pack(), PatchMap::packSize(), PatchMapMsg::patchMapData, ProxyMgr::setRecvSpanning(), ProxyMgr::setSendSpanning(), and Node::simParameters.

Referenced by Node::startup().

00983 {
00984   if ( CkNumPes() == 1 ) {
00985     patchMapArrived = true;
00986     return;
00987   }
00988 
00989   //Automatically enable spanning tree
00990   CProxy_Node nd(CkpvAccess(BOCclass_group).node);
00991   Node *node = nd.ckLocalBranch();
00992   SimParameters *params = node->simParameters;
00993   if( ( PatchMap::Object()->numPatches() <= CkNumPes()/4
00994 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
00995       || CkNumPes() > CkNumNodes()
00996       ) && ( CkNumNodes() > 1
00997 #endif
00998     ) && params->isSendSpanningTreeUnset() )
00999     ProxyMgr::Object()->setSendSpanning();
01000 
01001 #ifdef NODEAWARE_PROXY_SPANNINGTREE 
01002   if ( CkNumPes() > CkNumNodes() && CkNumNodes() > 1
01003         && params->isRecvSpanningTreeUnset() )
01004     ProxyMgr::Object()->setRecvSpanning();
01005 #endif
01006 
01007   int size = PatchMap::Object()->packSize();
01008 
01009   PatchMapMsg *mapMsg = new (size, 0) PatchMapMsg;
01010 
01011   PatchMap::Object()->pack(mapMsg->patchMapData, size);
01012 
01013   CProxy_WorkDistrib workProxy(thisgroup);
01014   workProxy[0].savePatchMap(mapMsg);
01015 }

void WorkDistrib::setDeviceLDBParams ( int  dt,
int  hs,
int  sp1,
int  pp1,
int  pp2 
)

Definition at line 3415 of file WorkDistrib.C.

03415                                                                               {
03416   #if defined(NAMD_MIC)
03417     mic_setDeviceLDBParams(dt, hs, sp1, pp1, pp2);
03418   #endif
03419 }

void WorkDistrib::setPatchMapArrived ( bool  s  )  [inline]

Definition at line 107 of file WorkDistrib.h.

Referenced by Node::startup().

00107 {patchMapArrived=s;}

void WorkDistrib::sortPmePes ( int *  pmepes,
int  xdim,
int  ydim 
) [static]

Definition at line 304 of file WorkDistrib.C.

References ResizeArray< Elem >::begin(), PatchMap::center(), j, PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), and recursive_bisect_coord().

Referenced by ComputePmeMgr::initialize().

00304                                                             {
00305   int numpes = CkNumPes();
00306   ResizeArray<int> count(numpes);
00307   ResizeArray<ScaledPosition> sumPos(numpes);
00308   ResizeArray<ScaledPosition> avgPos(numpes);
00309   for ( int i=0; i<numpes; ++i ) {
00310     count[i] = 0;
00311     sumPos[i] = 0;
00312     avgPos[i] = 0;
00313   }
00314   PatchMap *patchMap = PatchMap::Object();
00315   for ( int i=0, npatches=patchMap->numPatches(); i<npatches; ++i ) {
00316     int pe = patchMap->node(i);
00317     count[pe] += 1;
00318     sumPos[pe] += patchMap->center(i);
00319   }
00320   const int npmepes = xdim*ydim;
00321   ResizeArray<int> sortpes(npmepes);
00322   for ( int i=0; i<npmepes; ++i ) {
00323     int pe = sortpes[i] = pmepes[i];
00324     int cnt = count[pe];
00325     ScaledPosition sum = sumPos[pe];
00326     if ( cnt == 0 ) {
00327       // average over node
00328       int node = CkNodeOf(pe);
00329       int nsize = CkNodeSize(node);
00330       int pe2 = CkNodeFirst(node);
00331       for ( int j=0; j<nsize; ++j, ++pe2 )  {
00332         cnt += count[pe2];
00333         sum += sumPos[pe2];
00334       }
00335     }
00336     if ( cnt == 0 ) {
00337       // average over physical node
00338       int node = CmiPhysicalNodeID(pe);
00339       int nsize, *nlist;
00340       CmiGetPesOnPhysicalNode(node, &nlist, &nsize);
00341       for ( int j=0; j<nsize; ++j )  {
00342         int pe2 = nlist[j];
00343         cnt += count[pe2];
00344         sum += sumPos[pe2];
00345       }
00346     }
00347     if ( cnt ) {
00348       avgPos[pe] = sum / cnt;
00349     }
00350   }
00351   recursive_bisect_coord(0, xdim, 0, ydim, sortpes.begin(), avgPos.begin(), pmepes, ydim);
00352 }


Member Data Documentation

Definition at line 117 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering().

Definition at line 114 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering().


The documentation for this class was generated from the following files:

Generated on 16 Jun 2022 for NAMD by  doxygen 1.6.1