WorkDistrib Class Reference

#include <WorkDistrib.h>

Inheritance diagram for WorkDistrib:

Classes
struct	pe_sortop_compact
struct	pe_sortop_diffuse

Public Member Functions
	WorkDistrib ()
	~WorkDistrib (void)
void	enqueueWork (LocalWorkMsg *msg)
void	enqueueExcls (LocalWorkMsg *msg)
void	enqueueBonds (LocalWorkMsg *msg)
void	enqueueAngles (LocalWorkMsg *msg)
void	enqueueDihedrals (LocalWorkMsg *msg)
void	enqueueImpropers (LocalWorkMsg *msg)
void	enqueueThole (LocalWorkMsg *msg)
void	enqueueAniso (LocalWorkMsg *msg)
void	enqueueCrossterms (LocalWorkMsg *msg)
void	enqueueGromacsPair (LocalWorkMsg *msg)
void	enqueuePme (LocalWorkMsg *msg)
void	enqueueSelfA1 (LocalWorkMsg *msg)
void	enqueueSelfA2 (LocalWorkMsg *msg)
void	enqueueSelfA3 (LocalWorkMsg *msg)
void	enqueueSelfB1 (LocalWorkMsg *msg)
void	enqueueSelfB2 (LocalWorkMsg *msg)
void	enqueueSelfB3 (LocalWorkMsg *msg)
void	enqueueWorkA1 (LocalWorkMsg *msg)
void	enqueueWorkA2 (LocalWorkMsg *msg)
void	enqueueWorkA3 (LocalWorkMsg *msg)
void	enqueueWorkB1 (LocalWorkMsg *msg)
void	enqueueWorkB2 (LocalWorkMsg *msg)
void	enqueueWorkB3 (LocalWorkMsg *msg)
void	enqueueWorkC (LocalWorkMsg *msg)
void	enqueueCUDA (LocalWorkMsg *msg)
void	enqueueCUDAP2 (LocalWorkMsg *msg)
void	enqueueCUDAP3 (LocalWorkMsg *msg)
void	finishCUDAPatch (FinishWorkMsg *msg)
void	finishCUDA (LocalWorkMsg *msg)
void	finishCUDAP2 (LocalWorkMsg *msg)
void	finishCUDAP3 (LocalWorkMsg *msg)
void	enqueueMIC (LocalWorkMsg *msg)
void	finishMIC (LocalWorkMsg *msg)
void	enqueueLCPO (LocalWorkMsg *msg)
void	mapComputes (void)
void	sendPatchMap (void)
void	sendComputeMap (void)
void	saveComputeMapChanges (int, CkGroupID)
void	recvComputeMapChanges (ComputeMapChangeMsg *)
void	doneSaveComputeMap (CkReductionMsg *)
FullAtomList *	createAtomLists (const char *basename=0)
void	createHomePatches (void)
void	distributeHomePatches (void)
void	reinitAtoms (const char *basename=0)
void	patchMapInit (void)
void	assignNodeToPatch (void)
void	savePatchMap (PatchMapMsg *msg)
void	setPatchMapArrived (bool s)
void	initHostDeviceLDB ()
void	contributeHostDeviceLDB (int peSetLen, int *peSet)
void	setDeviceLDBParams (int dt, int hs, int sp1, int pp1, int pp2)

Static Public Member Functions
static void	messageEnqueueWork (Compute *)
static void	messageFinishCUDA (Compute *)
static void	messageFinishMIC (Compute *)
static void	sortPmePes (int *pmepes, int xdim, int ydim)
static void	peOrderingReady ()
static void	send_initHostDeviceLDB ()
static void	send_contributeHostDeviceLDB (int peSetLen, int *peSet)
static void	send_setDeviceLDBParams (int dt, int hs, int sp1, int pp1, int pp2)
static void	buildNodeAwarePeOrdering (void)

Static Public Attributes
static int	peOrderingInit
static int *	peDiffuseOrdering
static int *	peDiffuseOrderingIndex
static int *	peCompactOrdering
static int *	peCompactOrderingIndex

Detailed Description

Definition at line 42 of file WorkDistrib.h.

Constructor & Destructor Documentation

WorkDistrib::WorkDistrib

(

)

Definition at line 101 of file WorkDistrib.C.

References eventMachineProgress, and NAMD_bug().

{
  CkpvAccess(BOCclass_group).workDistrib = thisgroup;
  patchMapArrived = false;
  computeMapArrived = false;

#if CMK_SMP
#define MACHINE_PROGRESS
#else
#define MACHINE_PROGRESS { traceUserEvent(eventMachineProgress);  CmiMachineProgressImpl(); }
  if ( CkMyNodeSize() > 1 ) NAMD_bug("CkMyNodeSize() > 1 for non-smp build");
  eventMachineProgress = traceRegisterUserEvent("CmiMachineProgressImpl",233);
#endif
}

WorkDistrib::~WorkDistrib

(

void

)

Definition at line 117 of file WorkDistrib.C.

{ }

Member Function Documentation

void WorkDistrib::assignNodeToPatch

(

void

)

Definition at line 1319 of file WorkDistrib.C.

References endi(), Patch::getNumAtoms(), iINFO(), iout, Node::molecule, NAMD_die(), PatchMap::node(), Molecule::numAtoms, Node::numNodes(), PatchMap::numPatches(), PatchMap::Object(), Node::Object(), PatchMap::patch(), Node::simParameters, SimParameters::simulatedPEs, and SimParameters::simulateInitialMapping.

Referenced by Node::startup().

{
  PatchMap *patchMap = PatchMap::Object();
  int nNodes = Node::Object()->numNodes();
  SimParameters *simparam = Node::Object()->simParameters;
  if(simparam->simulateInitialMapping) {
          nNodes = simparam->simulatedPEs;
  }

#if (CMK_BLUEGENEP | CMK_BLUEGENEL) && USE_TOPOMAP 
  TopoManager tmgr;
  int numPes = tmgr.getDimNX() * tmgr.getDimNY() * tmgr.getDimNZ();
  if (numPes > patchMap->numPatches() && (assignPatchesTopoGridRecBisection() > 0)) {
    CkPrintf ("Blue Gene/L topology partitioner finished successfully \n");
  }
  else
#endif
  assignPatchesSpaceFillingCurve();       
  
  int *nAtoms = new int[nNodes];
  int numAtoms=0;
  int i;
  for(i=0; i < nNodes; i++)
    nAtoms[i] = 0;

  for(i=0; i < patchMap->numPatches(); i++)
  {
    //    iout << iINFO << "Patch " << i << " has " 
    //   << patchMap->patch(i)->getNumAtoms() << " atoms and "
    //   << patchMap->patch(i)->getNumAtoms() * 
    //            patchMap->patch(i)->getNumAtoms() 
    //   << " pairs.\n" << endi;         
#ifdef MEM_OPT_VERSION
      numAtoms += patchMap->numAtoms(i);
      nAtoms[patchMap->node(i)] += patchMap->numAtoms(i);         
#else
    if (patchMap->patch(i)) {
      numAtoms += patchMap->patch(i)->getNumAtoms();
      nAtoms[patchMap->node(i)] += patchMap->patch(i)->getNumAtoms();     
    }
#endif
  }

  if ( numAtoms != Node::Object()->molecule->numAtoms ) {
    for(i=0; i < nNodes; i++)
      iout << iINFO << nAtoms[i] << " atoms assigned to node " << i << "\n" << endi;
    iout << iINFO << "Assigned " << numAtoms << " atoms but expected " << Node::Object()->molecule->numAtoms << "\n" << endi;
    NAMD_die("Incorrect atom count in WorkDistrib::assignNodeToPatch\n");
  }

  delete [] nAtoms;
 
  //  PatchMap::Object()->printPatchMap();
}

void WorkDistrib::buildNodeAwarePeOrdering ( void  )

static

Definition at line 176 of file WorkDistrib.C.

References ResizeArray< T >::begin(), endi(), iout, iWARN(), peCompactOrdering, peCompactOrderingIndex, peDiffuseOrdering, peDiffuseOrderingIndex, peOrderingInit, peOrderingReady(), randtopo, Random::reorder(), and sort.

Referenced by build_ordering().

                                           {

 CmiMemLock();
 if ( ! peOrderingInit ) {
  //CkPrintf("WorkDistrib::buildNodeAwarePeOrdering on pe %d\n", CkMyPe());

  const int numPhys = CmiNumPhysicalNodes();
  const int numNode = CmiNumNodes();
  const int numPe = CmiNumPes();
  ResizeArray<int> numNodeInPhys(numPhys);
  ResizeArray<int> rankInPhysOfNode(numNode);

  peDiffuseOrdering = new int[numPe];
  peDiffuseOrderingIndex = new int[numPe];
  peCompactOrdering = new int[numPe];
  peCompactOrderingIndex = new int[numPe];

  int k = 0;
  for ( int ph=0; ph<numPhys; ++ph ) {
    int *pes, npes;
    CmiGetPesOnPhysicalNode(ph, &pes, &npes);
    for ( int i=0; i<npes; ++i, ++k ) {
      peCompactOrdering[k] = pes[i];
    }
    numNodeInPhys[ph] = 0;
    for ( int i=0, j=0; i<npes; i += CmiNodeSize(CmiNodeOf(pes[i])), ++j ) {
      rankInPhysOfNode[CmiNodeOf(pes[i])] = j;
      numNodeInPhys[ph] += 1;
    }
  }

  if ( randtopo && numPhys > 2 ) {
    if ( ! CkMyNode() ) {
      iout << iWARN << "RANDOMIZING PHYSICAL NODE ORDERING\n" << endi;
    }
    ResizeArray<int> randPhysOrder(numPhys);
    for ( int j=0; j<numPhys; ++j ) {
      randPhysOrder[j] = j;
    }
    Random(314159265).reorder(randPhysOrder.begin()+2, numPhys-2);
    for ( int j=0, k=0; j<numPhys; ++j ) {
      const int ph = randPhysOrder[j];
      int *pes, npes;
      CmiGetPesOnPhysicalNode(ph, &pes, &npes);
      for ( int i=0; i<npes; ++i, ++k ) {
        peCompactOrdering[k] = pes[i];
      }
    }
  }

  for ( int i=0; i<numPe; ++i ) {
    peDiffuseOrdering[i] = i;
  }
  std::sort(peDiffuseOrdering, peDiffuseOrdering+numPe,
    pe_sortop_bit_reversed(rankInPhysOfNode.begin()));

  for ( int i=0; i<numPe; ++i ) {
    peDiffuseOrderingIndex[peDiffuseOrdering[i]] = i;
    peCompactOrderingIndex[peCompactOrdering[i]] = i;
  }

  if ( 0 && CmiMyNode() == 0 ) for ( int i=0; i<numPe; ++i ) {
    CkPrintf("order %5d %5d %5d %5d %5d\n", i,
      peDiffuseOrdering[i],
      peDiffuseOrderingIndex[i],
      peCompactOrdering[i],
      peCompactOrderingIndex[i]);
  }

  peOrderingInit = 1;
 }
 CmiMemUnlock();
 peOrderingReady();

}

void WorkDistrib::contributeHostDeviceLDB	(	int	peSetLen,
		int *	peSet
	)

Definition at line 3397 of file WorkDistrib.C.

                                                                   {
  #if defined(NAMD_MIC)
    mic_contributeHostDeviceLDB(peSetLen, peSet);
  #endif
}

FullAtomList * WorkDistrib::createAtomLists

(

const char *

basename = 0

)

Definition at line 618 of file WorkDistrib.C.

References ResizeArray< T >::add(), SimParameters::alchOn, Lattice::apply_transform(), PatchMap::assignToPatch(), Molecule::atomcharge(), CompAtomExt::atomFixed, HydrogenGroupID::atomID, Molecule::atommass(), atoms, HydrogenGroupID::atomsInGroup, HydrogenGroupID::atomsInMigrationGroup, Molecule::atomvdwtype(), ResizeArray< T >::begin(), SimParameters::binaryOutput, CompAtom::charge, SimParameters::comMove, Node::configList, StringList::data, endi(), FALSE, ConfigList::find(), FullAtom::fixedPosition, PDB::get_all_positions(), Molecule::get_fep_type(), Molecule::getAtoms(), PatchMap::gridsize_a(), PatchMap::gridsize_b(), PatchMap::gridsize_c(), CompAtomExt::groupFixed, Molecule::hydrogenGroup, CompAtom::hydrogenGroupSize, CompAtomExt::id, PatchMap::index_a(), PatchMap::index_b(), PatchMap::index_c(), SimParameters::initialTemp, iout, Molecule::is_atom_fixed(), HydrogenGroupID::isGP, HydrogenGroupID::isMP, Molecule::langevin_param(), FullAtom::langevinParam, SimParameters::lattice, SimParameters::lesOn, FullAtom::mass, PatchMap::max_a(), PatchMap::max_b(), PatchMap::max_c(), FullAtom::migrationGroupSize, PatchMap::min_a(), PatchMap::min_b(), PatchMap::min_c(), Node::molecule, NAMD_bug(), NAMD_die(), Lattice::nearest(), CompAtom::nonbondedGroupSize, PDB::num_atoms(), PatchMap::numPatches(), numPatches, PatchMap::Object(), order, SimParameters::outputPatchDetails, SimParameters::pairInteractionOn, CompAtom::partition, Node::pdb, CompAtom::position, SimParameters::pressureProfileAtomTypes, read_binary_file(), FullAtom::recipMass, Molecule::rigid_bond_length(), RIGID_NONE, FullAtom::rigidBondLength, SimParameters::rigidBonds, Node::simParameters, ResizeArray< T >::size(), sortAtomsForPatches(), SimParameters::staticAtomAssignment, atom_constants::status, FullAtom::status, FullAtom::transform, TRUE, CompAtom::vdwType, and FullAtom::velocity.

Referenced by createHomePatches(), and reinitAtoms().

{
  int i;
  StringList *current;  //  Pointer used to retrieve configuration items
  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
  Node *node = nd.ckLocalBranch();
  PatchMap *patchMap = PatchMap::Object();
  CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
  PatchMgr *patchMgr = pm.ckLocalBranch();
  SimParameters *params = node->simParameters;
  Molecule *molecule = node->molecule;
  PDB *pdb = node->pdb;

  int numPatches = patchMap->numPatches();
  int numAtoms = pdb->num_atoms();

  Vector *positions = new Position[numAtoms];
  Vector *velocities = new Velocity[numAtoms];

 if ( basename ) {
  if ( params->binaryOutput ) {
    read_binary_file((std::string(basename)+".coor").c_str(), positions, numAtoms);
    read_binary_file((std::string(basename)+".vel").c_str(), velocities, numAtoms);
  } else {
    PDB coorpdb((std::string(basename)+".coor").c_str());
    if ( coorpdb.num_atoms() != numAtoms ) {
      NAMD_die("Incorrect atom count in coordinate pdb file");
    }
    coorpdb.get_all_positions(positions);
    velocities_from_PDB((std::string(basename)+".vel").c_str(), velocities, numAtoms);
  }
 } else {
  pdb->get_all_positions(positions);

  if ( params->initialTemp < 0.0 ) {
    Bool binvels=FALSE;

    //  Reading the veolcities from a PDB
    current = node->configList->find("velocities");

    if (current == NULL) {
      current = node->configList->find("binvelocities");
      binvels = TRUE;
    }

    if (!binvels) {
      velocities_from_PDB(current->data, velocities, numAtoms);
    }
    else {
      velocities_from_binfile(current->data, velocities, numAtoms);
    }
  }
  else {
    // Random velocities for a given temperature
    random_velocities(params->initialTemp, molecule, velocities, numAtoms);
  }
 }

  //  If COMMotion == no, remove center of mass motion
  if (!(params->comMove)) {
    remove_com_motion(velocities, molecule, numAtoms);
  }

  FullAtomList *atoms = new FullAtomList[numPatches];

  const Lattice lattice = params->lattice;

    if ( params->staticAtomAssignment ) {
      FullAtomList sortAtoms;
      for ( i=0; i < numAtoms; i++ ) {
        HydrogenGroupID &h = molecule->hydrogenGroup[i];
        if ( ! h.isMP ) continue;
        FullAtom a;
        a.id = i;
        a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
        a.position = positions[h.atomID];
        sortAtoms.add(a);
      } 
      int *order = new int[sortAtoms.size()];
      for ( i=0; i < sortAtoms.size(); i++ ) {
        order[i] = i;
      } 
      int *breaks = new int[numPatches];
      sortAtomsForPatches(order,breaks,sortAtoms.begin(),
                        sortAtoms.size(),numAtoms,
                        patchMap->gridsize_c(),
                        patchMap->gridsize_b(),
                        patchMap->gridsize_a());

      i = 0;
      for ( int pid = 0; pid < numPatches; ++pid ) {
        int iend = breaks[pid];
        for ( ; i<iend; ++i ) {
          FullAtom &sa = sortAtoms[order[i]];
          int mgs = sa.migrationGroupSize;
/*
CkPrintf("patch %d (%d %d %d) has group %d atom %d size %d at %.2f %.2f %.2f\n",
          pid, patchMap->index_a(pid), patchMap->index_b(pid), 
          patchMap->index_c(pid), order[i], sa.id, mgs,
          sa.position.x, sa.position.y, sa.position.z);
*/
          for ( int k=0; k<mgs; ++k ) {
            HydrogenGroupID &h = molecule->hydrogenGroup[sa.id + k];
            int aid = h.atomID;
            FullAtom a;
            a.id = aid;
            a.position = positions[aid];
            a.velocity = velocities[aid];
            a.vdwType = molecule->atomvdwtype(aid);
            a.status = molecule->getAtoms()[aid].status;
            a.langevinParam = molecule->langevin_param(aid);
            a.hydrogenGroupSize = h.isGP ? h.atomsInGroup : 0;
            a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
            if(params->rigidBonds != RIGID_NONE) {
              a.rigidBondLength = molecule->rigid_bond_length(aid);
            }else{
              a.rigidBondLength = 0.0;
            }
            atoms[pid].add(a);
          }
        }
CkPrintf("patch %d (%d %d %d) has %d atoms\n",
          pid, patchMap->index_a(pid), patchMap->index_b(pid), 
          patchMap->index_c(pid), atoms[pid].size());
      }
      delete [] order;
      delete [] breaks;
    } else
    {
    // split atoms into patches based on migration group and position
    int aid, pid=0;
    for(i=0; i < numAtoms; i++)
      {
      // Assign atoms to patches without splitting hydrogen groups.
      // We know that the hydrogenGroup array is sorted with group parents
      // listed first.  Thus, only change the pid if an atom is a group parent.
      HydrogenGroupID &h = molecule->hydrogenGroup[i];
      aid = h.atomID;
      FullAtom a;
      a.id = aid;
      a.position = positions[aid];
      a.velocity = velocities[aid];
      a.vdwType = molecule->atomvdwtype(aid);
      a.status = molecule->getAtoms()[aid].status;
      a.langevinParam = molecule->langevin_param(aid);
      a.hydrogenGroupSize = h.isGP ? h.atomsInGroup : 0;
      a.migrationGroupSize = h.isMP ? h.atomsInMigrationGroup : 0;
      if(params->rigidBonds != RIGID_NONE) {
        a.rigidBondLength = molecule->rigid_bond_length(aid);
      }else{
        a.rigidBondLength = 0.0;
      }
      if (h.isMP) {
        pid = patchMap->assignToPatch(positions[aid],lattice);
      } // else: don't change pid
      atoms[pid].add(a);
      }
    }

  delete [] positions;
  delete [] velocities;

  for(i=0; i < numPatches; i++)
  {
    ScaledPosition center(0.5*(patchMap->min_a(i)+patchMap->max_a(i)),
                          0.5*(patchMap->min_b(i)+patchMap->max_b(i)),
                          0.5*(patchMap->min_c(i)+patchMap->max_c(i)));

    int n = atoms[i].size();
    FullAtom *a = atoms[i].begin();
    int j;
//Modifications for alchemical fep
    Bool alchOn = params->alchOn;
//fepe
    Bool lesOn = params->lesOn;
  
    Bool pairInteractionOn = params->pairInteractionOn;

    Bool pressureProfileTypes = (params->pressureProfileAtomTypes > 1);

    Transform mother_transform;
    for(j=0; j < n; j++)
    {
      int aid = a[j].id;

      a[j].nonbondedGroupSize = 0;  // must be set based on coordinates

      a[j].atomFixed = molecule->is_atom_fixed(aid) ? 1 : 0;
      a[j].fixedPosition = a[j].position;

      if ( a[j].migrationGroupSize ) {
       if ( a[j].migrationGroupSize != a[j].hydrogenGroupSize ) {
            Position pos = a[j].position;
            int mgs = a[j].migrationGroupSize;
            int c = 1;
            for ( int k=a[j].hydrogenGroupSize; k<mgs;
                                k+=a[j+k].hydrogenGroupSize ) {
              pos += a[j+k].position;
              ++c;
            }
            pos *= 1./c;
            mother_transform = a[j].transform;  // should be 0,0,0
            pos = lattice.nearest(pos,center,&mother_transform);
            a[j].position = lattice.apply_transform(a[j].position,mother_transform);
            a[j].transform = mother_transform;
       } else {
        a[j].position = lattice.nearest(
                a[j].position, center, &(a[j].transform));
        mother_transform = a[j].transform;
       }
      } else {
        a[j].position = lattice.apply_transform(a[j].position,mother_transform);
        a[j].transform = mother_transform;
      }

      a[j].mass = molecule->atommass(aid);
      // Using double precision division for reciprocal mass.
      a[j].recipMass = ( a[j].mass > 0 ? (1. / a[j].mass) : 0 );
      a[j].charge = molecule->atomcharge(aid);

//Modifications for alchemical fep
      if ( alchOn || lesOn || pairInteractionOn || pressureProfileTypes) {
        a[j].partition = molecule->get_fep_type(aid);
      } 
      else {
        a[j].partition = 0;
      }
//fepe

    }

    int size, allfixed, k;
    for(j=0; j < n; j+=size) {
      size = a[j].hydrogenGroupSize;
      if ( ! size ) {
        NAMD_bug("Mother atom with hydrogenGroupSize of 0!");
      }
      allfixed = 1;
      for ( k = 0; k < size; ++k ) {
        allfixed = ( allfixed && (a[j+k].atomFixed) );
      }
      for ( k = 0; k < size; ++k ) {
        a[j+k].groupFixed = allfixed ? 1 : 0;
      }
    }

    if ( params->outputPatchDetails ) {
      int patchId = i;
      int numAtomsInPatch = n;
      int numFixedAtomsInPatch = 0;
      int numAtomsInFixedGroupsInPatch = 0;
      for(j=0; j < n; j++) {
        numFixedAtomsInPatch += ( a[j].atomFixed ? 1 : 0 );
        numAtomsInFixedGroupsInPatch += ( a[j].groupFixed ? 1 : 0 );
      }
      iout << "PATCH_DETAILS:"
           << " patch " << patchId
           << " atoms " << numAtomsInPatch
           << " fixed_atoms " << numFixedAtomsInPatch
           << " fixed_groups " << numAtomsInFixedGroupsInPatch
           << "\n" << endi;
    }
  }

  return atoms;

}

void WorkDistrib::createHomePatches

(

void

)

Definition at line 889 of file WorkDistrib.C.

References atoms, createAtomLists(), PatchMgr::createHomePatch(), DebugM, endi(), iINFO(), iout, PatchMap::numPatches(), numPatches, PatchMap::Object(), and ResizeArray< T >::size().

Referenced by Node::startup().

{
  int i;
  PatchMap *patchMap = PatchMap::Object();
  CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
  PatchMgr *patchMgr = pm.ckLocalBranch();

  int numPatches = patchMap->numPatches();

  FullAtomList *atoms = createAtomLists();
    
#ifdef MEM_OPT_VERSION
/*  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
  Node *node = nd.ckLocalBranch();
  node->molecule->delEachAtomSigs();
  node->molecule->delMassChargeSpace();
*/
#endif

  int maxAtoms = -1;
  int maxPatch = -1;
  for(i=0; i < numPatches; i++) {
    int numAtoms = atoms[i].size();
    if ( numAtoms > maxAtoms ) { maxAtoms = numAtoms; maxPatch = i; }
  }
  iout << iINFO << "LARGEST PATCH (" << maxPatch <<
        ") HAS " << maxAtoms << " ATOMS\n" << endi;

  for(i=0; i < numPatches; i++)
  {
    if ( ! ( i % 100 ) )
    {
      DebugM(3,"Created " << i << " patches so far.\n");
    }

    patchMgr->createHomePatch(i,atoms[i]);
  }

  delete [] atoms;
}

void WorkDistrib::distributeHomePatches

(

void

)

Definition at line 930 of file WorkDistrib.C.

References DebugM, PatchMgr::movePatch(), Node::myid(), PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), and PatchMgr::sendMovePatches().

Referenced by Node::startup().

                                        {
  // ref BOC
  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
  Node *node = nd.ckLocalBranch();
  CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
  PatchMgr *patchMgr = pm.ckLocalBranch();
  // ref singleton
  PatchMap *patchMap = PatchMap::Object();

  // Move patches to the proper node
  for(int i=0;i < patchMap->numPatches(); i++)
  {
    if (patchMap->node(i) != node->myid() )
    {
      DebugM(3,"patchMgr->movePatch("
        << i << "," << patchMap->node(i) << ")\n");
      patchMgr->movePatch(i,patchMap->node(i));
    }
  }
  patchMgr->sendMovePatches();
}

void WorkDistrib::doneSaveComputeMap

(

CkReductionMsg *

msg

)

Definition at line 423 of file WorkDistrib.C.

                                                        {
  delete msg;

  CkSendMsgBranch(saveComputeMapReturnEP, CkAllocMsg(0,0,0), 0, saveComputeMapReturnChareID);
}

void WorkDistrib::enqueueAngles

(

LocalWorkMsg *

msg

)

Definition at line 2968 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueAniso

(

LocalWorkMsg *

msg

)

Definition at line 2992 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueBonds

(

LocalWorkMsg *

msg

)

Definition at line 2962 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueCrossterms

(

LocalWorkMsg *

msg

)

Definition at line 2998 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                     {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueCUDA

(

LocalWorkMsg *

msg

)

Definition at line 3096 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                               {
  msg->compute->doWork();  MACHINE_PROGRESS
  // ComputeNonbondedCUDA *c = msg->compute;
  // if ( c->localWorkMsg != msg && c->localWorkMsg2 != msg )
  //   NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueCUDAP2

(

LocalWorkMsg *

msg

)

Definition at line 3102 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
}

void WorkDistrib::enqueueCUDAP3

(

LocalWorkMsg *

msg

)

Definition at line 3105 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
}

void WorkDistrib::enqueueDihedrals

(

LocalWorkMsg *

msg

)

Definition at line 2974 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                    {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueExcls

(

LocalWorkMsg *

msg

)

Definition at line 2956 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueGromacsPair

(

LocalWorkMsg *

msg

)

Definition at line 3005 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                      {
  msg->compute->doWork();
  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("\nWorkDistrib LocalWorkMsg recycling failed! Check enqueueGromacsPair from WorkDistrib.C\n");
}

void WorkDistrib::enqueueImpropers

(

LocalWorkMsg *

msg

)

Definition at line 2980 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                    {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueLCPO

(

LocalWorkMsg *

msg

)

Definition at line 3019 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, and NAMD_bug().

                                               {
  msg->compute->doWork();
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueMIC

(

LocalWorkMsg *

msg

)

Definition at line 3126 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                              {
  msg->compute->doWork();  MACHINE_PROGRESS
}

void WorkDistrib::enqueuePme

(

LocalWorkMsg *

msg

)

Definition at line 3013 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                              {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueSelfA1

(

LocalWorkMsg *

msg

)

Definition at line 3024 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueSelfA2

(

LocalWorkMsg *

msg

)

Definition at line 3029 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueSelfA3

(

LocalWorkMsg *

msg

)

Definition at line 3034 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueSelfB1

(

LocalWorkMsg *

msg

)

Definition at line 3040 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueSelfB2

(

LocalWorkMsg *

msg

)

Definition at line 3045 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueSelfB3

(

LocalWorkMsg *

msg

)

Definition at line 3050 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueThole

(

LocalWorkMsg *

msg

)

Definition at line 2986 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWork

(

LocalWorkMsg *

msg

)

Definition at line 2950 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                               {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWorkA1

(

LocalWorkMsg *

msg

)

Definition at line 3056 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWorkA2

(

LocalWorkMsg *

msg

)

Definition at line 3061 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWorkA3

(

LocalWorkMsg *

msg

)

Definition at line 3066 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWorkB1

(

LocalWorkMsg *

msg

)

Definition at line 3072 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWorkB2

(

LocalWorkMsg *

msg

)

Definition at line 3077 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWorkB3

(

LocalWorkMsg *

msg

)

Definition at line 3082 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                 {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::enqueueWorkC

(

LocalWorkMsg *

msg

)

Definition at line 3090 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::localWorkMsg, MACHINE_PROGRESS, and NAMD_bug().

                                                {
  msg->compute->doWork();  MACHINE_PROGRESS
  if ( msg->compute->localWorkMsg != msg )
    NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::finishCUDA

(

LocalWorkMsg *

msg

)

Definition at line 3113 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                              {
  msg->compute->doWork();  MACHINE_PROGRESS
  // ComputeNonbondedCUDA *c = msg->compute;
  // if ( c->localWorkMsg != msg && c->localWorkMsg2 != msg )
  //   NAMD_bug("WorkDistrib LocalWorkMsg recycling failed!");
}

void WorkDistrib::finishCUDAP2

(

LocalWorkMsg *

msg

)

Definition at line 3119 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                {
  msg->compute->doWork();  MACHINE_PROGRESS
}

void WorkDistrib::finishCUDAP3

(

LocalWorkMsg *

msg

)

Definition at line 3122 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                                {
  msg->compute->doWork();  MACHINE_PROGRESS
}

void WorkDistrib::finishCUDAPatch

(

FinishWorkMsg *

msg

)

Definition at line 3109 of file WorkDistrib.C.

References FinishWorkMsg::compute, FinishWorkMsg::data, and Compute::finishPatch().

                                                    {
  msg->compute->finishPatch(msg->data);
}

void WorkDistrib::finishMIC

(

LocalWorkMsg *

msg

)

Definition at line 3129 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), and MACHINE_PROGRESS.

                                             {
  msg->compute->doWork();  MACHINE_PROGRESS
}

void WorkDistrib::initHostDeviceLDB

(

)

Definition at line 3384 of file WorkDistrib.C.

                                    {
  #if defined(NAMD_MIC)
    mic_hostDeviceLDB();
  #endif
}

void WorkDistrib::mapComputes

(

void

)

Definition at line 2269 of file WorkDistrib.C.

References ComputeMap::allocateCids(), SimParameters::bondedCUDA, computeAnglesType, computeAnisoType, computeBondsType, computeConsForceType, computeConsTorqueType, computeCrosstermsType, computeCylindricalBCType, computeDihedralsType, computeEFieldType, computeEwaldType, computeExclsType, computeExtType, computeFmmType, computeFullDirectType, computeGBISserType, computeGlobalType, computeGridForceType, computeGromacsPairType, computeImpropersType, SimParameters::computeMapFilename, computeMsmMsaType, computeMsmSerialType, computeMsmType, computeNonbondedCUDA2Type, computeNonbondedCUDAType, computeNonbondedMICType, computePmeCUDAType, computePmeType, computeQMType, computeRestraintsType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfGromacsPairType, computeSelfImpropersType, computeSelfTholeType, computeSphericalBCType, computeStirType, computeTclBCType, computeTholeType, SimParameters::consForceOn, SimParameters::consTorqueOn, SimParameters::constraintsOn, SimParameters::cylindricalBCOn, DebugM, SimParameters::drudeOn, SimParameters::eFieldOn, SimParameters::extForcesOn, SimParameters::FMAOn, SimParameters::FMMOn, SimParameters::fullDirectOn, SimParameters::GBISserOn, SimParameters::globalForcesOn, SimParameters::goGroPair, SimParameters::LCPOOn, ComputeMap::loadComputeMap(), SimParameters::loadComputeMap, SimParameters::mgridforceOn, SimParameters::MSMOn, SimParameters::MsmSerialOn, NAMD_die(), PatchMap::Object(), Node::Object(), ComputeMap::Object(), SimParameters::pairInteractionOn, SimParameters::pairInteractionSelf, SimParameters::PMEOn, SimParameters::pressureProfileEwaldOn, SimParameters::qmForcesOn, ComputeMap::saveComputeMap(), Node::simParameters, SimParameters::sphericalBCOn, SimParameters::stirOn, SimParameters::storeComputeMap, SimParameters::tclBCOn, SimParameters::useCUDA2, SimParameters::useDPME, and SimParameters::usePMECUDA.

Referenced by Node::startup().

{
  PatchMap *patchMap = PatchMap::Object();
  ComputeMap *computeMap = ComputeMap::Object();
  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
  Node *node = nd.ckLocalBranch();

  DebugM(3,"Mapping computes\n");

  computeMap->allocateCids();

  // Handle full electrostatics
  if ( node->simParameters->fullDirectOn )
    mapComputeHomePatches(computeFullDirectType);
  if ( node->simParameters->FMAOn )
#ifdef DPMTA
    mapComputeHomePatches(computeDPMTAType);
#else
    NAMD_die("This binary does not include DPMTA (FMA).");
#endif
  if ( node->simParameters->PMEOn ) {
#ifdef DPME
    if ( node->simParameters->useDPME )
      mapComputeHomePatches(computeDPMEType);
    else {
      mapComputeHomePatches(computePmeType);
      if ( node->simParameters->pressureProfileEwaldOn )
        mapComputeHomePatches(computeEwaldType);
    }
#else
#if defined(NAMD_CUDA) || defined(NAMD_HIP)
    if (node->simParameters->usePMECUDA) {
      mapComputePatch(computePmeCUDAType);
    } else
#endif
    {
      mapComputePatch(computePmeType);
    }
    if ( node->simParameters->pressureProfileEwaldOn )
      mapComputeHomePatches(computeEwaldType);
#endif
  }

  if ( node->simParameters->globalForcesOn ) {
    DebugM(2,"adding ComputeGlobal\n");
    mapComputeHomePatches(computeGlobalType);
  }

  if ( node->simParameters->extForcesOn )
    mapComputeHomePatches(computeExtType);

  if ( node->simParameters->qmForcesOn )
    mapComputeHomePatches(computeQMType);

  if ( node->simParameters->GBISserOn )
    mapComputeHomePatches(computeGBISserType);

  if ( node->simParameters->MsmSerialOn )
    mapComputeHomePatches(computeMsmSerialType);
#ifdef CHARM_HAS_MSA
  else if ( node->simParameters->MSMOn )
    mapComputeHomePatches(computeMsmMsaType);
#else
  else if ( node->simParameters->MSMOn )
    mapComputeHomePatches(computeMsmType);
#endif

  if ( node->simParameters->FMMOn )
    mapComputeHomePatches(computeFmmType);

#if defined(NAMD_CUDA) || defined(NAMD_HIP)
#ifdef BONDED_CUDA
  if (node->simParameters->bondedCUDA) {
    mapComputeNode(computeBondedCUDAType);
  }
#endif
#endif

#if defined(NAMD_CUDA) || defined(NAMD_HIP)
  if (node->simParameters->useCUDA2) {
    mapComputeNode(computeNonbondedCUDA2Type);
  } else {
    mapComputeNode(computeNonbondedCUDAType);
  }
  mapComputeHomeTuples(computeExclsType);
  mapComputePatch(computeSelfExclsType);
#endif

#ifdef NAMD_MIC
  mapComputeNode(computeNonbondedMICType);
#endif

  mapComputeNonbonded();

  if ( node->simParameters->LCPOOn ) {
    mapComputeLCPO();
  }

  // If we're doing true pair interactions, no need for bonded terms.
  // But if we're doing within-group interactions, we do need them.
  if ( !node->simParameters->pairInteractionOn || 
      node->simParameters->pairInteractionSelf) { 
    mapComputeHomeTuples(computeBondsType);
    mapComputeHomeTuples(computeAnglesType);
    mapComputeHomeTuples(computeDihedralsType);
    mapComputeHomeTuples(computeImpropersType);
    mapComputeHomeTuples(computeCrosstermsType);
    mapComputePatch(computeSelfBondsType);
    mapComputePatch(computeSelfAnglesType);
    mapComputePatch(computeSelfDihedralsType);
    mapComputePatch(computeSelfImpropersType);
    mapComputePatch(computeSelfCrosstermsType);
  }

  if ( node->simParameters->goGroPair ) {
      // JLai
      mapComputeHomeTuples(computeGromacsPairType);
      mapComputePatch(computeSelfGromacsPairType);
    // End of JLai
  }

  if ( node->simParameters->drudeOn ) {
    mapComputeHomeTuples(computeTholeType);
    mapComputePatch(computeSelfTholeType);
    mapComputeHomeTuples(computeAnisoType);
    mapComputePatch(computeSelfAnisoType);
  }

  if ( node->simParameters->eFieldOn )
    mapComputePatch(computeEFieldType);
  /* BEGIN gf */
  if ( node->simParameters->mgridforceOn )
    mapComputePatch(computeGridForceType);
  /* END gf */
  if ( node->simParameters->stirOn )
    mapComputePatch(computeStirType);
  if ( node->simParameters->sphericalBCOn )
    mapComputePatch(computeSphericalBCType);
  if ( node->simParameters->cylindricalBCOn )
    mapComputePatch(computeCylindricalBCType);
  if ( node->simParameters->tclBCOn ) {
    mapComputeHomePatches(computeTclBCType);
  }
  if ( node->simParameters->constraintsOn )
    mapComputePatch(computeRestraintsType);
  if ( node->simParameters->consForceOn )
    mapComputePatch(computeConsForceType);
  if ( node->simParameters->consTorqueOn )
    mapComputePatch(computeConsTorqueType);

    // store the latest compute map
  SimParameters *simParams = Node::Object()->simParameters;
  if (simParams->storeComputeMap) {
    computeMap->saveComputeMap(simParams->computeMapFilename);
  }
    // override mapping decision
  if (simParams->loadComputeMap) {
    computeMap->loadComputeMap(simParams->computeMapFilename);
    CkPrintf("ComputeMap has been loaded from %s.\n", simParams->computeMapFilename);
  }
}

void WorkDistrib::messageEnqueueWork ( Compute *  compute )

static

Definition at line 2727 of file WorkDistrib.C.

References Compute::cid, LocalWorkMsg::compute, computeAnglesType, computeAnisoType, computeBondsType, computeCrosstermsType, computeDihedralsType, computeExclsType, computeGromacsPairType, computeImpropersType, computeNonbondedCUDA2Type, computeNonbondedCUDAType, computeNonbondedMICType, computeNonbondedPairType, computeNonbondedSelfType, computePmeCUDAType, computePmeType, computeSelfAnglesType, computeSelfAnisoType, computeSelfBondsType, computeSelfCrosstermsType, computeSelfDihedralsType, computeSelfExclsType, computeSelfGromacsPairType, computeSelfImpropersType, computeSelfTholeType, computeTholeType, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), SET_PRIORITY, and Compute::type().

Referenced by cuda_check_pme_forces(), Compute::enqueueWork(), CudaComputeNonbonded::messageEnqueueWork(), ComputePmeMgr::submitReductions(), and ComputePmeMgr::ungridCalc().

                                                     {
  LocalWorkMsg *msg = compute->localWorkMsg;
  int seq = compute->sequence();
  int gbisPhase = compute->getGBISPhase();

  if ( seq < 0 ) {
    NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageEnqueueWork");
  } else {
    SET_PRIORITY(msg,seq,compute->priority());
  }

  msg->compute = compute; // pointer is valid since send is to local Pe
  int type = compute->type();
  int cid = compute->cid;

  CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
  switch ( type ) {
  case computeExclsType:
  case computeSelfExclsType:
    wdProxy[CkMyPe()].enqueueExcls(msg);
    break;
  case computeBondsType:
  case computeSelfBondsType:
    wdProxy[CkMyPe()].enqueueBonds(msg);
    break;
  case computeAnglesType:
  case computeSelfAnglesType:
    wdProxy[CkMyPe()].enqueueAngles(msg);
    break;
  case computeDihedralsType:
  case computeSelfDihedralsType:
    wdProxy[CkMyPe()].enqueueDihedrals(msg);
    break;
  case computeImpropersType:
  case computeSelfImpropersType:
    wdProxy[CkMyPe()].enqueueImpropers(msg);
    break;
  case computeTholeType:
  case computeSelfTholeType:
    wdProxy[CkMyPe()].enqueueThole(msg);
    break;
  case computeAnisoType:
  case computeSelfAnisoType:
    wdProxy[CkMyPe()].enqueueAniso(msg);
    break;
  case computeCrosstermsType:
  case computeSelfCrosstermsType:
    wdProxy[CkMyPe()].enqueueCrossterms(msg);
    break;
  // JLai
  case computeGromacsPairType:
  case computeSelfGromacsPairType:
    wdProxy[CkMyPe()].enqueueGromacsPair(msg);
    break;    
  // End of JLai
  case computeLCPOType:
    wdProxy[CkMyPe()].enqueueLCPO(msg);
    break;
  case computeNonbondedSelfType:
    switch ( seq % 2 ) {
    case 0:
      //wdProxy[CkMyPe()].enqueueSelfA(msg);
      switch ( gbisPhase ) {
         case 1:
           wdProxy[CkMyPe()].enqueueSelfA1(msg);
           break;
         case 2:
           wdProxy[CkMyPe()].enqueueSelfA2(msg);
           break;
         case 3:
           wdProxy[CkMyPe()].enqueueSelfA3(msg);
           break;
      }
      break;
    case 1:
      //wdProxy[CkMyPe()].enqueueSelfB(msg);
      switch ( gbisPhase ) {
         case 1:
           wdProxy[CkMyPe()].enqueueSelfB1(msg);
           break;
         case 2:
           wdProxy[CkMyPe()].enqueueSelfB2(msg);
           break;
         case 3:
           wdProxy[CkMyPe()].enqueueSelfB3(msg);
           break;
      }
      break;
    default:
      NAMD_bug("WorkDistrib::messageEnqueueSelf case statement error!");
    }
    break;
  case computeNonbondedPairType:
    switch ( seq % 2 ) {
    case 0:
      //wdProxy[CkMyPe()].enqueueWorkA(msg);
      switch ( gbisPhase ) {
         case 1:
           wdProxy[CkMyPe()].enqueueWorkA1(msg);
           break;
         case 2:
           wdProxy[CkMyPe()].enqueueWorkA2(msg);
           break;
         case 3:
           wdProxy[CkMyPe()].enqueueWorkA3(msg);
           break;
      }
      break;
    case 1:
      //wdProxy[CkMyPe()].enqueueWorkB(msg);
      switch ( gbisPhase ) {
         case 1:
           wdProxy[CkMyPe()].enqueueWorkB1(msg);
           break;
         case 2:
           wdProxy[CkMyPe()].enqueueWorkB2(msg);
           break;
         case 3:
           wdProxy[CkMyPe()].enqueueWorkB3(msg);
           break;
      }
      break;
    case 2:
      wdProxy[CkMyPe()].enqueueWorkC(msg);
      break;
    default:
      NAMD_bug("WorkDistrib::messageEnqueueWork case statement error!");
    }
    break;
  case computeNonbondedCUDAType:
#if defined(NAMD_CUDA) || defined(NAMD_HIP)
  case computeNonbondedCUDA2Type:
//     CkPrintf("WorkDistrib[%d]::CUDA seq=%d phase=%d\n", CkMyPe(), seq, gbisPhase);
    //wdProxy[CkMyPe()].enqueueCUDA(msg);
    switch ( gbisPhase ) {
       case 1:
         wdProxy[CkMyPe()].enqueueCUDA(msg);
         break;
       case 2:
         wdProxy[CkMyPe()].enqueueCUDAP2(msg);
         break;
       case 3:
         wdProxy[CkMyPe()].enqueueCUDAP3(msg);
         break;
    }
#else
    msg->compute->doWork();  MACHINE_PROGRESS
#endif
    break;
  case computeNonbondedMICType:
#ifdef NAMD_MIC
    wdProxy[CkMyPe()].enqueueMIC(msg);
#endif
    break;
  case computePmeType:
    // CkPrintf("PME %d %d %x\n", CkMyPe(), seq, compute->priority());
    wdProxy[CkMyPe()].enqueuePme(msg);
    break;
#if defined(NAMD_CUDA) || defined(NAMD_HIP)
  case computePmeCUDAType:
    wdProxy[CkMyPe()].enqueuePme(msg);
    break;
#endif
  default:
    wdProxy[CkMyPe()].enqueueWork(msg);
  }
}

void WorkDistrib::messageFinishCUDA ( Compute *  compute )

static

Definition at line 2896 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), and SET_PRIORITY.

Referenced by cuda_check_local_progress(), and cuda_check_remote_progress().

                                                    {
  LocalWorkMsg *msg = compute->localWorkMsg;
  int seq = compute->sequence();
  int gbisPhase = compute->getGBISPhase();

  if ( seq < 0 ) {
    NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageEnqueueWork");
  } else {
    SET_PRIORITY(msg,seq,compute->priority());
  }

  msg->compute = compute; // pointer is valid since send is to local Pe
  CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);

#if defined(NAMD_CUDA) || defined(NAMD_HIP)
    //wdProxy[CkMyPe()].finishCUDA(msg);
    switch ( gbisPhase ) {
       case 1:
         wdProxy[CkMyPe()].finishCUDA(msg);
         break;
       case 2:
         wdProxy[CkMyPe()].finishCUDAP2(msg);
         break;
       case 3:
         wdProxy[CkMyPe()].finishCUDAP3(msg);
         break;
    }
#else
    msg->compute->doWork();  MACHINE_PROGRESS
#endif
}

void WorkDistrib::messageFinishMIC ( Compute *  compute )

static

Definition at line 2929 of file WorkDistrib.C.

References LocalWorkMsg::compute, Compute::doWork(), Compute::getGBISPhase(), Compute::localWorkMsg, MACHINE_PROGRESS, NAMD_bug(), Compute::priority(), Compute::sequence(), and SET_PRIORITY.

                                                   {
  LocalWorkMsg *msg = compute->localWorkMsg;
  int seq = compute->sequence();
  int gbisPhase = compute->getGBISPhase();

  if ( seq < 0 ) {
    NAMD_bug("compute->sequence() < 0 in WorkDistrib::messageFinishMIC");
  } else {
    SET_PRIORITY(msg,seq,compute->priority());
  }

  msg->compute = compute; // pointer is valid since send is to local Pe
  CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);

#ifdef NAMD_MIC
    wdProxy[CkMyPe()].finishMIC(msg);
#else
    msg->compute->doWork();  MACHINE_PROGRESS
#endif
}

void WorkDistrib::patchMapInit

(

void

)

Definition at line 1105 of file WorkDistrib.C.

References Lattice::a_p(), Lattice::a_r(), Lattice::b_p(), Lattice::b_r(), Lattice::c_p(), Lattice::c_r(), DebugM, endi(), PDB::find_extremes(), SimParameters::FMAOn, SimParameters::FMMOn, PDB::get_extremes(), iINFO(), iout, isOutputProcessor(), iWARN(), SimParameters::lattice, SimParameters::LCPOOn, PatchMap::makePatches(), SimParameters::maxPatches, SimParameters::minAtomsPerPatch, Node::molecule, SimParameters::MSMOn, NAMD_die(), SimParameters::noPatchesOnOne, SimParameters::noPatchesOnOutputPEs, SimParameters::noPatchesOnZero, PDB::num_atoms(), Molecule::numAtoms, SimParameters::numoutputprocs, numPatches, PatchMap::Object(), Node::Object(), Lattice::origin(), SimParameters::patchDimension, Node::pdb, SimParameters::replicaUniformPatchGrids, Node::simParameters, SimParameters::simulatedPEs, SimParameters::simulateInitialMapping, PatchMap::sizeGrid(), SimParameters::staticAtomAssignment, SimParameters::twoAwayX, SimParameters::twoAwayY, SimParameters::twoAwayZ, Vector::x, Vector::y, and Vector::z.

Referenced by Node::startup().

{
  PatchMap *patchMap = PatchMap::Object();
  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
  Node *node = nd.ckLocalBranch();
  SimParameters *params = node->simParameters;
  Lattice lattice = params->lattice;

  BigReal patchSize = params->patchDimension;

#ifndef MEM_OPT_VERSION
  const int totalAtoms = node->pdb->num_atoms();
#else
  const int totalAtoms = node->molecule->numAtoms;
#endif

  ScaledPosition xmin, xmax;

  double maxNumPatches = 1.e9;  // need to adjust fractional values
  if ( params->minAtomsPerPatch > 0 )
    maxNumPatches = totalAtoms / params->minAtomsPerPatch;

  DebugM(3,"Mapping patches\n");
  if ( lattice.a_p() && lattice.b_p() && lattice.c_p() ) {
    xmin = 0.;  xmax = 0.;
  }
  else if ( params->FMAOn || params->MSMOn || params->FMMOn ) {
  // Need to use full box for FMA to match NAMD 1.X results.
#if 0
    node->pdb->find_extremes(&(xmin.x),&(xmax.x),lattice.a_r());
    node->pdb->find_extremes(&(xmin.y),&(xmax.y),lattice.b_r());
    node->pdb->find_extremes(&(xmin.z),&(xmax.z),lattice.c_r());
#endif
    node->pdb->find_extremes(lattice);
    node->pdb->get_extremes(xmin, xmax);
#if 0
    printf("+++ center=%.4f %.4f %.4f\n",
        lattice.origin().x, lattice.origin().y, lattice.origin().z);
    printf("+++ xmin=%.4f  xmax=%.4f\n", xmin.x, xmax.x);
    printf("+++ ymin=%.4f  ymax=%.4f\n", xmin.y, xmax.y);
    printf("+++ zmin=%.4f  zmax=%.4f\n", xmin.z, xmax.z);
#endif
  // Otherwise, this allows a small number of stray atoms.
  }
  else {
#if 0
    node->pdb->find_extremes(&(xmin.x),&(xmax.x),lattice.a_r(),0.9);
    node->pdb->find_extremes(&(xmin.y),&(xmax.y),lattice.b_r(),0.9);
    node->pdb->find_extremes(&(xmin.z),&(xmax.z),lattice.c_r(),0.9);
#endif
    node->pdb->find_extremes(lattice, 1.0);
    node->pdb->get_extremes(xmin, xmax);
    iout << iINFO << "ORIGINAL ATOMS MINMAX IS " << xmin << "  " << xmax << "\n" << endi;
    double frac = ( (double)totalAtoms - 10000. ) / (double)totalAtoms;
    if ( frac < 0.9 ) { frac = 0.9; }
    node->pdb->find_extremes(lattice, frac);
    node->pdb->get_extremes(xmin, xmax);
    iout << iINFO << "ADJUSTED ATOMS MINMAX IS " << xmin << "  " << xmax << "\n" << endi;
  }

#if 0
  BigReal origin_shift;
  origin_shift = lattice.a_r() * lattice.origin();
  xmin.x -= origin_shift;
  xmax.x -= origin_shift;
  origin_shift = lattice.b_r() * lattice.origin();
  xmin.y -= origin_shift;
  xmax.y -= origin_shift;
  origin_shift = lattice.c_r() * lattice.origin();
  xmin.z -= origin_shift;
  xmax.z -= origin_shift;
#endif

  // SimParameters default is -1 for unset
  int twoAwayX = params->twoAwayX;
  int twoAwayY = params->twoAwayY;
  int twoAwayZ = params->twoAwayZ;

  // SASA implementation is not compatible with twoAway patches
  if (params->LCPOOn && patchSize < 32.4) {
    if ( twoAwayX > 0 || twoAwayY > 0 || twoAwayZ > 0 ) {
      iout << iWARN << "Ignoring twoAway[XYZ] due to LCPO SASA implementation.\n" << endi;
    }
    twoAwayX = twoAwayY = twoAwayZ = 0;
  }

  // if you think you know what you're doing go right ahead
  if ( twoAwayX > 0 ) maxNumPatches = 1.e9;
  if ( twoAwayY > 0 ) maxNumPatches = 1.e9;
  if ( twoAwayZ > 0 ) maxNumPatches = 1.e9;
  if ( params->maxPatches > 0 ) {
      maxNumPatches = params->maxPatches;
      iout << iINFO << "LIMITING NUMBER OF PATCHES TO " <<
                                maxNumPatches << "\n" << endi;
  }

  int numpes = CkNumPes();
  SimParameters *simparam = Node::Object()->simParameters;
  if(simparam->simulateInitialMapping) {
    numpes = simparam->simulatedPEs;
    delete [] patchMap->nPatchesOnNode;
    patchMap->nPatchesOnNode = new int[numpes];
    memset(patchMap->nPatchesOnNode, 0, numpes*sizeof(int));    
  }

#if defined(NAMD_CUDA) || defined(NAMD_HIP) || defined(NAMD_MIC)
  // for CUDA be sure there are more patches than pes

  int numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
        twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
  if ( numPatches < numpes && twoAwayX < 0 ) {
    twoAwayX = 1;
    numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
        twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
  }
  if ( numPatches < numpes && twoAwayY < 0 ) {
    twoAwayY = 1;
    numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
        twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
  }
  if ( numPatches < numpes && twoAwayZ < 0 ) {
    twoAwayZ = 1;
    numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,maxNumPatches,params->staticAtomAssignment,
        twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
  }
  if ( numPatches < numpes ) {
    #if defined(NAMD_MIC)
    NAMD_die("MIC-enabled NAMD requires at least one patch per thread.");
    #endif
  }
  if ( numPatches % numpes && numPatches <= 1.4 * numpes ) {
    int exactFit = numPatches - numPatches % numpes;
    int newNumPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,exactFit,params->staticAtomAssignment,
        twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);
    if ( newNumPatches == exactFit ) {
      iout << iINFO << "REDUCING NUMBER OF PATCHES TO IMPROVE LOAD BALANCE\n" << endi;
      maxNumPatches = exactFit;
    }
  }

  patchMap->makePatches(xmin,xmax,lattice,patchSize,maxNumPatches,
        params->staticAtomAssignment, params->replicaUniformPatchGrids, params->LCPOOn,
        twoAwayX>0 ? 2 : 1, twoAwayY>0 ? 2 : 1, twoAwayZ>0 ? 2 : 1);

#else

  int availPes = numpes;
  if ( params->noPatchesOnZero && numpes > 1 ) {
      availPes -= 1;
      if(params->noPatchesOnOne && numpes > 2)
        availPes -= 1;
  }
#ifdef MEM_OPT_VERSION
  if(params->noPatchesOnOutputPEs && numpes - params->numoutputprocs >2)
    {
      availPes -= params->numoutputprocs;
      if ( params->noPatchesOnZero && numpes > 1 && isOutputProcessor(0)){
        availPes++;
      }
      if ( params->noPatchesOnOne && numpes > 2 && isOutputProcessor(1)){
        availPes++;
      }
    }
#endif

  int numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
        twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
  if ( ( numPatches > (0.3*availPes) || numPatches > maxNumPatches
       ) && twoAwayZ < 0 ) {
    twoAwayZ = 0;
    numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
        twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
  }
  if ( ( numPatches > (0.6*availPes) || numPatches > maxNumPatches
       ) && twoAwayY < 0 ) {
    twoAwayY = 0;
    numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
        twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
  }
  if ( ( numPatches > availPes || numPatches > maxNumPatches
       ) && twoAwayX < 0 ) {
    twoAwayX = 0;
    numPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,1.e9,params->staticAtomAssignment,
        twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
  }
  if ( numPatches > availPes && numPatches <= (1.4*availPes) && availPes <= maxNumPatches ) {
    int newNumPatches = patchMap->sizeGrid(
        xmin,xmax,lattice,patchSize,availPes,params->staticAtomAssignment,
        twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);
    if ( newNumPatches <= availPes && numPatches <= (1.4*newNumPatches) ) {
      iout << iINFO << "REDUCING NUMBER OF PATCHES TO IMPROVE LOAD BALANCE\n" << endi;
      maxNumPatches = availPes;
    }
  }

  patchMap->makePatches(xmin,xmax,lattice,patchSize,maxNumPatches,
        params->staticAtomAssignment, params->replicaUniformPatchGrids, params->LCPOOn,
        twoAwayX ? 2 : 1, twoAwayY ? 2 : 1, twoAwayZ ? 2 : 1);

#endif

}

void WorkDistrib::peOrderingReady ( )

static

Definition at line 166 of file WorkDistrib.C.

References cuda_initialize(), and mic_initialize().

Referenced by buildNodeAwarePeOrdering().

                                  {
  //CkPrintf("WorkDistrib::peOrderingReady on %d\n", CkMyPe());
#if defined(NAMD_CUDA) || defined(NAMD_HIP)
  cuda_initialize();
#endif
#ifdef NAMD_MIC
  mic_initialize();
#endif
}

void WorkDistrib::recvComputeMapChanges

(

ComputeMapChangeMsg *

msg

)

Definition at line 370 of file WorkDistrib.C.

References ALLBUTME, BUFSIZE, COMPUTEMAPTAG, MOStream::end(), MIStream::get(), NAMD_bug(), ComputeMap::newNode(), ComputeMap::newNumPartitions(), ComputeMap::numComputes(), ComputeMap::Object(), MOStream::put(), ComputeMap::setNewNode(), and ComputeMap::setNewNumPartitions().

                                                                {

  delete msg;

  ComputeMap *computeMap = ComputeMap::Object();

  int i;
  int nc = computeMap->numComputes();
  
  if ( ! CkMyPe() ) { // send
    // CkPrintf("At %f on %d WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
    MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, COMPUTEMAPTAG, BUFSIZE);
    msg->put(nc);
    for (i=0; i<nc; i++) {
      int data = computeMap->newNode(i);
      msg->put(data);
    }
    msg->put(nc);
    for (i=0; i<nc; i++) {
      char data = computeMap->newNumPartitions(i);
      msg->put(data);
    }
    msg->put(nc);
    msg->end();
    delete msg;
    // CkPrintf("At %f on %d done WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
  } else if ( ! CkMyRank() ) { // receive
    // if ( CkMyNode() == 1 ) CkPrintf("At %f on %d WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
    MIStream *msg = CkpvAccess(comm)->newInputStream(0, COMPUTEMAPTAG);
    msg->get(i);
    if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 1 failed\n");
    for (i=0; i<nc; i++) {
      int data;
      msg->get(data);
      computeMap->setNewNode(i,data);
    }
    msg->get(i);
    if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 2 failed\n");
    for (i=0; i<nc; i++) {
      char data;
      msg->get(data);
      computeMap->setNewNumPartitions(i,data);
    }
    msg->get(i);
    if ( i != nc ) NAMD_bug("WorkDistrib::recvComputeMapChanges check 3 failed\n");
    delete msg;
    // if ( CkMyNode() == 1 ) CkPrintf("At %f on %d done WorkDistrib::recvComputeMapChanges %d\n", CmiWallTimer(), CkMyPe(), nc);
  }

  CkCallback cb(CkIndex_WorkDistrib::doneSaveComputeMap(NULL), 0, thisgroup);
  contribute(0, NULL, CkReduction::random, cb);
}

void WorkDistrib::reinitAtoms

(

const char *

basename = 0

)

Definition at line 952 of file WorkDistrib.C.

References atoms, createAtomLists(), PatchMap::numPatches(), numPatches, PatchMap::Object(), and PatchMgr::sendAtoms().

                                                  {

  PatchMap *patchMap = PatchMap::Object();
  CProxy_PatchMgr pm(CkpvAccess(BOCclass_group).patchMgr);
  PatchMgr *patchMgr = pm.ckLocalBranch();

  int numPatches = patchMap->numPatches();

  FullAtomList *atoms = createAtomLists(basename);

  for(int i=0; i < numPatches; i++) {
    patchMgr->sendAtoms(i,atoms[i]);
  }

  delete [] atoms;

}

void WorkDistrib::saveComputeMapChanges	(	int	ep,
		CkGroupID	chareID
	)

Definition at line 352 of file WorkDistrib.C.

Referenced by ComputeMgr::updateComputes2().

{
  saveComputeMapReturnEP = ep;
  saveComputeMapReturnChareID = chareID;

  ComputeMapChangeMsg *mapMsg = new (0, 0, 0) ComputeMapChangeMsg;
  CProxy_WorkDistrib(thisgroup).recvComputeMapChanges(mapMsg);

/*
    // store the latest compute map
  SimParameters *simParams = Node::Object()->simParameters;
  if (simParams->storeComputeMap) {
    computeMap->saveComputeMap(simParams->computeMapFilename);
    CkPrintf("ComputeMap has been stored in %s.\n", simParams->computeMapFilename);
  }
*/
}

void WorkDistrib::savePatchMap

(

PatchMapMsg *

msg

)

Definition at line 1014 of file WorkDistrib.C.

References SimParameters::isRecvSpanningTreeUnset(), SimParameters::isSendSpanningTreeUnset(), numPatches, PatchMap::Object(), ProxyMgr::Object(), PatchMapMsg::patchMapData, ProxyMgr::setRecvSpanning(), ProxyMgr::setSendSpanning(), Node::simParameters, split(), and PatchMap::unpack().

{
  // Use a resend to forward messages before processing.  Otherwise the
  // map distribution is slow on many CPUs.  We need to use a tree
  // rather than a broadcast because some implementations of broadcast
  // generate a copy of the message on the sender for each recipient.
  // This is because MPI doesn't allow re-use of an outstanding buffer.

  if ( CkMyRank() ) patchMapArrived = true;

  if ( patchMapArrived && CkMyPe() ) {
    PatchMap::Object()->unpack(msg->patchMapData);

    //Automatically enable spanning tree
    CProxy_Node nd(CkpvAccess(BOCclass_group).node);
    Node *node = nd.ckLocalBranch();
    SimParameters *params = node->simParameters;
    if( ( PatchMap::Object()->numPatches() <= CkNumPes()/4
#ifdef NODEAWARE_PROXY_SPANNINGTREE 
        || CkNumPes() > CkNumNodes()
        ) && ( CkNumNodes() > 1
#endif
      ) && params->isSendSpanningTreeUnset() )
      ProxyMgr::Object()->setSendSpanning();

#ifdef NODEAWARE_PROXY_SPANNINGTREE 
    if ( CkNumPes() > CkNumNodes() && CkNumNodes() > 1
          && params->isRecvSpanningTreeUnset() )
      ProxyMgr::Object()->setRecvSpanning();
#endif
  }

  if ( patchMapArrived ) {
    if ( CkMyRank() + 1 < CkNodeSize(CkMyNode()) ) {
      ((CProxy_WorkDistrib(thisgroup))[CkMyPe()+1]).savePatchMap(msg);
    } else {
      delete msg;
    }
    return;
  }

  patchMapArrived = true;

  int self = CkMyNode();
  int range_begin = 0;
  int range_end = CkNumNodes();
  while ( self != range_begin ) {
    ++range_begin;
    int split = range_begin + ( range_end - range_begin ) / 2;
    if ( self < split ) { range_end = split; }
    else { range_begin = split; }
  }
  int send_near = self + 1;
  int send_far = send_near + ( range_end - send_near ) / 2;

  int pids[3];
  int npid = 0;
  if ( send_far < range_end ) pids[npid++] = CkNodeFirst(send_far);
  if ( send_near < send_far ) pids[npid++] = CkNodeFirst(send_near);
  pids[npid++] = CkMyPe();  // always send the message to ourselves
  CProxy_WorkDistrib(thisgroup).savePatchMap(msg,npid,pids);
}

void WorkDistrib::send_contributeHostDeviceLDB ( int  peSetLen, int *  peSet  )

static

Definition at line 3390 of file WorkDistrib.C.

                                                                        {
  #if defined(NAMD_MIC)
    CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
    wdProxy[0].contributeHostDeviceLDB(peSetLen, peSet);
  #endif
}

void WorkDistrib::send_initHostDeviceLDB ( )

static

Definition at line 3377 of file WorkDistrib.C.

                                         {
  #if defined(NAMD_MIC)
    CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
    wdProxy.initHostDeviceLDB();
  #endif
}

void WorkDistrib::send_setDeviceLDBParams ( int  dt, int  hs, int  sp1, int  pp1, int  pp2  )

static

Definition at line 3403 of file WorkDistrib.C.

                                                                                   {
  #if defined(NAMD_MIC)
    CProxy_WorkDistrib wdProxy(CkpvAccess(BOCclass_group).workDistrib);
    wdProxy.setDeviceLDBParams(dt, hs, sp1, pp1, pp2);
  #endif
}

void WorkDistrib::sendComputeMap

(

void

)

Definition at line 1078 of file WorkDistrib.C.

References ALLBUTME, BUFSIZE, COMPUTEMAPTAG, MOStream::end(), ComputeMap::initPtrs(), ComputeMap::Object(), ComputeMap::pack(), and ComputeMap::unpack().

Referenced by Node::startup().

{
  if ( CkMyRank() ) return;

  if ( CkNumNodes() == 1 ) {
    computeMapArrived = true;
    ComputeMap::Object()->initPtrs();
    return;
  }

  if ( ! CkMyPe() ) { // send
    MOStream *msg = CkpvAccess(comm)->newOutputStream(ALLBUTME, COMPUTEMAPTAG, BUFSIZE);
    ComputeMap::Object()->pack(msg);
    msg->end();
    delete msg;
  } else if ( ! CkMyRank() ) { // receive
    MIStream *msg = CkpvAccess(comm)->newInputStream(0, COMPUTEMAPTAG);
    ComputeMap::Object()->unpack(msg);
    delete msg;
  }

  computeMapArrived = true;
  ComputeMap::Object()->initPtrs();
}

void WorkDistrib::sendPatchMap

(

void

)

Definition at line 978 of file WorkDistrib.C.

References SimParameters::isRecvSpanningTreeUnset(), SimParameters::isSendSpanningTreeUnset(), numPatches, PatchMap::Object(), ProxyMgr::Object(), PatchMap::pack(), PatchMap::packSize(), PatchMapMsg::patchMapData, ProxyMgr::setRecvSpanning(), ProxyMgr::setSendSpanning(), and Node::simParameters.

Referenced by Node::startup().

{
  if ( CkNumPes() == 1 ) {
    patchMapArrived = true;
    return;
  }

  //Automatically enable spanning tree
  CProxy_Node nd(CkpvAccess(BOCclass_group).node);
  Node *node = nd.ckLocalBranch();
  SimParameters *params = node->simParameters;
  if( ( PatchMap::Object()->numPatches() <= CkNumPes()/4
#ifdef NODEAWARE_PROXY_SPANNINGTREE 
      || CkNumPes() > CkNumNodes()
      ) && ( CkNumNodes() > 1
#endif
    ) && params->isSendSpanningTreeUnset() )
    ProxyMgr::Object()->setSendSpanning();

#ifdef NODEAWARE_PROXY_SPANNINGTREE 
  if ( CkNumPes() > CkNumNodes() && CkNumNodes() > 1
        && params->isRecvSpanningTreeUnset() )
    ProxyMgr::Object()->setRecvSpanning();
#endif

  int size = PatchMap::Object()->packSize();

  PatchMapMsg *mapMsg = new (size, 0) PatchMapMsg;

  PatchMap::Object()->pack(mapMsg->patchMapData, size);

  CProxy_WorkDistrib workProxy(thisgroup);
  workProxy[0].savePatchMap(mapMsg);
}

void WorkDistrib::setDeviceLDBParams	(	int	dt,
		int	hs,
		int	sp1,
		int	pp1,
		int	pp2
	)

Definition at line 3410 of file WorkDistrib.C.

                                                                              {
  #if defined(NAMD_MIC)
    mic_setDeviceLDBParams(dt, hs, sp1, pp1, pp2);
  #endif
}

void WorkDistrib::setPatchMapArrived ( bool  s )

inline

Definition at line 107 of file WorkDistrib.h.

Referenced by Node::startup().

{patchMapArrived=s;}

void WorkDistrib::sortPmePes ( int *  pmepes, int  xdim, int  ydim  )

static

Definition at line 300 of file WorkDistrib.C.

References ResizeArray< T >::begin(), PatchMap::center(), PatchMap::node(), PatchMap::numPatches(), PatchMap::Object(), and recursive_bisect_coord().

Referenced by ComputePmeMgr::initialize().

                                                            {
  int numpes = CkNumPes();
  ResizeArray<int> count(numpes);
  ResizeArray<ScaledPosition> sumPos(numpes);
  ResizeArray<ScaledPosition> avgPos(numpes);
  for ( int i=0; i<numpes; ++i ) {
    count[i] = 0;
    sumPos[i] = 0;
    avgPos[i] = 0;
  }
  PatchMap *patchMap = PatchMap::Object();
  for ( int i=0, npatches=patchMap->numPatches(); i<npatches; ++i ) {
    int pe = patchMap->node(i);
    count[pe] += 1;
    sumPos[pe] += patchMap->center(i);
  }
  const int npmepes = xdim*ydim;
  ResizeArray<int> sortpes(npmepes);
  for ( int i=0; i<npmepes; ++i ) {
    int pe = sortpes[i] = pmepes[i];
    int cnt = count[pe];
    ScaledPosition sum = sumPos[pe];
    if ( cnt == 0 ) {
      // average over node
      int node = CkNodeOf(pe);
      int nsize = CkNodeSize(node);
      int pe2 = CkNodeFirst(node);
      for ( int j=0; j<nsize; ++j, ++pe2 )  {
        cnt += count[pe2];
        sum += sumPos[pe2];
      }
    }
    if ( cnt == 0 ) {
      // average over physical node
      int node = CmiPhysicalNodeID(pe);
      int nsize, *nlist;
      CmiGetPesOnPhysicalNode(node, &nlist, &nsize);
      for ( int j=0; j<nsize; ++j )  {
        int pe2 = nlist[j];
        cnt += count[pe2];
        sum += sumPos[pe2];
      }
    }
    if ( cnt ) {
      avgPos[pe] = sum / cnt;
    }
  }
  recursive_bisect_coord(0, xdim, 0, ydim, sortpes.begin(), avgPos.begin(), pmepes, ydim);
}

Member Data Documentation

int * WorkDistrib::peCompactOrdering

static

Definition at line 117 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering().

int * WorkDistrib::peCompactOrderingIndex

static

Definition at line 118 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering(), ParallelIOMgr::initialize(), WorkDistrib::pe_sortop_compact::operator()(), and TopoManagerWrapper::pe_sortop_topo::operator()().

int * WorkDistrib::peDiffuseOrdering

static

Definition at line 115 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering(), generatePmePeList2(), ParallelIOMgr::initialize(), and ComputePmeMgr::initialize().

int * WorkDistrib::peDiffuseOrderingIndex

static

Definition at line 116 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering(), and WorkDistrib::pe_sortop_diffuse::operator()().

int WorkDistrib::peOrderingInit

static

Definition at line 114 of file WorkDistrib.h.

Referenced by buildNodeAwarePeOrdering().

---

The documentation for this class was generated from the following files:

• WorkDistrib.h
• WorkDistrib.C