NAMD
PatchMgr.C
Go to the documentation of this file.
1 
7 #include "InfoStream.h"
8 #include "PatchMgr.decl.h"
9 #include "PatchMgr.h"
10 
11 #include "NamdTypes.h"
12 //#include "Compute.h"
13 #include "HomePatch.h"
14 #include "PatchMap.h"
15 #include "AtomMap.h"
16 
17 #include "ComputeMsmMsa.h" // needed for MsmMsaData definition
18 #include "main.decl.h"
19 #include "main.h"
20 
21 #include "WorkDistrib.decl.h"
22 #include "WorkDistrib.h"
23 #include "Node.h"
24 #include "SimParameters.h"
25 
26 #include "packmsg.h"
27 
28 // #define DEBUGM
29 #define MIN_DEBUG_LEVEL 3
30 #include "Debug.h"
31 
32 
33 // BOC constructor
35 {
36  // CkPrintf("[%d] PatchMgr Created\n", CkMyPe());
37 
38  // Singleton pattern
39  if (CkpvAccess(PatchMgr_instance) == NULL) {
40  CkpvAccess(PatchMgr_instance) = this;
41  } else {
42  NAMD_bug("PatchMgr instanced twice on same processor!");
43  }
44 
45  // Get PatchMap singleton started
46  patchMap = PatchMap::Instance();
47  patchMap->registerPatchMgr(this);
48 
49  recvCheckpointReq_index = CmiRegisterHandler((CmiHandler)recvCheckpointReq_handler);
50  recvCheckpointLoad_index = CmiRegisterHandler((CmiHandler)recvCheckpointLoad_handler);
51  recvCheckpointStore_index = CmiRegisterHandler((CmiHandler)recvCheckpointStore_handler);
52  recvCheckpointAck_index = CmiRegisterHandler((CmiHandler)recvCheckpointAck_handler);
53 
54  recvExchangeReq_index = CmiRegisterHandler((CmiHandler)recvExchangeReq_handler);
55  recvExchangeMsg_index = CmiRegisterHandler((CmiHandler)recvExchangeMsg_handler);
56 
57  // Message combining initialization
58  migrationCountdown = 0;
59  combineMigrationMsgs = new MigrateAtomsCombinedMsg*[CkNumPes()];
60  int numPes = CkNumPes();
61  for ( int i = 0; i < numPes; ++i ) combineMigrationMsgs[i] = 0;
62 }
63 
65 {
66  HomePatchListIter hi(homePatches);
67  for ( hi = hi.begin(); hi != hi.end(); hi++) {
68  HomePatchElem* elem = homePatches.find(HomePatchElem(hi->pid));
69  delete elem->patch;
70  }
71  delete [] combineMigrationMsgs;
72 }
73 
75 {
76  HomePatch *patch = new HomePatch(pid, a);
77  homePatches.load(HomePatchElem(pid, patch));
78  patchMap->registerPatch(pid, patch);
79 }
80 
81 
82 // Add a HomePatch to a list of patches to be moved
83 // HomePatches are actually moved by invoking sendMovePatches() below
85 {
86  move.load(MovePatch(pid,nodeID));
87 }
88 
89 void PatchMgr::sendOneHomePatch(int patchId, int nodeId){
90  HomePatch *p = homePatch(patchId);
91  patchMap->unregisterPatch(patchId, p);
92 
93  MovePatchesMsg *msg = new MovePatchesMsg(patchId, p->atom);
94 
95  // Deleting the HomePatchElem will call a destructor for clean up
96  // but the msg elements are safe since they use a container template
97  // that uses ref counting.
98  delete p;
99  homePatches.del(HomePatchElem(patchId));
100 
101  if ( msg->atom.shared() ) NAMD_bug("shared message array in PatchMgr::sendOneHomePatch");
102 
103  // Sending to PatchMgr::recvMovePatches on remote node
104  CProxy_PatchMgr cp(thisgroup);
105  cp[nodeId].recvMovePatches(msg);
106 }
107 
108 // Uses list constructed by movePatch() and dispatches
109 // HomePatch(es) to new nodes
111 {
112  if (! move.size())
113  return;
114 
115  MovePatchListIter m(move);
116  for ( m = m.begin(); m != m.end(); m++) {
117  HomePatch *p = homePatch(m->pid);
118  patchMap->unregisterPatch(m->pid, p);
119 
120  MovePatchesMsg *msg = new MovePatchesMsg(m->pid, p->atom);
121 
122  // Deleting the HomePatchElem will call a destructor for clean up
123  // but the msg elements are safe since they use a container template
124  // that uses ref counting.
125  delete p;
126  homePatches.del(HomePatchElem(m->pid));
127 
128  if ( msg->atom.shared() ) NAMD_bug("shared message array in PatchMgr::sendMovePatches");
129 
130  // Sending to PatchMgr::recvMovePatches on remote node
131  CProxy_PatchMgr cp(thisgroup);
132  cp[m->nodeID].recvMovePatches(msg);
133  }
134  move.resize(0);
135 }
136 
138  // Make a new HomePatch
139  createHomePatch(msg->pid, msg->atom);
140  delete msg;
141 
142  // Tell sending PatchMgr we received MovePatchMsg
143 // AckMovePatchesMsg *ackmsg =
144 // new AckMovePatchesMsg;
145 // CSendMsgBranch(PatchMgr,ackMovePatches, ackmsg, thisgroup, msg->fromNodeID);
146 }
147 
148 
149 //void PatchMgr::ackMovePatches(AckMovePatchesMsg *msg)
150 //{
151 // delete msg;
152 // if (! --ackMovePending)
153 // WorkDistrib::messageMovePatchDone();
154 //}
155 
156 
158 
159  MovePatchesMsg *msg = new MovePatchesMsg(pid, a);
160 
161  if ( msg->atom.shared() ) NAMD_bug("shared message array in PatchMgr::sendAtoms");
162 
163  CProxy_PatchMgr cp(thisgroup);
164  cp[patchMap->node(pid)].recvAtoms(msg);
165 
166 }
167 
169  patchMap->homePatch(msg->pid)->reinitAtoms(msg->atom);
170  delete msg;
171 }
172 
173 // Called by HomePatch to migrate atoms off to new patches
174 // Message combining occurs here
176 /*
177  for (int i=0; i < numMsgs; i++) {
178  PatchMgr::Object()->sendMigrationMsg(src, m[i]);
179  }
180 */
181  if ( ! migrationCountdown ) // (re)initialize
182  {
183  // DebugM(3,"migrationCountdown (re)initialize\n");
184  numHomePatches = patchMap->numHomePatches();
185  migrationCountdown = numHomePatches;
186  combineMigrationDestPes.resize(0);
187  }
188  for (int i=0; i < numMsgs; i++) { // buffer messages
189  int destNodeID = m[i].destNodeID;
190  if ( 1 ) // destNodeID != CkMyPe() )
191  {
192  if ( ! combineMigrationMsgs[destNodeID] )
193  {
194  combineMigrationMsgs[destNodeID] = new MigrateAtomsCombinedMsg();
195  combineMigrationDestPes.add(destNodeID);
196  }
197  combineMigrationMsgs[destNodeID]->add(src,m[i].destPatchID,m[i].mList);
198  }
199  else
200  {
201  // for now buffer local messages too
202  }
203  }
204  migrationCountdown -= 1;
205  // DebugM(3,"migrationCountdown = " << migrationCountdown << "\n");
206  if ( ! migrationCountdown ) // send out combined messages
207  {
208  int n = combineMigrationDestPes.size();
209  for ( int i = 0; i < n; ++i ) {
210  int destNodeID = combineMigrationDestPes[i];
211  DebugM(3,"Sending MigrateAtomsCombinedMsg to node " << destNodeID << "\n");
212  CProxy_PatchMgr cp(thisgroup);
213  cp[destNodeID].recvMigrateAtomsCombined(combineMigrationMsgs[destNodeID]);
214  combineMigrationMsgs[destNodeID] = 0;
215  }
216  }
217 }
218 
220 {
221  DebugM(3,"Received MigrateAtomsCombinedMsg with " << msg->srcPatchID.size() << " messages.\n");
222  msg->distribute();
223  delete msg;
224 }
225 
227  LocalID lid = AtomMap::Object()->localID(msg->atomid);
228  if ( lid.pid != notUsed ) {
229  HomePatch *hp = patchMap->homePatch(lid.pid);
230  if ( hp ) {
231  FullAtom &a = hp->atom[lid.index];
232  if ( msg->moveto ) {
233  a.fixedPosition = msg->coord;
234  } else {
236  a.fixedPosition += msg->coord;
237  }
239  }
240  }
241  delete msg;
242 }
243 
245  // loop over homePatches, moving every atom
246  for (HomePatchElem *elem = homePatches.begin(); elem != homePatches.end(); elem++) {
247  HomePatch *hp = elem->patch;
248  for (int i=0; i<hp->getNumAtoms(); i++) {
249  FullAtom &a = hp->atom[i];
251  a.fixedPosition += msg->offset;
253  }
254  }
255  delete msg;
256 }
257 
259  // loop over homePatches, setting the lattice to the new value.
260  for (HomePatchElem *elem = homePatches.begin(); elem != homePatches.end(); elem++) {
261  HomePatch *hp = elem->patch;
262  hp->lattice = msg->lattice;
263  }
264  // Must also do this for SimParameters in order for pressure profile to work!
266 }
267 
268 
269 // initiating replica
270 void PatchMgr::sendCheckpointReq(int pid, int remote, const char *key, int task) {
271  CheckpointAtomsReqMsg *msg = new (1+strlen(key),0) CheckpointAtomsReqMsg;
272  msg->pid = pid;
273  msg->pe = CkMyPe();
274  msg->replica = CmiMyPartition();
275  msg->task = task;
276  strcpy(msg->key,key);
277  envelope *env = UsrToEnv(CheckpointAtomsReqMsg::pack(msg));
278  CmiSetHandler(env,recvCheckpointReq_index);
279 #if CMK_HAS_PARTITION
280  CmiInterSyncSendAndFree(CkMyPe(),remote,env->getTotalsize(),(char*)env);
281 #else
282  CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
283 #endif
284 }
285 
286 // responding replica
287 extern "C" {
288  void recvCheckpointReq_handler(envelope *env) {
289  PatchMgr::Object()->recvCheckpointReq(CheckpointAtomsReqMsg::unpack(EnvToUsr(env)));
290  }
291 }
292 
293 // responding replica
295  int patchnode = patchMap->node(msg->pid);
296  if ( CkMyPe() != patchnode ) {
297  thisProxy[patchnode].recvCheckpointReq(msg);
298  } else {
299  HomePatch *hp = patchMap->homePatch(msg->pid);
300  if ( ! hp ) NAMD_bug("null HomePatch pointer in PatchMgr::recvCheckpointReq");
301  hp->recvCheckpointReq(msg->task, msg->key, msg->replica, msg->pe);
302  delete msg;
303  }
304 }
305 
306 
307 // responding replica
308 void PatchMgr::sendCheckpointLoad(CheckpointAtomsMsg *msg, int dst, int dstpe) {
309  envelope *env = UsrToEnv(CheckpointAtomsMsg::pack(msg));
310  CmiSetHandler(env,recvCheckpointLoad_index);
311 #if CMK_HAS_PARTITION
312  CmiInterSyncSendAndFree(dstpe,dst,env->getTotalsize(),(char*)env);
313 #else
314  CmiSyncSendAndFree(dstpe,env->getTotalsize(),(char*)env);
315 #endif
316 }
317 
318 // initiating replica
319 extern "C" {
320  void recvCheckpointLoad_handler(envelope *env) {
321  PatchMgr::Object()->recvCheckpointLoad(CheckpointAtomsMsg::unpack(EnvToUsr(env)));
322  }
323 }
324 
325 // initiating replica
327  HomePatch *hp = patchMap->homePatch(msg->pid);
328  hp->recvCheckpointLoad(msg);
329 }
330 
331 
332 // initiating replica
333 void PatchMgr::sendCheckpointStore(CheckpointAtomsMsg *msg, int dst, int dstpe) {
334  envelope *env = UsrToEnv(CheckpointAtomsMsg::pack(msg));
335  CmiSetHandler(env,recvCheckpointStore_index);
336 #if CMK_HAS_PARTITION
337  CmiInterSyncSendAndFree(dstpe,dst,env->getTotalsize(),(char*)env);
338 #else
339  CmiSyncSendAndFree(dstpe,env->getTotalsize(),(char*)env);
340 #endif
341 }
342 
343 // responding replica
344 extern "C" {
345  void recvCheckpointStore_handler(envelope *env) {
346  PatchMgr::Object()->recvCheckpointStore(CheckpointAtomsMsg::unpack(EnvToUsr(env)));
347  }
348 }
349 
350 // responding replica
352  HomePatch *hp = patchMap->homePatch(msg->pid);
353  hp->recvCheckpointStore(msg);
354 }
355 
356 
357 // responding replica
358 void PatchMgr::sendCheckpointAck(int pid, int dst, int dstpe) {
360  msg->pid = pid;
361  envelope *env = UsrToEnv(CheckpointAtomsReqMsg::pack(msg));
362  CmiSetHandler(env,recvCheckpointAck_index);
363 #if CMK_HAS_PARTITION
364  CmiInterSyncSendAndFree(dstpe,dst,env->getTotalsize(),(char*)env);
365 #else
366  CmiSyncSendAndFree(dstpe,env->getTotalsize(),(char*)env);
367 #endif
368 }
369 
370 // initiating replica
371 extern "C" {
372  void recvCheckpointAck_handler(envelope *env) {
373  PatchMgr::Object()->recvCheckpointAck(CheckpointAtomsReqMsg::unpack(EnvToUsr(env)));
374  }
375 }
376 
377 // initiating replica
379  HomePatch *hp = patchMap->homePatch(msg->pid);
380  if ( ! hp ) NAMD_bug("null HomePatch pointer in PatchMgr::recvCheckpointAck");
381  hp->recvCheckpointAck();
382  delete msg;
383 }
384 
385 
386 void PatchMgr::sendExchangeReq(int pid, int src) {
388  msg->pid = pid;
389  msg->dstpe = CkMyPe();
390  envelope *env = UsrToEnv(ExchangeAtomsReqMsg::pack(msg));
391  CmiSetHandler(env,recvExchangeReq_index);
392 #if CMK_HAS_PARTITION
393  CmiInterSyncSendAndFree(CkMyPe(),src,env->getTotalsize(),(char*)env);
394 #else
395  CmiSyncSendAndFree(CkMyPe(),env->getTotalsize(),(char*)env);
396 #endif
397 }
398 
399 extern "C" {
400  void recvExchangeReq_handler(envelope *env) {
401  PatchMgr::Object()->recvExchangeReq(ExchangeAtomsReqMsg::unpack(EnvToUsr(env)));
402  }
403 }
404 
406  int patchnode = patchMap->node(msg->pid);
407  if ( CkMyPe() != patchnode ) {
408  thisProxy[patchnode].recvExchangeReq(msg);
409  } else {
410  HomePatch *hp = patchMap->homePatch(msg->pid);
411  if ( ! hp ) NAMD_bug("null HomePatch pointer in PatchMgr::recvExchangeReq");
412  hp->recvExchangeReq(msg->dstpe);
413  delete msg;
414  }
415 }
416 
417 void PatchMgr::sendExchangeMsg(ExchangeAtomsMsg *msg, int dst, int dstpe) {
418  envelope *env = UsrToEnv(ExchangeAtomsMsg::pack(msg));
419  CmiSetHandler(env,recvExchangeMsg_index);
420 #if CMK_HAS_PARTITION
421  CmiInterSyncSendAndFree(dstpe,dst,env->getTotalsize(),(char*)env);
422 #else
423  CmiSyncSendAndFree(dstpe,env->getTotalsize(),(char*)env);
424 #endif
425 }
426 
427 extern "C" {
428  void recvExchangeMsg_handler(envelope *env) {
429  PatchMgr::Object()->recvExchangeMsg(ExchangeAtomsMsg::unpack(EnvToUsr(env)));
430  }
431 }
432 
434  HomePatch *hp = patchMap->homePatch(msg->pid);
435  hp->recvExchangeMsg(msg);
436 }
437 
439  PACK(fromNodeID);
440  PACK(pid);
441  PACK_RESIZE(atom);
442 )
443 
444 
445 #include "PatchMgr.def.h"
446 
static Node * Object()
Definition: Node.h:86
void recvCheckpointLoad(CheckpointAtomsMsg *msg)
Definition: HomePatch.C:3516
void recvCheckpointReq_handler(envelope *env)
Definition: PatchMgr.C:288
HomePatch * patch
Definition: HomePatchList.h:23
Position fixedPosition
Definition: NamdTypes.h:102
Lattice & lattice
Definition: Patch.h:126
SimParameters * simParameters
Definition: Node.h:178
void sendOneHomePatch(int patchId, int nodeId)
Definition: PatchMgr.C:89
void recvExchangeReq_handler(envelope *env)
Definition: PatchMgr.C:400
void createHomePatch(PatchID pid, FullAtomList &a)
Definition: PatchMgr.C:74
void sendAtoms(PatchID pid, FullAtomList &a)
Definition: PatchMgr.C:157
void add(PatchID source, PatchID destination, MigrationList &m)
#define DebugM(x, y)
Definition: Debug.h:59
void recvExchangeMsg(ExchangeAtomsMsg *msg)
Definition: PatchMgr.C:433
void sendExchangeReq(int pid, int src)
Definition: PatchMgr.C:386
int atomid
Definition: PatchMgr.h:55
Position position
Definition: NamdTypes.h:53
void recvExchangeReq(ExchangeAtomsReqMsg *msg)
Definition: PatchMgr.C:405
#define PACK_MSG(MSGTYPE, MSGDATA)
Definition: packmsg.h:35
HomePatch * homePatch(PatchID pid)
Definition: PatchMap.h:240
void recvCheckpointReq(int task, const char *key, int replica, int pe)
Definition: HomePatch.C:3486
FullAtomList atom
Definition: PatchMgr.h:38
void movePatch(PatchID, NodeID)
Definition: PatchMgr.C:84
void recvCheckpointAck()
Definition: HomePatch.C:3569
Vector offset
Definition: PatchMgr.h:62
void unregisterPatch(PatchID pid, HomePatch *pptr)
Definition: PatchMap.C:796
void sendCheckpointAck(int pid, int dst, int dstpe)
Definition: PatchMgr.C:358
void recvCheckpointLoad(CheckpointAtomsMsg *msg)
Definition: PatchMgr.C:326
ResizeArrayIter< T > end(void) const
void recvCheckpointStore(CheckpointAtomsMsg *msg)
Definition: PatchMgr.C:351
PatchID pid
Definition: PatchMgr.h:37
void recvAtoms(MovePatchesMsg *msg)
Definition: PatchMgr.C:168
void recvMigrateAtomsCombined(MigrateAtomsCombinedMsg *)
Definition: PatchMgr.C:219
bool shared() const
Definition: ResizeArray.h:72
void NAMD_bug(const char *err_msg)
Definition: common.C:123
int index
Definition: NamdTypes.h:195
iterator end(void)
Definition: ResizeArray.h:37
ResizeArray< PatchID > srcPatchID
void sendMovePatches()
Definition: PatchMgr.C:110
void recvCheckpointLoad_handler(envelope *env)
Definition: PatchMgr.C:320
NodeID destNodeID
Definition: Migration.h:21
LocalID localID(AtomID id)
Definition: AtomMap.h:74
void sendMigrationMsgs(PatchID, MigrationInfo *, int)
Definition: PatchMgr.C:175
int moveto
Definition: PatchMgr.h:56
void setLattice(SetLatticeMsg *msg)
Definition: PatchMgr.C:258
int PatchID
Definition: NamdTypes.h:182
void recvCheckpointAck_handler(envelope *env)
Definition: PatchMgr.C:372
PatchID pid
Definition: NamdTypes.h:194
static AtomMap * Object()
Definition: AtomMap.h:36
int add(const Elem &elem)
Definition: ResizeArray.h:97
void sendCheckpointLoad(CheckpointAtomsMsg *msg, int dst, int dstpe)
Definition: PatchMgr.C:308
void moveAtom(MoveAtomMsg *msg)
Definition: PatchMgr.C:226
#define PACK_RESIZE(DATA)
Definition: packmsg.h:125
void resize(int i)
Definition: ResizeArray.h:84
int node(int pid) const
Definition: PatchMap.h:114
void recvCheckpointAck(CheckpointAtomsReqMsg *msg)
Definition: PatchMgr.C:378
Position apply_transform(Position data, const Transform &t) const
Definition: Lattice.h:132
void recvCheckpointStore_handler(envelope *env)
Definition: PatchMgr.C:345
static void registerPatchMgr(PatchMgr *pmgr)
Definition: PatchMap.h:49
Position reverse_transform(Position data, const Transform &t) const
Definition: Lattice.h:138
int numHomePatches(void)
Definition: PatchMap.C:432
int getNumAtoms()
Definition: Patch.h:105
~PatchMgr()
Definition: PatchMgr.C:64
#define PACK(DATA)
Definition: packmsg.h:123
void del(const Elem &elem)
Definition: SortedArray.h:59
void recvExchangeReq(int req)
Definition: HomePatch.C:3595
void moveAllBy(MoveAllByMsg *msg)
Definition: PatchMgr.C:244
Elem * find(const Elem &elem)
int size(void) const
Definition: ResizeArray.h:127
HomePatch * homePatch(PatchID pid)
Definition: PatchMgr.h:165
PatchMgr()
Definition: PatchMgr.C:34
void registerPatch(PatchID pid, HomePatch *pptr)
Definition: PatchMap.C:786
int load(const Elem &elem)
Definition: SortedArray.h:50
int NodeID
Definition: NamdTypes.h:184
static PatchMap * Instance()
Definition: PatchMap.C:32
void sendCheckpointStore(CheckpointAtomsMsg *msg, int dst, int dstpe)
Definition: PatchMgr.C:333
Lattice lattice
Definition: PatchMgr.h:67
void recvCheckpointReq(CheckpointAtomsReqMsg *msg)
Definition: PatchMgr.C:294
void sendCheckpointReq(int pid, int remote, const char *key, int task)
Definition: PatchMgr.C:270
static PatchMgr * Object()
Definition: PatchMgr.h:152
void recvExchangeMsg_handler(envelope *env)
Definition: PatchMgr.C:428
Vector coord
Definition: PatchMgr.h:57
void sendExchangeMsg(ExchangeAtomsMsg *msg, int dst, int dstpe)
Definition: PatchMgr.C:417
void recvCheckpointStore(CheckpointAtomsMsg *msg)
Definition: HomePatch.C:3555
ResizeArrayIter< T > begin(void) const
void recvExchangeMsg(ExchangeAtomsMsg *msg)
Definition: HomePatch.C:3606
Transform transform
Definition: NamdTypes.h:116
void recvMovePatches(MovePatchesMsg *msg)
Definition: PatchMgr.C:137
iterator begin(void)
Definition: ResizeArray.h:36