NAMD
ProxyMgr.h
Go to the documentation of this file.
1 
7 #ifndef PROXYMGR_H
8 #define PROXYMGR_H
9 
10 
11 #include "charm++.h"
12 #include "envelope.h"
13 
14 #include "main.h"
15 #include "NamdTypes.h"
16 #include "PatchTypes.h"
17 #include "UniqueSet.h"
18 #include "UniqueSetIter.h"
19 #include "ProcessorPrivate.h"
20 #include "ProxyMgr.decl.h"
21 
23 extern int proxySpanDim;
24 extern int inNodeProxySpanDim;
25 
26 #if CMK_PERSISTENT_COMM
27 #define USE_PERSISTENT_TREE 0
28 #endif
29 
30 class ProxyGBISP1ResultMsg: public CMessage_ProxyGBISP1ResultMsg {
31  public:
32  int destPe;
33  int origPe;
36  int psiSumLen;// = numAtoms
37 };
38 class ProxyGBISP2DataMsg: public CMessage_ProxyGBISP2DataMsg {
39 public:
40  int origPe;
41  int destPe;
43  Real *bornRad;//numAtoms
45  };
46 class ProxyGBISP2ResultMsg: public CMessage_ProxyGBISP2ResultMsg {
47 public:
48  int destPe;
49  int origPe;
52  int dEdaSumLen;//numAtoms
53  };
54 class ProxyGBISP3DataMsg: public CMessage_ProxyGBISP3DataMsg {
55 public:
56  int origPe;
57  int destPe;
59  Real *dHdrPrefix;//numAtoms
61  };
62 
63 class RegisterProxyMsg : public CMessage_RegisterProxyMsg {
64 public:
67 };
68 
69 class UnregisterProxyMsg : public CMessage_UnregisterProxyMsg {
70 public:
73 };
74 
75 //1. This class represents for both msg types: one that
76 //is originally known as ProxyAllMsg which is sent
77 //at the step where atoms migrate; and the other is
78 //sent during the steps between two migrations.
79 //2. In the case of memory optimized version, the scenario
80 //becomes tricky as load balancer will move compute objects
81 //around so that new ProxyPatches will be created where
82 //the CompAtomExt list information is not available. If
83 //the step immediately after the load balancing is a normal
84 //step, then the CompAtomExt list info has to be resent by
85 //the HomePatch. Because of the current Proxy msg communication
86 //scheme where msg is sent to ProxyMgr first, and then retransmitted
87 //to ProxyPatches, there's overhead when we want to resend CompAtomExt
88 //list as not all the ProxyPatches that are managed by ProxyMgr are
89 //newly created ProxyPatches.
90 //--Chao Mei
91 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
92 class ProxyDataMsg_not_32_byte : public CMessage_ProxyDataMsg {
93 #else
94 class ProxyDataMsg : public CMessage_ProxyDataMsg {
95 #endif
96 public:
99 
100  int plLen;
101 
103  int avgPlLen;
105  // BEGIN LA
106  int vlLen;
108  // END LA
109 
110  Real *intRadList;// gbis atom intrinsic radii
111 
112  int *lcpoTypeList;// LCPO atom type
113 
114  //1. The following field will be only
115  //useful for memory optimized version.
116  //2. In normal case, adding this field only
117  //increases the msg length by 4 bytes which
118  //can be ignored considering the current fast
119  //communication network
120  //--Chao Mei
121  int plExtLen;
124 
125 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR) && (CMK_SMP) && defined(NAMDSRC_IMMQD_HACK)
126  //In smp layer, the couter for msg creation and process of communication
127  //thread is not included in the quiescence detection process. In addition,
128  //the immediate messages from other nodes are executed on the communication
129  //thread. If inside the process of immediate messages, some normal Charm++
130  //messages sent out which will be processed on worker threads. Then QD will
131  //be a problem that the process of the normal messages sent from communication
132  //thread is recorded, but the creation of such messages (although recorded
133  //in the comm thread) is virtually not recorded, i.e., not visible the
134  //QD process. So we need to artificially increase the QD counter to
135  //compensate for aforementioned msg creation loss.
136  //The idea is to use the following variable to indicate the normal message
137  //is sent from the communication thread inside a processing of immediate
138  //message. If the variable is set, then we should increase the QD counter.
139  //Chao Mei
140  char isFromImmMsgCall; //hack for imm msg with QD in SMP
141 #endif
142 
143  // DMK - Atom Separation (water vs. non-water)
144  #if NAMD_SeparateWaters != 0
145  int numWaterAtoms; // Number of atoms in positionList (from start)
146  // that are part of water hydrogen groups.
147  #endif
148 
149 };
150 
151 #ifdef REMOVE_PROXYDATAMSG_EXTRACOPY
152 class ProxyDataMsg : public ProxyDataMsg_not_32_byte {
153  // Adding padding bytes to make sure that positionList is
154  // 32-byte aligned which usually gives better cache performance.
155  char padding[(32-(sizeof(envelope)+sizeof(ProxyDataMsg_not_32_byte))%32)%32];
156 };
157 class assert_ProxyDataMsg {
158  char assert_sizeof_envelope_is_multiple_of_ptr[(sizeof(envelope)%sizeof(void*))?-1:1];
159  char assert_sizeof_ProxyDataMsg_is_multiple_of_32[((sizeof(envelope)+sizeof(ProxyDataMsg))%32)?-1:1];
160 };
161 #endif
162 
163 
164 class ProxyResultMsg : public CMessage_ProxyResultMsg {
165 public:
169  static void* pack(ProxyResultMsg *msg);
170  static ProxyResultMsg* unpack(void *ptr);
171 private:
172  ForceList forceListInternal[Results::maxNumForces];
173 };
174 
175 class ProxyResultVarsizeMsg: public CMessage_ProxyResultVarsizeMsg{
176 public:
180 
182  //Indicate the position of the force list that has zero value
183  //which is not recorded in the above force array.
184  char *isZero;
185 
186  //add padding bytes to make sure the beginning
187  //of force arrays is 8-byte aligned as it is originally.
188  //Therefore, we have to put the forceArr field as
189  //the first variable of varsize array type
190  char padding[(8-(sizeof(envelope)+sizeof(NodeID)+sizeof(PatchID)+sizeof(int)*Results::maxNumForces+2*sizeof(void *))%8)%8];
191 
192  //The length of "fls" is Results::maxNumForces
193  static ProxyResultVarsizeMsg *getANewMsg(NodeID nid, PatchID pid, int prioSize, ForceList *fls);
194 };
195 
197  char assert_sizeof_envelope_is_multiple_of_ptr[(sizeof(envelope)%sizeof(void*))?-1:1];
198  char assert_sizeof_ProxyResultVarsizeMsg_is_multiple_of_8[((sizeof(envelope)+sizeof(ProxyResultVarsizeMsg))%8)?-1:1];
199 };
200 
201 class ProxyNodeAwareSpanningTreeMsg: public CMessage_ProxyNodeAwareSpanningTreeMsg{
202 public:
207  int *allPes;
208 
209  static ProxyNodeAwareSpanningTreeMsg *getANewMsg(PatchID pid, NodeID nid, proxyTreeNode *tree, int size);
210 
211  //For debug
212  void printOut(char *tag);
213 };
214 
215 class ProxyCombinedResultRawMsg : public CMessage_ProxyCombinedResultRawMsg {
216 public:
217  int nodeSize;
219 
220  #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
221  //since this msg may be processed by comm thread in the smp mode,
222  //this variable helps comm thread to find which proc will actually process it.
223  NodeID destPe;
224  #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
225  //Mainly for QD in the presence of the optimization of using immediate
226  //message. Refer to the explanation from ProxyDataMsg for the same
227  //variable. --Chao Mei
228  char isFromImmMsgCall;
229  #endif
230  #endif
232 
235  //The beginning address of this variable should be 8-byte aligned!!! -Chao Mei
237 };
238 
239 class ProxyCombinedResultMsg : public CMessage_ProxyCombinedResultMsg {
240 public:
241  #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
242  //since this msg may be processed by comm thread in the smp mode,
243  //this variable helps comm thread to find which proc will actually process it.
244  NodeID destPe;
245  #if CMK_SMP && defined(NAMDSRC_IMMQD_HACK)
246  //Mainly for QD in the presence of the optimization of using immediate
247  //message. Refer to the explanation from ProxyDataMsg for the same
248  //variable. --Chao Mei
249  char isFromImmMsgCall;
250  #endif
251  #endif
257 private:
258  ForceList forceListInternal[Results::maxNumForces];
259 };
260 
261 class ProxySpanningTreeMsg : public CMessage_ProxySpanningTreeMsg {
262 public:
266  static void* pack(ProxySpanningTreeMsg *msg);
267  static ProxySpanningTreeMsg* unpack(void *ptr);
268 };
269 
270 class ProxyPatch;
271 class PatchMap;
272 
273 struct ProxyElem {
274  ProxyElem() : proxyPatch(0) { };
275  ProxyElem(PatchID pid) : patchID(pid), proxyPatch(0) { };
276  ProxyElem(PatchID pid, ProxyPatch *p) : patchID(pid), proxyPatch(p) { };
277 
278  int hash() const { return patchID; }
279  int operator==(const ProxyElem & pe) const { return patchID == pe.patchID; }
280 
283 };
284 
287 
288 class ProxyTree { // keep track of the spanning trees
289  public:
292 #ifdef NODEAWARE_PROXY_SPANNINGTREE
293  //a node-aware spanning tree array, each element of which
294  //is a spanning tree for all proxies of a patch
295  proxyTreeNodeList *naTrees;
296 #else
298  int *sizes;
299 #endif
300 
301  public:
303  proxyMsgCount = 0;
304  proxylist = NULL;
305 #ifdef NODEAWARE_PROXY_SPANNINGTREE
306  naTrees = NULL;
307 #else
308  trees = NULL;
309  sizes = NULL;
310 #endif
311  }
313  }
314 };
315 
316 class ProxyMgr : public CBase_ProxyMgr
317 {
318 public:
319  ProxyMgr();
320  ~ProxyMgr();
321 
322  void removeProxies(void);
323  void removeUnusedProxies(void);
324  void createProxies(void);
325 
326  void createProxy(PatchID pid);
327  void removeProxy(PatchID pid);
328 
329  void registerProxy(PatchID pid);
330  void recvRegisterProxy(RegisterProxyMsg *);
331 
332  void unregisterProxy(PatchID pid);
333  void recvUnregisterProxy(UnregisterProxyMsg *);
334 
335  void setSendSpanning();
336  int getSendSpanning();
337 
338  void setRecvSpanning();
339  int getRecvSpanning();
340 
341  void setProxyTreeBranchFactor(int dim);
342 
343  void buildProxySpanningTree();
344  void sendSpanningTrees();
345  void sendSpanningTreeToHomePatch(int pid, int *tree, int n);
346  void recvSpanningTreeOnHomePatch(int pid, int *tree, int n);
347  void sendSpanningTree(ProxySpanningTreeMsg *);
348  void recvSpanningTree(ProxySpanningTreeMsg *);
349 
350  void sendNodeAwareSpanningTreeToHomePatch(int pid, proxyTreeNode *tree, int n);
351  void recvNodeAwareSpanningTreeOnHomePatch(ProxyNodeAwareSpanningTreeMsg *msg);
352  void sendNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
353  void recvNodeAwareSpanningTree(ProxyNodeAwareSpanningTreeMsg *);
354  //set the proxy patch's parent field
355  void recvNodeAwareSTParent(int patch, int parent);
356 
357  void buildProxySpanningTree2(); // centralized version
358  void sendProxies(int pid, int *list, int n);
359  void recvProxies(int pid, int *list, int n);
360  void recvPatchProxyInfo(PatchProxyListMsg *msg);
361 
362 #ifdef NODEAWARE_PROXY_SPANNINGTREE
363  void buildNodeAwareSpanningTree0();
364  static void buildSinglePatchNodeAwareSpanningTree(PatchID pid, NodeIDList &proxyList,
365  proxyTreeNodeList &ptnTree);
366 #else
367  void buildSpanningTree0();
368 #endif
369 
370  void sendResults(ProxyResultVarsizeMsg *);
371  void recvResults(ProxyResultVarsizeMsg *);
372  void sendResults(ProxyResultMsg *);
373  void recvResults(ProxyResultMsg *);
374  void sendResults(ProxyCombinedResultMsg *);
375 
376  void sendResult(ProxyGBISP1ResultMsg *);//psiSum
377  void recvResult(ProxyGBISP1ResultMsg *);
378  void recvData( ProxyGBISP2DataMsg *);
379  void sendResult(ProxyGBISP2ResultMsg *);//dEdaSum
380  void recvResult(ProxyGBISP2ResultMsg *);
381  void recvData( ProxyGBISP3DataMsg *);
382 
383  void recvResults(ProxyCombinedResultRawMsg *);
384  void recvImmediateResults(ProxyCombinedResultRawMsg *);
385 
386  void sendProxyData(ProxyDataMsg *, int, int*);
387  void recvImmediateProxyData(ProxyDataMsg *);
388  void recvProxyData(ProxyDataMsg *);
389 
390  void sendProxyAll(ProxyDataMsg *, int, int*);
391  void recvImmediateProxyAll(ProxyDataMsg *);
392  void recvProxyAll(ProxyDataMsg *);
393 
394  static ProxyMgr *Object() { return CkpvAccess(ProxyMgr_instance); }
395 
396  int numProxies() { return proxySet.size(); }
397 
398  static int nodecount;
399  ProxyTree &getPtree();
400 
401 private:
402  ProxySet proxySet;
403  ProxyTree ptree;
404 
405  void printProxySpanningTree();
406 };
407 
409  int patchID;
411  int *proxyList; //record which PE the proxy is on
412 };
413 
414 class PatchProxyListMsg: public CMessage_PatchProxyListMsg {
415 public:
417  int *patchIDs;
419  int *proxyPEs;
420 
421 public:
422  PatchProxyListMsg(int num) { numPatches = num; }
423  static PatchProxyListMsg *createPatchProxyListMsg(PatchProxyListMsg **bufs, int bufSize, ProxyListInfo *info, int size);
424 };
425 
426 class NodeProxyMgr : public CBase_NodeProxyMgr
427 {
428 private:
429 /*The following vars are for node-aware spanning tree if NodeProxyMgr is used*/
430  //Potential TODO: change it to a hashtable if necessary
431  proxyTreeNode **proxyInfo;
432  int numPatches;
433 
434  CkGroupID localProxyMgr; //a charm Group variable
435  PatchMap **localPatchMaps;
436 
437 /* The following vars are for managing sending proxy list of a patch to PE 0*/
438  int parentNode; //-1 for root node
439  int numKidNodes;
440  int kidRecved;
441  //about local home patches
442  int numHomePatches; //the number of homepatches on this node
443  int homepatchRecved;
444  ProxyListInfo *localProxyLists;
445  PatchProxyListMsg **remoteProxyLists;
446  CmiNodeLock localDepositLock;
447  CmiNodeLock remoteDepositLock;
448 
449 public:
451  proxyInfo = NULL;
452  numPatches = 0;
453  localPatchMaps = new PatchMap *[CkMyNodeSize()];
454 
455  parentNode = -1;
456  numKidNodes = 0;
457  kidRecved = 0;
458  numHomePatches = 0;
459  homepatchRecved = 0;
460  localProxyLists = NULL;
461  remoteProxyLists = NULL;
462  localDepositLock = CmiCreateLock();
463  remoteDepositLock = CmiCreateLock();
464  }
466  for(int i=0; i<numPatches; i++) {
467  delete proxyInfo[i];
468  }
469  delete [] proxyInfo;
470  delete [] localPatchMaps;
471 
472  CmiDestroyLock(localDepositLock);
473  CmiDestroyLock(remoteDepositLock);
474  }
475 
476  void createProxyInfo(int numPs){
477  numPatches = numPs;
478  proxyInfo = new proxyTreeNode *[numPs];
479  memset(proxyInfo, 0, sizeof(proxyTreeNode *)*numPs);
480  }
481  void registerPatch(int patchID, int numPes, int *pes);
483  return proxyInfo[patchID];
484  }
485 
486  void registerLocalProxyMgr(CkGroupID one){
487  localProxyMgr = one;
488  }
489  const CkGroupID &getLocalProxyMgr(){
490  return localProxyMgr;
491  }
492  void registerLocalPatchMap(int rank, PatchMap *one){
493  localPatchMaps[rank] = one;
494  }
496  return localPatchMaps[rank];
497  }
498 
499  void recvImmediateProxyData(ProxyDataMsg *msg);
500  void recvImmediateProxyAll(ProxyDataMsg *msg);
501  void recvImmediateResults(ProxyCombinedResultRawMsg *);
502 
503  //initialize the spanning tree of home patches
504  void createSTForHomePatches(PatchMap *pmap);
505  //direct call from local home patches
506  void sendProxyList(int pid, int *plist, int size);
507  //remote call to send this node's proxy list info
508  void sendProxyListInfo(PatchProxyListMsg *msg);
509  void contributeToParent();
510 };
511 
512 #endif /* PATCHMGR_H */
513 
void registerLocalProxyMgr(CkGroupID one)
Definition: ProxyMgr.h:486
ProxyTree()
Definition: ProxyMgr.h:302
int * lcpoTypeList
Definition: ProxyMgr.h:112
int operator==(const ProxyElem &pe) const
Definition: ProxyMgr.h:279
int proxyRecvSpanning
Definition: ProxyMgr.C:46
static ProxyMgr * Object()
Definition: ProxyMgr.h:394
proxyTreeNode * getPatchProxyInfo(int patchID)
Definition: ProxyMgr.h:482
__global__ void const int const TileList *__restrict__ TileExcl *__restrict__ const int *__restrict__ const int const float2 *__restrict__ const int *__restrict__ const float3 const float3 const float3 const float4 *__restrict__ const float cudaTextureObject_t cudaTextureObject_t cudaTextureObject_t const int const float const PatchPairRecord *__restrict__ const int *__restrict__ const int2 *__restrict__ const unsigned int *__restrict__ unsigned int *__restrict__ int *__restrict__ int *__restrict__ TileListStat *__restrict__ const BoundingBox *__restrict__ float4 *__restrict__ float4 *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ const int numPatches
Definition: Vector.h:64
static __thread unsigned int * plist
float Real
Definition: common.h:107
ProxyElem(PatchID pid)
Definition: ProxyMgr.h:275
CompAtom * avgPositionList
Definition: ProxyMgr.h:104
int numProxies
Definition: ProxyMgr.h:410
CompAtom * velocityList
Definition: ProxyMgr.h:107
void registerLocalPatchMap(int rank, PatchMap *one)
Definition: ProxyMgr.h:492
UniqueSet< ProxyElem > ProxySet
Definition: ProxyMgr.h:285
void recvData(DataMessage *dmsg)
Definition: DataExchanger.C:93
ProxyElem()
Definition: ProxyMgr.h:274
NodeIDList tree
Definition: ProxyMgr.h:265
const CkGroupID & getLocalProxyMgr()
Definition: ProxyMgr.h:489
void createProxyInfo(int numPs)
Definition: ProxyMgr.h:476
GBReal * dEdaSum
Definition: ProxyMgr.h:51
CudaAtom * cudaAtomList
Definition: ProxyMgr.h:123
ProxyPatch * proxyPatch
Definition: ProxyMgr.h:282
int hash() const
Definition: ProxyMgr.h:278
PatchID patch
Definition: ProxyMgr.h:66
int * sizes
Definition: ProxyMgr.h:298
int inNodeProxySpanDim
Definition: ProxyMgr.C:49
int * proxyListLen
Definition: ProxyMgr.h:418
UniqueSetIter< ProxyElem > ProxySetIter
Definition: ProxyMgr.h:286
PatchID patch
Definition: ProxyMgr.h:167
int plExtLen
Definition: ProxyMgr.h:121
NodeIDList * trees
Definition: ProxyMgr.h:297
ProxyElem(PatchID pid, ProxyPatch *p)
Definition: ProxyMgr.h:276
int PatchID
Definition: NamdTypes.h:182
PatchID patch
Definition: ProxyMgr.h:97
Flags flags
Definition: ProxyMgr.h:98
Real * intRadList
Definition: ProxyMgr.h:110
CompAtom * positionList
Definition: ProxyMgr.h:102
~ProxyTree()
Definition: ProxyMgr.h:312
PatchID patchID
Definition: ProxyMgr.h:281
NodeID node
Definition: ProxyMgr.h:166
NodeIDList * proxylist
Definition: ProxyMgr.h:291
Real * dHdrPrefix
Definition: ProxyMgr.h:59
int proxyMsgCount
Definition: ProxyMgr.h:290
CompAtomExt * positionExtList
Definition: ProxyMgr.h:122
static int nodecount
Definition: ProxyMgr.h:398
int avgPlLen
Definition: ProxyMgr.h:103
int proxySpanDim
Definition: ProxyMgr.C:48
int * proxyList
Definition: ProxyMgr.h:411
int NodeID
Definition: NamdTypes.h:184
int numProxies()
Definition: ProxyMgr.h:396
PatchProxyListMsg(int num)
Definition: ProxyMgr.h:422
float GBReal
Definition: ComputeGBIS.inl:17
PatchMap * getLocalPatchMap(int rank)
Definition: ProxyMgr.h:495
int proxySendSpanning
Definition: ProxyMgr.C:45