NAMD
Sync.C
Go to the documentation of this file.
1 
8 /*
9  Sync will ensure that all homepatches finished updating before Computes starts and all proxies finished updating themselves.
10 */
11 
12 #if !defined(WIN32) || defined(__CYGWIN__)
13 #include <unistd.h>
14 #endif
15 #include <stdio.h>
16 
17 #include "InfoStream.h"
18 #include "Patch.h"
19 #include "PatchMap.h"
20 #include "ProxyMgr.h"
21 #include "Compute.h"
22 #include "ComputeMap.h"
23 
24 #include "Sync.h"
25 
26 #define MIN_DEBUG_LEVEL 3
27 //#define DEBUGM
28 #include "Debug.h"
29 
30 // make sure all HomePatches get their positions data and sendProxyData to
31 // their proxies before computes get positionsReady.
32 
33 // useProxySync will make sure all proxies get updated before computes'
34 // positionsReady triggered so that real computation starts.
35 // when these two combined, it will make sure that homepatch get all its force
36 // and positions data, and proxies receive its updated data before all
37 // computes start.
38 
39 static int eventHoldComputes;
41 
42 Sync::Sync(): INCREASE(600), step(0), counter(0), homeReady(0)
43 {
44  if (CkpvAccess(Sync_instance) == NULL) {
45  CkpvAccess(Sync_instance) = this;
46  } else {
47  NAMD_bug("Sync instanced twice on same processor!");
48  }
49  capacity = INCREASE;
50  clist = new _clist[capacity];
51  cnum = 0;
52  nPatcheReady = 0;
53  numPatches = -1;
54  eventHoldComputes = traceRegisterUserEvent("Sync::holdComputes", 133);
55  eventReleaseComputes = traceRegisterUserEvent("Sync::releaseComputes", 134);
56 }
57 
59 {
60  delete [] clist;
61 }
62 
63 void Sync::openSync(void)
64 {
65  int reportPe = 1;
66  while ( 2 * reportPe < CkNumPes() ) reportPe *= 2;
67  step = -1;
68  useSync = 1;
69  if ( PatchMap::Object()->numPatches() >= 4 * CkNumPes() ) useSync = 0;
70  if ( CmiNumNodes() < 2 ) useSync = 0;
71  if ( CmiNumPhysicalNodes() < 2 ) useSync = 0;
72 #if defined(NAMD_CUDA) || defined(NAMD_MIC)
73  useSync = 0;
74 #endif
75  useProxySync = 0;
76  if (useSync) {
77  // if use proxy spanning tree, proxy sync is forced
78  if (!useProxySync && (proxySendSpanning || proxyRecvSpanning)
79  && PatchMap::Object()->numPatches() < 4 * CkNumPes() ) {
80  // If on BG/P, useProxySync should not be turned on for better performance
81  #if ! (CMK_BLUEGENEQ || CMK_BLUEGENEP)
82  // CmiPrintf("[%d] useProxySync is turned on. \n", CkMyPe());
83  useProxySync = 1;
84  #endif
85  }
86 #if defined(NODEAWARE_PROXY_SPANNINGTREE) && defined(USE_NODEPATCHMGR)
87  // immediate messages can be processed by any PE
88  if (CkMyNodeSize() > 2) useProxySync = 0;
89 #endif
90  // no proxies on this node, no need to use proxy sync.
91  if (useProxySync && ProxyMgr::Object()->numProxies() == 0) {
92  // CmiPrintf("[%d] useProxySync is turned off because no proxy. \n", CkMyPe());
93  useProxySync = 0;
94  }
95  // if no proxy sync and no home patch, then disable home patch sync as well
96  if (!useProxySync && PatchMap::Object()->numHomePatches() == 0) useSync = 0;
97  }
98  if(CkMyPe() == reportPe)
99  iout << iINFO << "useSync: " << useSync << " useProxySync: " << useProxySync << "\n" << endi;
100 }
101 
102 // called from Patch::positionsReady()
103 int Sync::holdComputes(PatchID pid, Compute **cbegin, Compute **cend, int doneMigration, int seq)
104 {
105  if (!useSync) return 0;
106  if (step < 0) step = seq;
107  if (!useProxySync) {
108  // only hold when homepatches are not ready
109  PatchMap *patchMap = PatchMap::Object();
110  if (homeReady && seq == step) {
111  nPatcheReady++;
112  triggerCompute();
113  return 0;
114  }
115  }
116  traceUserEvent(eventHoldComputes);
117 
118  int slot = 0;
119  for (; slot < cnum; slot++)
120  if (clist[slot].pid == -1) break;
121  if (slot == cnum) {
122  cnum++;
123  // table is full, expand the list
124  if (cnum == capacity) {
125  capacity += INCREASE;
126  struct _clist *tmp = new _clist[capacity];
127  memcpy(tmp, clist, cnum*sizeof(_clist));
128  delete [] clist;
129  clist = tmp;
130  //CmiPrintf("[%d] Info:: Sync buffer overflow and expanded!\n", CkMyPe());
131  }
132  }
133 
134  clist[slot].cbegin = cbegin;
135  clist[slot].cend = cend;
136  clist[slot].pid = pid;
137  clist[slot].doneMigration = doneMigration;
138  clist[slot].step = seq;
139 
140 // CkPrintf("REG[%d]: patch:%d step:%d-%d slot:%d\n", CkMyPe(), pid, patchMap->patch(pid)->flags.sequence, step, slot);
141 
142  if (clist[slot].step == step) {
143  nPatcheReady++;
144  triggerCompute();
145  }
146  return 1;
147 }
148 
149 // called from HomePatch::positionsReady()
151 {
152  if ( useSync ) {
153  counter ++;
154  triggerCompute();
155  }
156 }
157 
158 void Sync::releaseComputes()
159 {
160  PatchMap *patchMap = PatchMap::Object();
161 
162  traceUserEvent(eventReleaseComputes);
163 
164  for (int i= 0; i<cnum; i++) {
165  int &pid = clist[i].pid;
166  if (pid == -1) continue;
167  if (clist[i].step != step) {
168  continue;
169  }
170  // CkPrintf(" %d-%d-%d ",
171  // clist[i].pid, clist[i].step,
172  // patchMap->patch(pid)->flags.sequence);
173 
174  Compute **cend = clist[i].cend;
175  for(Compute **cid = clist[i].cbegin; cid != cend; cid++) {
176  (*cid)->patchReady(pid,clist[i].doneMigration,step);
177  }
178  pid = -1;
179  }
180 // CkPrintf("\n");
181 }
182 
183 void Sync::triggerCompute()
184 {
185  PatchMap *patchMap = PatchMap::Object();
186  const int numHomePatches = patchMap->numHomePatches();
187 
188  if (numPatches == -1)
189  numPatches = ProxyMgr::Object()->numProxies() + numHomePatches;
190 
191 // if (CkMyPe()<=8) CkPrintf("SYNC[%d]: PATCHREADY:%d %d patches:%d %d\n", CkMyPe(), counter, numHomePatches, nPatcheReady, numPatches);
192 // CkPrintf("SYNC[%d]: PATCHREADY:%d %d patches:%d %d\n", CkMyPe(), counter, PatchMap::Object()->numHomePatches(), nPatcheReady, numPatches);
193 
194  if (homeReady == 0 && counter >= numHomePatches) {
195  homeReady = 1;
196  // if (CkMyPe()<=8) CkPrintf("HOMEREADY[%d]\n", CkMyPe());
197  if (!useProxySync) releaseComputes();
198  }
199 
200  if (homeReady && nPatcheReady == numPatches)
201  {
202 // if (CkMyPe()<=8) CkPrintf("TRIGGERED[%d]\n", CkMyPe());
203 // CkPrintf("TRIGGERED[%d]\n", CkMyPe());
204  if (useProxySync) releaseComputes();
205 
206  // reset counter
207  numPatches = -1;
208  step++;
209  nPatcheReady = 0;
210  for (int i= 0; i<cnum; i++) {
211  if (clist[i].pid != -1 && clist[i].step == step) ++nPatcheReady;
212  }
213  homeReady = 0;
214  if ( numHomePatches ) {
215  counter -= numHomePatches;
216  if (counter >= numHomePatches) triggerCompute();
217  }
218  }
219 }
220 
221 
222 #if 0
223 // hack for SUN MPCC
224 // force compiler to instantiate ArrayElementT
225 void *frightenCompilerIntoInstantiatingHack(void) {
226  return new ArrayElementT<int>;
227 }
228 #endif
229 
230 
231 #include "Sync.def.h"
std::ostream & iINFO(std::ostream &s)
Definition: InfoStream.C:107
int proxyRecvSpanning
Definition: ProxyMgr.C:46
static ProxyMgr * Object()
Definition: ProxyMgr.h:394
static PatchMap * Object()
Definition: PatchMap.h:27
__global__ void const int const TileList *__restrict__ TileExcl *__restrict__ const int *__restrict__ const int const float2 *__restrict__ const int *__restrict__ const float3 const float3 const float3 const float4 *__restrict__ const float cudaTextureObject_t cudaTextureObject_t cudaTextureObject_t const int const float const PatchPairRecord *__restrict__ const int *__restrict__ const int2 *__restrict__ const unsigned int *__restrict__ unsigned int *__restrict__ int *__restrict__ int *__restrict__ TileListStat *__restrict__ const BoundingBox *__restrict__ float4 *__restrict__ float4 *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ float *__restrict__ const int numPatches
Sync(void)
Definition: Sync.C:42
#define iout
Definition: InfoStream.h:87
~Sync(void)
Definition: Sync.C:58
void openSync()
Definition: Sync.C:63
static int eventReleaseComputes
Definition: Sync.C:40
void NAMD_bug(const char *err_msg)
Definition: common.C:123
int PatchID
Definition: NamdTypes.h:182
void PatchReady(void)
Definition: Sync.C:150
int numHomePatches(void)
Definition: PatchMap.C:432
static int eventHoldComputes
Definition: Sync.C:39
int holdComputes(PatchID pid, Compute **cbegin, Compute **cend, int doneMigration, int seq)
Definition: Sync.C:103
infostream & endi(infostream &s)
Definition: InfoStream.C:38
int numProxies()
Definition: ProxyMgr.h:396
int proxySendSpanning
Definition: ProxyMgr.C:45