NAMD
ComputeMoa.C
Go to the documentation of this file.
1 /* *
2  * *** Copyright (c) 1995, 1996, 1997, 1998, 1999, 2000 by
3  * *** The Board of Trustees of the University of Illinois.
4  * *** All rights reserved.
5  * **
6  * =====================================================================================
7  *
8  * Filename: ComputeMoa.C
9  *
10  * Description:
11  *
12  * Version: Stub File
13  * Created: 03/08/2012 03:55:27 PM
14  * Revision:
15  * Compiler: charm++
16  *
17  * Author: Christopher B Harrison, Ph.D.
18  * Email: charris5@gmail.com, char1@illinois.edu, char@ks.uiuc.edu
19  * Company:
20  *
21  * =====================================================================================
22  */
23 
24 #include "InfoStream.h"
25 #include "Node.h"
26 #include "PDB.h"
27 #include "PatchMap.h"
28 #include "PatchMap.inl"
29 #include "ComputeMoa.h"
30 #include "ComputeMoaMgr.decl.h"
31 #include "PatchMgr.h"
32 #include "Molecule.h"
33 #include "ReductionMgr.h"
34 #include "ComputeMgr.h"
35 #include "ComputeMgr.decl.h"
36 // #define DEBUGM
37 #define MIN_DEBUG_LEVEL 3
38 #include "Debug.h"
39 #include "SimParameters.h"
40 #include "WorkDistrib.h"
41 #include "varsizemsg.h"
42 #include <stdlib.h>
43 #include <stdio.h>
44 #include <errno.h>
45 #include "pup_stl.h"
46 
47 #ifdef CHARM_HAS_MSA
48 
49 #ifdef OPENATOM_VERSION
50 
51 
52 void MoaData::pup(PUP::er &p)
53 {
54  p|K1, p|K2, p|K3;
55  p|order;
56  p|orig_x, p|orig_y, p|orig_z;
57 
58  p|num_clients;
59  p|num_clients_qh, p|num_clients_bh;
60 
61  p|qh;
62  p|sh;
63  p|bh;
64 
65  p|k1r, p|k2r, p|k3r;
66  p|gbqs;
67  p|hasLQ;
68  p|hasLB;
69  p|hasLS;
70 }
71 
72 void MoaData::print()
73 {
74 #if 0
75  printf("MoaData:\n");
76  printf("K1,K2,K3 = %d, %d, %d\n", K1, K2, K3);
77  printf("order = %d\n", order);
78  printf("origin = %g %g %g\n", orig_x, orig_y, orig_z);
79 #endif
80 }
81 
82 class ComputeMoaMgr : public CBase_ComputeMoaMgr {
83 public:
84  ComputeMoaMgr(); // entry
85  ~ComputeMoaMgr();
86 
87  CkCallback doneMoa;
88 
89  void initialize(CkQdMsg *); // entry with message
90 
91  void setCompute(ComputeMoa *c) { moaCompute = c; } // local
92 
93  void recvMoaData(const MoaData &); // entry with parameter marshalling
94 
95  void initWorkers(CkQdMsg *); // entry with message
96  void startWorkers(CkQdMsg *); // entry with message
97 
98  void recvQ(int g, int numgrids, int nq, float qgrid[nq], CkCallback doneMoa);
99 
100  void recvB(int K2_start, int K2_end, int K3_start, int K3_end, int K1_len, int K2_len, int K3_len, double bm1[K1_len], double bm2[K2_len], double bm3[K3_len], int order); // entry with message
101 
102  const MoaData &getMoaData() const { return moaData; } // local
103 
104 private:
105  CProxy_ComputeMoaMgr moaProxy;
106  ComputeMoa *moaCompute;
107 
108  MoaData moaData;
109 
110  CProxy_MoaPatch moaPatchProxy; // 1D chare array
111  CProxy_MoaS moaSProxy; // 1D chare array
112 
113  Moa3Grid::Write qhwrit // write charge grid
114  Moa3Grid::Accum qhacc; // accumulate charge grid
115  Moa3Grid::Read qhread; // read charge grid
116  Moa3Grid::Write shwrit; // writes S grid
117  Moa3Grid::Accum shacc; // accumulate S grid
118  Moa3Grid::Read shread; // read S grid
119  Moa3Grid::Write bhwrit; // writes b1, b2, b3 values
120  Moa1Grid::Read bhread; // read b1, b2, b3 values
121 };
122 
123 // class MoaPatch : public CBase_MoaPatch {
124 // public:
125 // MoaPatch(Moa3Grid &qh, Moa3Grid &eh);
126 // MoaPatch(CkMigrateMessage *m) { }
127 // void compute();
128 // private:
129 // };
130 
131 class MoaS : public CBase_MoaS {
132 public:
133  MoaS(Moa3Grid &q, Moa3Grid &b);
134  MoaS(CkMigrateMessage *m) { }
135  ~MoaS();
136  void compute();
137 private:
138  Moa3Grid qh, bh;
139  Moa3Grid::Write shwrite;
140  Moa3Grid::Read qhread, bhread;
141  int moaS_setup;
142  int lia, lja, lka; // local indices of sub-lattice
143  int lib, ljb, lkb; // local indices of sub-lattice
144  SubmitReduction *reduction;
145 };
146 
147 
148 ComputeMoa::ComputeMoa(ComputeID c) :
149  ComputeHomePatches(c)
150 {
151 #ifdef OPENATOM_VERSION_DEBUG
152  CkPrintf("Instantiated ComputeMoa %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
153 #endif //OPENATOM_VERSION_DEBUG
154 
155  CProxy_ComputeMoaMgr::ckLocalBranch(
156  CkpvAccess(BOCclass_group).computeMoaMgr)->setCompute(this);
157  SimParameters *simParams = Node::Object()->simParameters;
158  // reduction = ReductionMgr::Object()->willSubmit(REDUCTIONS_BASIC);
159 }
160 
161 ComputeMoa::~ComputeMoa()
162 {
163 }
164 
165 void ComputeMoa::doWork()
166 {
167 #ifdef OPENATOM_VERSION_DEBUG
168  CkPrintf("Starting ComputeMoa::doWork() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
169 #endif //OPENATOM_VERSION_DEBUG
170 
171  // allocate message
172 
173  CProxy_ComputeMoaMgr moaProxy(CkpvAccess(BOCclass_group).computeMoaMgr);
174  moaS_Proxy[CkMyPe()].compute(new CkQdMsg);
175 
176 }
177 
178 ComputeMoaMgr::ComputeMoaMgr() :
179  moaProxy(thisgroup), moaCompute(0)
180 {
181 #ifdef OPENATOM_VERSION_DEBUG
182  CkPrintf("Instantiating ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
183 #endif //OPENATOM_VERSION_DEBUG
184  CkpvAccess(BOCclass_group).computeMoaMgr = thisgroup;
185 }
186 
187 ComputeMoaMgr::~ComputeMoaMgr()
188 {
189 #ifdef OPENATOM_VERSION_DEBUG
190  CkPrintf("Destructing ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
191 #endif //OPENATOM_VERSION_DEBUG
192 
193 }
194 
195 
196 void ComputeMoaMgr::initialize(CkQdMsg *msg)
197 {
198 #ifdef OPENATOM_VERSION_DEBUG
199  CkPrintf("Initializing ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
200 #endif //OPENATOM_VERSION_DEBUG
201 
202  delete msg;
203  if (CkMyPe() != 0) return; // initialize only on PE 0, broadcast MoaData
204 
205  // initialize MOA here
206  SimParameters *simParams = Node::Object()->simParameters;
207  Molecule *mol = Node::Object()->molecule;
208 
209  // setup grids and vectors
210  MoaData &p = moaData;
211  // p.hasLQ=0;
212  // p.hasLB=0;
213 
214  extern int isPmeProcessor(int);
215  int i, num_clients=0;
216  for (i=0;i<CkNumPes(), i++)
217  {
218  if (isPmeProcessor(i)) p.num_clients++;
219  }
220 
221  // RESTORE VDW PARAMETERS
222 
223  // RESTORE MOLECULE OBJECT PARAMETERS fepAtomFlag atoms vdw 0 and charge bCol
224 
225  int done, alldone;
226 
227 
228 #ifdef MOA_DEBUG
229  CkPrintf("global num clients = %d\n", p.num_clients);
230  for (i = 0; i < p.num_clients; i++) {
231  CkPrintf("num clients bh[%d] = %d\n", i, p.num_clients_bh[i]);
232  CkPrintf("num clients qh[%d] = %d\n", i, p.num_clients_qh[i]);
233  CkPrintf("num clients sh[%d] = %d\n", i, p.num_clients_bh[i]);
234  }
235 #endif
236 
237  p.hasLB.resize(p.num_clients);
238  p.hasLQ.resize(p.num_clients);
239  p.bh.resize(p.num_clients);
240  p.qh.resize(p.num_clients);
241  p.sh.resize(p.num_clients);
242 
243  for (i = 0; i < p.num_clients; i++) {
244  ia = p.k1r[i].nx;
245  ib = p.k1r[i].ny;
246  ja = p.k2r[i].nx;
247  jb = p.k2r[i].ny;
248  ka = p.k3r[i].nx;
249  kb = p.k3r[i].ny;
250 
251  p.bh[i] = Moa3Grid(ia, ib, ja, jb, ka, kb, p.num_clients_qh[i]);
252  p.qh[i] = Moa3Grid(ia, ib, ja, jb, ka, kb, p.num_clients_qh[i]);
253  p.sh[i] = Moa3Grid(ia, ib, ja, jb, ka, kb, p.num_clients_sh[i]);
254 
255  }
256  moaProxy.recvMoaData(moaData); // broadcast MoaData to chare group
257 
258 #ifdef OPENATOM_VERSION_DEBUG
259  CkPrintf("Initialized ComputeMoaMgr %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
260 #endif //OPENATOM_VERSION_DEBUG
261 
262 }
263 
264 void ComputeMoaMgr::recvB(int K2_start, int K2_end, int K3_start, int K3_end, int K1_len, int K2_len, int K3_len, double bm1[K1_len], double bm2[K2_len], double bm3[K3_len], int order)
265 {
266 #ifdef OPENATOM_VERSION_DEBUG
267  CkPrintf("Started recvB() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
268 #endif //OPENATOM_VERSION_DEBUG
269 
270 // const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
271 
272  MoaData &p = moaData;
273  int j,m,n;
274 
275  p.k1r[thisIndex].nx = 0;
276  p.k1r[thisIndex].ny = K1_len;
277  p.k2r[thisIndex].nx = K2_start;
278  p.k2r[thisIndex].ny = K2_end;
279  p.k3r[thisIndex].nx = K3_start;
280  p.k3r[thisIndex].ny = k3_end;
281 
282  p.bh[thisIndex].enroll();
283  bhwrit = p.bh[thisIndex].syncToWrite();
284 
285  for ( j=0; j < K1_len; j++ ) {
286  bhwrit.set(j,K2_start,K3_start) = bm1[j];
287  for ( m=K2_start; m < K2_end; m++ ) {
288  bhwrit.set(j,m,K3_start) = bm2[m];
289  for ( n=K3_start; n < K3_end; n++ ) {
290  bhwrit.set(j,m,n) = bm3[n];
291  }
292  }
293  }
294 
295  bhread = bhwrit.syncToRead();
296 
297  p.hasLB[thisIndex]=1;
298  p.gbqs.nx++;
299 
300 }
301 
302 void ComputeMoaMgr::recvQ(int g, int numgrids, int nq, float qgrid[nq], CkCallback doneMoa)
303 {
304 #ifdef OPENATOM_VERSION_DEBUG
305  CkPrintf("Started recvQ() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
306 #endif //OPENATOM_VERSION_DEBUG
307 
308 // const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
309  MoaData &p = moaData;
310 
311  p.qh[g].enroll(p.num_clients_qh[thisIndex]);
312  Eqhacc = p.qh[g].getInitialAccum();
313 
314  int i,j,m,n;
315 
316  int K1_len = p.k1r[thisIndex].ny;
317  int K2s = p.k2r[thisIndex].nx;
318  int K2e = p.k2r[thisIndex].ny;
319  int K3s = p.k3r[thisIndex].nx;
320  int K3e = p.k3r[thisIndex].ny;
321 
322  // qhwrit = p.qh[thisIndex].syncToWrite();
323  bhread = p.bh[thisIndex].syncToRead();
324 
325 // for ( i=0, i < nq, i+2) {
326 // qhrwrit.set(i)=qgrid[i];
327 // qhiwrit.set(i)=qgrid[i+1];
328 // }
329 
330 
331  for ( j=CkIndex(); j < K1_len; j++ ) {
332  qhwrit.set(j,K2s,K3s) = qgrid[j];
333  for ( m=K2s; m < K2e; m++ ) {
334  bhwrit.set(j,m,K3s) = qgrid[m];
335  for ( n=K3s; n < K3e; n++ ) {
336  bhwrit.set(j,m,n) = qgrid[n];
337  }
338  }
339  }
340 
341 
342  // for ( j = nq*g; j <= (nq * g + nq); j++)
343  for ( i = 0; i <= nq ; i++) {
344  p.qhwrite[g][i] = qgrid[i];
345  }
346 
347  if (g = numgrids) {
348  p.gbqs.ny++;
349  p.hasLQ[thisIndex]=1;
350  }
351 
352 }
353 
354 void ComputeMoaMgr::sendS2OA()
355 {
356  // syncToRead BGrid, pack it
357  // syncToRead SGrid, pack it
358  // send msg to openatom containing BGrid and SGrid
359  if (CkMyPe() != 0) return;
360 
361 #ifdef OPENATOM_VERSION_DEBUG
362  CkPrintf("Started sendS2OA() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
363 #endif //OPENATOM_VERSION_DEBUG
364 
365  CProxy_CP_LargeSP_RhoGSpacePlane oaRGSP_Proxy;
366 
367  CkCallback resumeMoa(CkIndex_ComputeMoaMgr::recvSGrid(), thishandle);
368  oaRGSP_Proxy[CkMyPe()].recvMDSg(g, ns, sg, resumeMoa);
369 
370 }
371 
372 void ComputeMoaMgr::recvSGrid(int g, int nq, double oaSg[nq])
373 {
374  if (CkMyPe() != 0) return;
375 #ifdef OPENATOM_VERSION_DEBUG
376  CkPrintf("Started recvSGrid() %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
377 #endif //OPENATOM_VERSION_DEBUG
378 
379  MoaData &p = moaData;
380  numgrids = p.numgrids;
381 
382 
383 }
384 
385 void ComputeMoaMgr::initWorkers(CkQdMsg *msg)
386 {
387  delete msg;
388  //printf("MOA initWorkers PE=%d\n", CkMyPe());
389  if (CkMyPe() != 0) return;
390  // PE 0 creates the compute chare arrays
391 
392  MoaData &p = moaData;
393  int n;
394  moaPatchProxy = CProxy_MoaPatch::ckNew(); // create empty chare array
395 #ifdef OPENATOM_VERSION_DEBUG
396  CkPrintf("Instantiated MoaPatch chare %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
397 #endif // OPENATOM_VERSION_DEBUG
398 
399  moaSProxy = CProxy_MoaS::ckNew(p.qh[0], p.eh[0], p.num_energy_chares);
400 
401 #ifdef OPENATOM_VERSION_DEBUG
402  CkPrintf("Created MoaS chare %d at PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
403 #endif // OPENATOM_VERSION_DEBUG
404 }
405 
406 
407 void ComputeMoaMgr::startWorkers(CkQdMsg *msg)
408 {
409  delete msg;
410  if (CkMyPe() != 0) return;
411  // only PE 0 can start workers;
412 #ifdef OPENATOM_VERSION_DEBUG
413  CkPrintf("Starting MoaMgr Workers on chare %d at PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
414 #endif // OPENATOM_VERSION_DEBUG
415 
416  // moaPatchProxy.compute();
417  moaSProxy.compute();
418 }
419 
420 
421 // /* Archaic soln */
422 // MoaPatch::MoaPatch(Moa3Grid &qh, Moa3Grid &sh, Moa3Grid &bh)
423 // {
424 // #ifdef OPENATOM_VERSION_DEBUG
425 // CkPrintf("Instantiated MoaPatch %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
426 // #endif // OPENATOM_VERSION_DEBUG
427 //
428 // const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch(CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
429 // int pIndex = thisIndex;
430 // const Int3 &dg = p.dim_gridcutoff_chares[pIndex];
431 // calcS = CProxy_MoaSGrid::ckNew(pIndex, qh, eh, dg.nx, dg.ny, dg.nz);
432 // }
433 
434 
435 MoaS::MoaS(Moa3Grid &qh, Moa3Grid &bh)
436 {
437 #ifdef OPENATOM_VERSION_DEBUG
438  CkPrintf("Instantiating MoaS %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
439 #endif //OPENATOM_VERSION_DEBUG
440 
441  const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
442 
443 
444  lia = p.k1r.nx;
445  lib = p.k1r.ny;
446  lja = p.k2r.nx;
447  ljb = p.k2r.ny;
448  lka = p.k3r.nx;
449  lkb = p.k3r.ny;
450  buff_len = (lib - lia) + (ljb - lja) + (lkb - lka);
451  qhbuff = new float[buff_len];
452  bhbuff = new int[buff_len];
453 
454  moaS_setup = 0;
455 
456  reduction = ReductionMgr:Object()->willSubmit(REDUCTIONS_BASIC);
457 }
458 
459 MoaS::~MoaS()
460 {
461 #ifdef OPENATOM_VERSION_DEBUG
462  CkPrintf("Destructing MoaS %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
463 #endif //OPENATOM_VERSION_DEBUG
464 
465 }
466 
467 MoaS::compute()
468 {
469 #ifdef OPENATOM_VERSION_DEBUG
470  CkPrintf("Starting MoaS compute %d on PE: %d, at %f.\n", thisIndex, CkMyPe(), CkWallTimer() );
471 #endif //OPENATOM_VERSION_DEBUG
472 
473  const MoaData &p = CProxy_ComputeMoaMgr::ckLocalBranch( CkpvAccess(BOCclass_group).computeMoaMgr)->getMoaData();
474 
475  if ( ! moaS_setup ) {
476  bh.enroll(p.num_clients);
477  qh[0].enroll(p.num_clients);
478  sh.enroll(p.num_clients);
479 
480  bhread = bh.syncToRead();
481  qhacc = qh.getInitialAccum();
482  qhread = qhacc.syncToRead();
483  shwrit = sh.getInitialWrite();
484  moaS_setup = 1;
485  }
486 
487  bhread = bh.syncToRead();
488 
489  int j,m,n,index;
490 
491  for (index = 0, j = lia; j <= lib; j++) {
492  for (m = lja; m <= ljb; m++) {
493  for (n = lka; n <= lkb; n++, index++) {
494  bhbuff[index] = bhread.get(j,m,n);
495  }
496  }
497  }
498 
499  qhacc = qhread.syncToEAccum();
500  qhread = qhacc.syncToRead();
501  for (index = 0, j = lia; j <= lib; j++) {
502  for (m = lja; m <= ljb; m++) {
503  for (n = lka; n <= lkb; n++, index++) {
504  qhbuff[index] = qhread.get(j,m,n);
505  }
506  }
507  }
508 
509  shwrit = sh.getInitialWrite();
510  for (index = 0, j = lia; j <= lib; j++) {
511  for (m = lja; m <= ljb; m++) {
512  for (n = lka; n <= lkb; n++, index++) {
513  shwrite.set(j,m,n) = qhbuff[index] * bhbuff[index] * bhbuff[index+1] * bhbuff[index+2]
514  }
515  }
516  }
517 
518 }
519 
520 
521 #include "ComputeMoaMgr.def.h"
522 
523 #endif // OPENTATOM_VERSION
524 
525 #endif // CHARM_HAS_MSA
526 
527 
Node::Object
static Node * Object()
Definition: Node.h:86
MoaData::hasLS
std::vector< int > hasLS
Definition: ComputeMoa.h:106
MoaData::gbqs
Int3 gbqs
Definition: ComputeMoa.h:103
ComputeID
int ComputeID
Definition: NamdTypes.h:183
MoaData::bh
std::vector< Moa3Grid > bh
Definition: ComputeMoa.h:100
MoaData::hasLQ
std::vector< int > hasLQ
Definition: ComputeMoa.h:104
Moa3Grid
int Moa3Grid
Definition: ComputeMoa.h:44
Node::simParameters
SimParameters * simParameters
Definition: Node.h:178
MoaData::k1r
std::vector< Int2 > k1r
Definition: ComputeMoa.h:102
ReductionMgr::willSubmit
SubmitReduction * willSubmit(int setID, int size=-1)
Definition: ReductionMgr.C:365
MoaData::num_clients_qh
std::vector< int > num_clients_qh
Definition: ComputeMoa.h:90
MoaData::order
int order
Definition: ComputeMoa.h:86
Int3::nx
int nx
Definition: ComputeMoa.h:65
MoaData::orig_x
float orig_x
Definition: ComputeMoa.h:87
MoaData::num_clients_sh
std::vector< int > num_clients_sh
Definition: ComputeMoa.h:92
isPmeProcessor
int isPmeProcessor(int)
Definition: ComputePme.C:604
MoaData::K3
int K3
Definition: ComputeMoa.h:85
ComputeMoa::~ComputeMoa
virtual ~ComputeMoa()
Int3::ny
int ny
Definition: ComputeMoa.h:65
order
#define order
Definition: PmeRealSpace.C:235
ComputeMoa::ComputeMoa
ComputeMoa(ComputeID c)
MoaData::num_clients_bh
std::vector< int > num_clients_bh
Definition: ComputeMoa.h:91
MoaData::orig_y
float orig_y
Definition: ComputeMoa.h:87
MoaData::num_clients
int num_clients
Definition: ComputeMoa.h:89
MoaData::K1
int K1
Definition: ComputeMoa.h:85
MoaData::qh
std::vector< Moa3Grid > qh
Definition: ComputeMoa.h:95
MoaData::pup
void pup(PUP::er &p)
MoaData::print
void print()
simParams
#define simParams
Definition: Output.C:127
MoaData::k3r
std::vector< Int2 > k3r
Definition: ComputeMoa.h:102
MoaData::K2
int K2
Definition: ComputeMoa.h:85
MoaData::k2r
std::vector< Int2 > k2r
Definition: ComputeMoa.h:102
MoaData::orig_z
float orig_z
Definition: ComputeMoa.h:87
ComputeMoa::doWork
void doWork()
MoaData::sh
std::vector< Moa3Grid > sh
Definition: ComputeMoa.h:96
Node::molecule
Molecule * molecule
Definition: Node.h:176
MoaData::hasLB
std::vector< int > hasLB
Definition: ComputeMoa.h:105
Generated by   doxygen 1.8.5