Changes for load balancing support using reconverse

Author: adityapb

src/ck-core/ck.C

@@ -12,9 +12,9 @@ clients, including the rest of Charm++, are actually C++.
 
 #include "pathHistory.h"
 
-#if CMK_LBDB_ON
+#if 1
 #include "LBManager.h"
-#endif // CMK_LBDB_ON
+#endif // 1
 
 #ifndef CMK_CHARE_USE_PTR
 #include <map>
@@ -598,14 +598,14 @@ inline Chare *_popObj(void) {
 }
 
 inline void _ckStartTiming(void) {
-#if CMK_LBDB_ON
+#if 1
   auto *active = CkActiveLocRec();
   if (active) active->startTiming();
 #endif
 }
 
 inline void _ckStopTiming(void) {
-#if CMK_LBDB_ON
+#if 1
   auto *active = CkActiveLocRec();
   if (active) active->stopTiming();
 #endif
@@ -919,6 +919,8 @@ static CkGroupID _groupCreate(envelope *env)
      groupNum.idx = CkpvAccess(_numGroups)++;
   else
      groupNum.idx = _getGroupIdx(CkNumPes(),CkMyPe(),CkpvAccess(_numGroups)++);
+  printf("Created group %d\n", groupNum.idx);
+  fflush(stdout);
   _createGroup(groupNum, env);
   return groupNum;
 }

src/ck-core/ckarray.C

@@ -1871,7 +1871,7 @@ void CkArray::recvMsg(CkArrayMessage* msg, CmiUInt8 id, CkDeliver_t type, int op
 
 void CkArray::recordSend(const CmiUInt8 id, const unsigned int bytes, int pe, const int opts)
 {
-#if CMK_LBDB_ON
+#if 1
   if (!(opts & CK_MSG_LB_NOTRACE) && locMgr->getLBMgr()->CollectingCommStats())
   {
     // LB deals in IDs with collection information only when CMK_GLOBAL_LOCATION_UPDATE
@@ -1901,7 +1901,7 @@ void CkArray::sendToPe(CkArrayMessage* msg, int pe, CkDeliver_t type, int opts)
   // queuing infrastructure
   if (pe != CkMyPe() || type == CkDeliver_queue)
   {
-#if CMK_LBDB_ON
+#if 1
     // Track message sends if the LB framework is collecting comm stats
     if (msg->array_hops() == 0 && !(opts & CK_MSG_LB_NOTRACE) &&
         locMgr->getLBMgr()->CollectingCommStats())
@@ -1943,7 +1943,7 @@ void CkArray::sendToPe(CkArrayMessage* msg, int pe, CkDeliver_t type, int opts)
         demandCreateElement(idx, ctor);
       }
     }
-#if CMK_LBDB_ON
+#if 1
     // This ensures that communication is tracked even for inline sends
     if (msg->array_hops() == 0 && !(opts & CK_MSG_LB_NOTRACE) &&
         locMgr->getLBMgr()->CollectingCommStats())

src/ck-core/cklocation.C

@@ -23,15 +23,15 @@
 #include <stdarg.h>
 #include <vector>
 
-#if CMK_LBDB_ON
+#if 1
 #  include "LBManager.h"
 #  include "MetaBalancer.h"
 #  if CMK_GLOBAL_LOCATION_UPDATE
 #    include "BaseLB.h"
 #    include "init.h"
 #  endif
 CkpvExtern(int, _lb_obj_index);  // for lbdb user data for obj index
-#endif                           // CMK_LBDB_ON
+#endif                           // 1
 
 CpvExtern(std::vector<NcpyOperationInfo*>, newZCPupGets);  // used for ZC Pup
 #ifndef CMK_CHARE_USE_PTR
@@ -78,7 +78,7 @@ bool useNodeBlkMapping;
 /// unlocated array elements
 int _messageBufferingThreshold;
 
-#if CMK_LBDB_ON
+#if 1
 
 #  if CMK_GLOBAL_LOCATION_UPDATE
 void UpdateLocation(MigrateInfo& migData)
@@ -1723,7 +1723,7 @@ void CkMigratable::commonInit(void)
   prev_load = 0.0;
   can_reset = false;
 
-#if CMK_LBDB_ON
+#if 1
   if (_lb_args.metaLbOn())
   {
     atsync_iteration = myRec->getMetaBalancer()->get_iteration();
@@ -1750,7 +1750,7 @@ void CkMigratable::pup(PUP::er& p)
   p | can_reset;
   p | usesAutoMeasure;
 
-#if CMK_LBDB_ON
+#if 1
   bool readyMigrate = false;
   if (p.isPacking())
     readyMigrate = myRec->isReadyMigrate();
@@ -1795,7 +1795,7 @@ CkMigratable::~CkMigratable()
     prefetchObjID = -1;
   }
 #endif
-#if CMK_LBDB_ON
+#if 1
   if (barrierRegistered)
   {
     DEBL((AA "Removing barrier for element %s\n" AB, idx2str(thisIndexMax)));
@@ -1832,7 +1832,7 @@ void CkMigratable::UserSetLBLoad()
   CkAbort("::UserSetLBLoad() not defined for this array element!\n");
 }
 
-#if CMK_LBDB_ON  // For load balancing:
+#if 1  // For load balancing:
 // user can call this helper function to set obj load (for model-based lb)
 void CkMigratable::setObjTime(double cputime) { myRec->setObjTime(cputime); }
 double CkMigratable::getObjTime() { return myRec->getObjTime(); }
@@ -2079,7 +2079,7 @@ CkLocRec::CkLocRec(CkLocMgr* mgr, bool fromMigration, bool ignoreArrival,
                    const CkArrayIndex& idx_, CmiUInt8 id_)
     : myLocMgr(mgr), idx(idx_), id(id_), deletedMarker(NULL), running(false)
 {
-#if CMK_LBDB_ON
+#if 1
   DEBL((AA "Registering element %s with load balancer\n" AB, idx2str(idx)));
   nextPe = -1;
   asyncMigrate = false;
@@ -2109,7 +2109,7 @@ CkLocRec::~CkLocRec()
 {
   if (deletedMarker != NULL)
     *deletedMarker = true;
-#if CMK_LBDB_ON
+#if 1
   stopTiming();
   DEBL((AA "Unregistering element %s from load balancer\n" AB, idx2str(idx)));
   lbmgr->UnregisterObj(ldHandle);
@@ -2122,7 +2122,7 @@ void CkLocRec::migrateMe(int toPe)  // Leaving this processor
   myLocMgr->emigrate(this, toPe);
 }
 
-#if CMK_LBDB_ON
+#if 1
 void CkLocRec::startTiming(int ignore_running)
 {
   if (!ignore_running)
@@ -2196,7 +2196,7 @@ bool CkLocRec::invokeEntry(CkMigratable* obj, void* msg, int epIdx, bool doFree)
     }
   }
 #endif
-#if CMK_LBDB_ON
+#if 1
   if (!isDeleted)
     checkBufferedMigration();  // check if should migrate
 #endif
@@ -2206,7 +2206,7 @@ bool CkLocRec::invokeEntry(CkMigratable* obj, void* msg, int epIdx, bool doFree)
   return true;
 }
 
-#if CMK_LBDB_ON
+#if 1
 
 void CkLocRec::staticMetaLBResumeWaitingChares(LDObjHandle h, int lb_ideal_period)
 {
@@ -2433,7 +2433,7 @@ CkLocMgr::CkLocMgr(CkArrayOptions opts)
   compressor = ck::FixedArrayIndexCompressor::make(bounds);
 
   // Find and register with the load balancer
-#if CMK_LBDB_ON
+#if 1
   lbmgrID = _lbmgr;
   metalbID = _metalb;
   initLB(lbmgrID, metalbID);
@@ -2448,7 +2448,7 @@ CkLocMgr::CkLocMgr(CkMigrateMessage* m)
 
 CkLocMgr::~CkLocMgr()
 {
-#if CMK_LBDB_ON
+#if 1
   syncBarrier->removeBeginReceiver(lbBarrierBeginReceiver);
   syncBarrier->removeEndReceiver(lbBarrierEndReceiver);
   lbmgr->UnregisterOM(myLBHandle);
@@ -2489,7 +2489,7 @@ void CkLocMgr::pup(PUP::er& p)
     compressor = ck::FixedArrayIndexCompressor::make(bounds);
   }
 
-#if CMK_LBDB_ON
+#if 1
   p | lbmgrID;
   p | metalbID;
   if (p.isUnpacking())
@@ -2997,7 +2997,7 @@ void CkLocMgr::emigrate(CkLocRec* rec, int toPe)
   // Allocate and pack into message
   CkArrayElementMigrateMessage* msg = new (bufSize, 0)
       CkArrayElementMigrateMessage(idx, id,
-#if CMK_LBDB_ON
+#if 1
                                    rec->isAsyncMigrate(),
 #else
                                    false,
@@ -3032,7 +3032,7 @@ void CkLocMgr::emigrate(CkLocRec* rec, int toPe)
   cache->recordEmigration(id, toPe);
   informHome(idx, toPe);
 
-#if !CMK_LBDB_ON && CMK_GLOBAL_LOCATION_UPDATE
+#if !1 && CMK_GLOBAL_LOCATION_UPDATE
   DEBM((AA "Global location update. idx %s "
            "assigned to %d \n" AB,
         idx2str(idx), toPe));
@@ -3042,7 +3042,7 @@ void CkLocMgr::emigrate(CkLocRec* rec, int toPe)
   CK_MAGICNUMBER_CHECK
 }
 
-#if CMK_LBDB_ON
+#if 1
 void CkLocMgr::informLBPeriod(CkLocRec* rec, int lb_ideal_period)
 {
   callMethod(rec, &CkMigratable::recvLBPeriod, (void*)&lb_ideal_period);
@@ -3216,13 +3216,13 @@ unsigned int CkLocMgr::numLocalElements()
 
 /********************* LocMgr: LOAD BALANCE ****************/
 
-#if !CMK_LBDB_ON
+#if !1
 // Empty versions of all load balancer calls
 void CkLocMgr::startInserting(void) {}
 void CkLocMgr::doneInserting(void) {}
 #endif
 
-#if CMK_LBDB_ON
+#if 1
 void CkLocMgr::initLB(CkGroupID lbmgrID_, CkGroupID metalbID_)
 {  // Find and register with the load balancer
   lbmgr = (LBManager*)CkLocalBranch(lbmgrID_);

src/ck-core/cklocation.h

@@ -46,7 +46,7 @@ class CkArrayMessage : public CkMessage
 };
 
 /* Utility */
-//#if CMK_LBDB_ON
+//#if 1
 #include "LBManager.h"
 #include "MetaBalancer.h"
 //#endif
@@ -465,7 +465,7 @@ class CkLocMgr : public IrrGroup
   void callMethod(CkLocRec* rec, CkMigratable_voidfn_arg_t fn, void*);
   void callMethod(CkLocRec* rec, CkMigratable_voidfn_t fn);
 
-#if CMK_LBDB_ON
+#if 1
   CkGroupID lbmgrID;
   CkGroupID metalbID;
   CkSyncBarrier* syncBarrier;
@@ -680,7 +680,7 @@ class CkLocMgr : public IrrGroup
   void informLBPeriod(CkLocRec* rec, int lb_ideal_period);
   void metaLBCallLB(CkLocRec* rec);
 
-#if CMK_LBDB_ON
+#if 1
   LBManager* getLBMgr(void) const { return lbmgr; }
   MetaBalancer* getMetaBalancer(void) const { return the_metalb; }
   const LDOMHandle& getOMHandle(void) const { return myLBHandle; }

src/ck-core/ckmemcheckpoint.C

@@ -864,7 +864,7 @@ public:
 // restore the bitmap vector for LB
 void CkMemCheckPT::resetLB(int diepe)
 {
-#if CMK_LBDB_ON
+#if 1
   int i;
   std::vector<char> bitmap;
   // set processor available bitmap

src/ck-core/ckmigratable.h

@@ -94,7 +94,7 @@ class CkMigratable : public Chare {
 
   void ReadyMigrate(bool ready);
   //void ckFinishConstruction(int epoch = -1);
-  void ckFinishConstruction(int epoch = -1) { } //reconverse
+  void ckFinishConstruction(int epoch = -1); //reconverse
   void setMigratable(int migratable);
   void setPupSize(size_t obj_pup_size);
 #else

src/ck-core/ckmulticast.C

@@ -282,7 +282,7 @@ mCastEntry::mCastEntry (mCastEntry *old):
   parentGrp = old->parentGrp;
   allElem = old->allElem;
   allGrpElem = old->allGrpElem;
-#if CMK_LBDB_ON
+#if 1
   allObjKeys = old->allObjKeys;
 #endif
   pe = old->pe;
@@ -311,7 +311,7 @@ void CkMulticastMgr::setSection(CkSectionInfo &_id, CkArrayID aid, CkArrayIndex
     entry->allObjKeys.reserve(n);
     for (int i=0; i<n; i++) {
         entry->allElem[i] = al[i];
-#if CMK_LBDB_ON
+#if 1
         CmiUInt8 _key;
         if(CProxy_ArrayBase(aid).ckLocMgr()->lookupID(al[i], _key))
             entry->allObjKeys.push_back(_key);
@@ -350,7 +350,7 @@ void CkMulticastMgr::setSection(CProxySection_ArrayElement &proxy)
   entry->allObjKeys.reserve(proxy.ckGetNumElements());
   for (int i=0; i<proxy.ckGetNumElements(); i++) {
     entry->allElem[i] = al[i];
-#if CMK_LBDB_ON
+#if 1
     CmiUInt8 _key;
     if(CProxy_ArrayBase(aid).ckLocMgr()->lookupID(al[i], _key))
       entry->allObjKeys.push_back(_key);
@@ -407,7 +407,7 @@ void CkMulticastMgr::prepareCookie(mCastEntry *entry, CkSectionID &sid, const Ck
   entry->allObjKeys.reserve(count);
   for (int i=0; i<count; i++) {
     entry->allElem[i] = al[i];
-#if CMK_LBDB_ON
+#if 1
     CmiUInt8 _key;
     if(CProxy_ArrayBase(aid).ckLocMgr()->lookupID(al[i], _key))
       entry->allObjKeys.push_back(_key);
@@ -931,7 +931,7 @@ void CkMulticastMgr::sendToSection(CkDelegateData *pd,int ep,void *m, CkSectionI
       s.get_val() = entry;
     }
 
-#if CMK_LBDB_ON
+#if 1
     if(!entry->isGrpSec()){
       // fixme: running obj?
       envelope *env = UsrToEnv(msg);

src/ck-ldb/BaseLB.C

@@ -7,7 +7,7 @@
 #include "BaseLB.h"
 #include "LBSimulation.h"
 
-#if CMK_LBDB_ON
+#if 1
 
 void BaseLB::initLB(const CkLBOptions &opt) {
   seqno = opt.getSeqNo();
@@ -44,14 +44,14 @@ void BaseLB::flushStates() {}
 
 void BaseLB::turnOn()
 {
-#if CMK_LBDB_ON
+#if 1
   lbmgr->TurnOnStartLBFn(startLbFnHdl);
 #endif
 }
 
 void BaseLB::turnOff()
 {
-#if CMK_LBDB_ON
+#if 1
   lbmgr->TurnOffStartLBFn(startLbFnHdl);
 #endif
 }
@@ -168,7 +168,7 @@ void BaseLB::LDStats::deleteCommHash() {
 
 int BaseLB::LDStats::getHash(const CmiUInt8 &oid, const LDOMid &mid)
 {
-#if CMK_LBDB_ON
+#if 1
     CmiAssert(hashSize > 0);
     int hash = ObjKey(oid, hashSize);
 
@@ -215,7 +215,7 @@ void BaseLB::LDStats::clearCommHash() {
 
 void BaseLB::LDStats::computeNonlocalComm(int &nmsgs, int &nbytes)
 {
-#if CMK_LBDB_ON
+#if 1
     	nmsgs = 0;
 	nbytes = 0;
 
@@ -295,7 +295,7 @@ void BaseLB::LDStats::normalize_speed() {
 
 void BaseLB::LDStats::print()
 {
-#if CMK_LBDB_ON
+#if 1
   int i = 0;
   CkPrintf("------------- Processor Data: %zu -------------\n", procs.size());
   for (const auto& proc : procs)

src/ck-ldb/BaseLB.h

@@ -272,7 +272,7 @@ class LBVectorMigrateMsg : public CMessage_LBVectorMigrateMsg {
 //                       sequence ticket,
 // AllocateFooLB():      allocate the class instead of a BOC
 // static void lbinit(): an init call for charm module registration
-#if CMK_LBDB_ON
+#if 1
 
 #  define CreateLBFunc_Def(x, str)                                                       \
     CMK_DEPRECATED_MSG("Use LBRegisterBalancer() instead of CreateLBFunc_Def()")         \