diff --git a/src/exchangeAMRDonors.C b/src/exchangeAMRDonors.C index eba12c0..d863f8c 100644 --- a/src/exchangeAMRDonors.C +++ b/src/exchangeAMRDonors.C @@ -342,7 +342,7 @@ void tioga::checkComm(void) sndPack[i].intData[1]=1; } // - pc_cart->sendRecvPackets(sndPack,rcvPack); + pc_cart->sendRecvPackets2(sndPack,rcvPack); pc_cart->clearPackets(sndPack,rcvPack); TIOGA_FREE(sndMap); TIOGA_FREE(rcvMap); diff --git a/src/exchangeBoxes.C b/src/exchangeBoxes.C index f3ad56b..2df1aa9 100644 --- a/src/exchangeBoxes.C +++ b/src/exchangeBoxes.C @@ -240,7 +240,7 @@ void tioga::exchangeBoxes(void) // << std::setw(3) << key_recv << " " // << std::setw(3) << key_send << std::endl; } - pc->sendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); for (int k=0; kgetMap(&nsend,&nrecv,&sndMap,&rcvMap); - if (nsend == 0) return; // // create packets to send and receive // and initialize them to zero @@ -74,7 +73,8 @@ void tioga::exchangeDonors(void) // // communicate donors (comm1) // - pc->sendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); + //if (nsend == 0) return; // Initialize linked lists and populate donor data from rcvPack for (int ib=0;ibsendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); std::vector ninterp(nblocks,0); for(int k=0;ksendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); for (int k=0; ksendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); // for(int ib=0;ibclearIblanks(); diff --git a/src/exchangeSearchData.C b/src/exchangeSearchData.C index cc661b5..58a8e62 100644 --- a/src/exchangeSearchData.C +++ b/src/exchangeSearchData.C @@ -113,7 +113,7 @@ void tioga::exchangeSearchData(int at_points) sndPack[k].realData[m++] = real_data[ii][j]; } } - pc->sendRecvPackets(sndPack, rcvPack); + pc->sendRecvPackets2(sndPack, rcvPack); // Reset MeshBlock data structures for (int ib=0;ib snd_int_displs(numprocs+1, 0); + std::vector rcv_int_displs(numprocs+1, 0); + for (int i=1; i <= numprocs; i++) { + snd_int_displs[i] = snd_int_displs[i-1] + sint[i-1]; + rcv_int_displs[i] = rcv_int_displs[i-1] + rint[i-1]; + } + for (int i=0; i < nsend; i++) { + int displ = snd_int_displs[sndMap[i]]; + for(int j=0; j < sint[sndMap[i]]; j++){ + all_snd_intData[displ+j] = sndPack[i].intData[j]; + } + } + MPI_Request int_request; + MPI_Ialltoallv(all_snd_intData, + sint, + snd_int_displs.data(), + MPI_INT, + all_rcv_intData, + rint, + rcv_int_displs.data(), + MPI_INT, + scomm, + &int_request); + + int all_snd_nreals = std::accumulate(sreal, sreal + numprocs, 0); + int all_rcv_nreals = std::accumulate(rreal, rreal + numprocs, 0); + REAL *all_snd_realData, *all_rcv_realData; + all_snd_realData=(REAL *) malloc(sizeof(REAL)*all_snd_nreals); + all_rcv_realData=(REAL *) malloc(sizeof(REAL)*all_rcv_nreals); + for (int i = 0; i < all_snd_nreals; i++) { + all_snd_realData[i] = 0; + } + for (int i = 0; i < all_rcv_nreals; i++) { + all_rcv_realData[i] = 0; + } + std::vector snd_real_displs(numprocs+1, 0); + std::vector rcv_real_displs(numprocs+1, 0); + for (int i=1; i <= numprocs; i++) { + snd_real_displs[i] = snd_real_displs[i-1] + sreal[i-1]; + rcv_real_displs[i] = rcv_real_displs[i-1] + rreal[i-1]; + } + for (int i=0; i < nsend; i++) { + int displ = snd_real_displs[sndMap[i]]; + for(int j=0; j < sreal[sndMap[i]]; j++){ + all_snd_realData[displ+j] = sndPack[i].realData[j]; + } + } + MPI_Request real_request; + MPI_Ialltoallv(all_snd_realData, + sreal, + snd_real_displs.data(), + MPI_DOUBLE, + all_rcv_realData, + rreal, + rcv_real_displs.data(), + MPI_DOUBLE, + scomm, + &real_request); + + // FIXME: here and above I think I should move this a bit lower + MPI_Wait(&int_request, MPI_STATUS_IGNORE); + for(i=0;i 0) { + rcvPack[i].intData=(int *) malloc(sizeof(int)*rcvPack[i].nints); + } + if (rcvPack[i].nreals > 0) { + rcvPack[i].realData=(REAL *) malloc(sizeof(REAL)*rcvPack[i].nreals); + } + } + + // FIXME: here and above I think I should move this a bit lower + MPI_Wait(&real_request, MPI_STATUS_IGNORE); + for (int i=0; i < nrecv; i++) { + int displ = rcv_int_displs[rcvMap[i]]; + for(int j=0; j < rint[rcvMap[i]]; j++){ + rcvPack[i].intData[j] = all_rcv_intData[displ+j]; + } + } + for (int i=0; i < nrecv; i++) { + int displ = rcv_real_displs[rcvMap[i]]; + for(int j=0; j < rreal[rcvMap[i]]; j++){ + rcvPack[i].realData[j] = all_rcv_realData[displ+j]; + } + } + + TIOGA_FREE(all_snd_intData); + TIOGA_FREE(all_rcv_intData); + TIOGA_FREE(all_snd_realData); + TIOGA_FREE(all_rcv_realData); + TIOGA_FREE(sint); + TIOGA_FREE(sreal); + TIOGA_FREE(rint); + TIOGA_FREE(rreal); +} + void parallelComm::sendRecvPackets(PACKET *sndPack,PACKET *rcvPack) { int i; diff --git a/src/parallelComm.h b/src/parallelComm.h index c941a1e..6796e7a 100644 --- a/src/parallelComm.h +++ b/src/parallelComm.h @@ -58,6 +58,8 @@ class parallelComm { void sendRecvPacketsAll(PACKET *sndPack, PACKET *rcvPack); + void sendRecvPackets2(PACKET *sndPack, PACKET *rcvPack); + void sendRecvPackets(PACKET *sndPack, PACKET *rcvPack); void sendRecvPacketsCheck(PACKET *sndPack, PACKET *rcvPack); diff --git a/src/tioga.C b/src/tioga.C index 3405419..02e4984 100644 --- a/src/tioga.C +++ b/src/tioga.C @@ -492,7 +492,8 @@ void tioga::performConnectivity(void) this->myTimer("tioga::exchangeDonors",1); //this->reduce_fringes(); //outputStatistics(); - MPI_Allreduce(&ihigh,&ihighGlobal,1,MPI_INT,MPI_MAX,scomm); + //MPI_Allreduce(&ihigh,&ihighGlobal,1,MPI_INT,MPI_MAX,scomm); + ihighGlobal = 0; //if (ihighGlobal) { this->myTimer("tioga::getCellIblanks",0); for (int ib=0;ibgetMap(&nsend,&nrecv,&sndMap,&rcvMap); - if (nsend==0) return; + //if (nsend==0) return; sndPack=(PACKET *)malloc(sizeof(PACKET)*nsend); rcvPack=(PACKET *)malloc(sizeof(PACKET)*nrecv); // @@ -807,7 +808,7 @@ void tioga::dataUpdate(int nvar,int interptype, int at_points) // // communicate the data across // - pc->sendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); // // decode the packets and update the data // @@ -968,7 +969,7 @@ void tioga::getReceptorInfo(std::vector& receptors) if (nsend == 0) { receptors.clear(); - return; + //return; } // @@ -1037,7 +1038,7 @@ void tioga::getReceptorInfo(std::vector& receptors) } } - pc->sendRecvPackets(sndPack, rcvPack); + pc->sendRecvPackets2(sndPack, rcvPack); int rsize=0; for (int k=0; ksendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); // for (int k=0; ksendRecvPackets(sndPack,rcvPack); + pc->sendRecvPackets2(sndPack,rcvPack); // for(int ib=0;ibclearIblanks();