patchkernel: add tracking of vertices during adaptation/partitioning

Author: andrea-iob

src/patchkernel/adaption.hpp

@@ -56,7 +56,8 @@ namespace adaption
 	enum Entity {
 		ENTITY_UNKNOWN = -1,
 		ENTITY_CELL,
-		ENTITY_INTERFACE
+		ENTITY_INTERFACE,
+		ENTITY_VERTEX
 	};
 
 	struct Info

src/patchkernel/patch_kernel.cpp

@@ -790,8 +790,12 @@ std::vector<adaption::Info> PatchKernel::adaptionPrepare(bool trackAdaption)
 
 	Information available for tracking purposes are the following:
 	 - internal cells that have been coarsened/refined;
+	 - new internal vertices that have been created;
+     - internal vertices that have been deleted;
 	 - new ghost cells that have been created;
-	 - ghost cells that have been deleted.
+	 - new ghost vertices that have been created;
+	 - ghost cells that have been deleted;
+	 - ghost vertices that have been deleted.
 
 	\param trackAdaption if set to true the function will return the changes
 	done to the patch during the adaption

src/patchkernel/patch_kernel_parallel.cpp

@@ -1576,7 +1576,8 @@ std::vector<adaption::Info> PatchKernel::partitioningPrepare(MPI_Comm communicat
 
 	Information available on the sender side for tracking purposes are the
 	following:
-	 - internal cells that will be send.
+	 - internal cells that will be send;
+	 - internal vertices that will be send.
 
 	No information about tracking are provided on the receiver side.
 
@@ -1675,16 +1676,24 @@ std::vector<adaption::Info> PatchKernel::partitioningPrepare(const std::unordere
 	Information available on the sender side for tracking purposes are the
 	following:
 	 - internal cells that have been sent;
+	 - internal vertices that have been sent;
 	 - new ghost cells that have been created (some of the internal cells that
 	   have been sent may have become ghosts cells);
-	 - ghost cells that have been deleted.
+	 - new ghost vertices that have been created (some of the internal vertices
+	   that have been sent may have become ghosts vertices);
+	 - ghost cells that have been deleted;
+	 - ghost vertices that have been deleted.
 
 	Information available on the receiver side for tracking purposes are the
 	following:
 	 - internal cells that have been received;
+	 - internal vertices that have been received;
 	 - new ghost cells that have been created;
+	 - new ghost vertices that have been created;
 	 - ghost cells that have been deleted (some ghost cells may have been
-	   replaced by internal cells that have just been received).
+	   replaced by internal cells that have just been received);
+	 - ghost vertices that have been deleted (some ghost vertices may have been
+	   replaced by internal vertices that have just been received).
 
 	\param trackPartitioning if set to true the function will return the changes
 	done to the patch during the partitioning
@@ -2141,6 +2150,13 @@ std::vector<adaption::Info> PatchKernel::_partitioningPrepare(const std::unorder
 			}
 
 			// Fill tracking data structures
+			partitioningData.emplace_back();
+			adaption::Info &partitioningVertexInfo = partitioningData.back();
+			partitioningVertexInfo.entity   = adaption::ENTITY_VERTEX;
+			partitioningVertexInfo.type     = adaption::TYPE_PARTITION_SEND;
+			partitioningVertexInfo.rank     = recvRank;
+			partitioningVertexInfo.previous = getOrderedCellsVertices(cellsToSend, true, false);
+
 			partitioningData.emplace_back();
 			adaption::Info &partitioningCellInfo = partitioningData.back();
 			partitioningCellInfo.entity   = adaption::ENTITY_CELL;
@@ -2389,6 +2405,15 @@ std::vector<adaption::Info> PatchKernel::_partitioningAlter_deleteGhosts(bool tr
 
     // Track changes
     if (trackPartitioning) {
+        partitioningData.emplace_back();
+        adaption::Info &partitioningVertexInfo = partitioningData.back();
+        partitioningVertexInfo.entity= adaption::ENTITY_VERTEX;
+        partitioningVertexInfo.type  = adaption::TYPE_DELETION;
+        partitioningVertexInfo.current.reserve(getGhostVertexCount());
+        for (VertexConstIterator itr = ghostVertexConstBegin(); itr != ghostVertexConstEnd(); ++itr) {
+            partitioningVertexInfo.current.push_back(itr.getId());
+        }
+
         partitioningData.emplace_back();
         adaption::Info &partitioningCellInfo = partitioningData.back();
         partitioningCellInfo.entity = adaption::ENTITY_CELL;
@@ -2839,6 +2864,13 @@ std::vector<adaption::Info> PatchKernel::_partitioningAlter_sendCells(const std:
             // two processes are able to exchange cell data without additional
             // communications (they already know the list of cells for which
             // data is needed and the order in which these data will be sent).
+            partitioningData.emplace_back();
+            adaption::Info &partitioningVertexInfo = partitioningData.back();
+            partitioningVertexInfo.entity  = adaption::ENTITY_VERTEX;
+            partitioningVertexInfo.type    = adaption::TYPE_PARTITION_SEND;
+            partitioningVertexInfo.rank    = recvRank;
+            partitioningVertexInfo.current = getOrderedCellsVertices(cellSendList, true, false);
+
             partitioningData.emplace_back();
             adaption::Info &partitioningCellInfo = partitioningData.back();
             partitioningCellInfo.entity   = adaption::ENTITY_CELL;
@@ -3118,6 +3150,16 @@ std::vector<adaption::Info> PatchKernel::_partitioningAlter_sendCells(const std:
             cellCreationInfo.type    = adaption::TYPE_CREATION;
             cellCreationInfo.rank    = getRank();
             cellCreationInfo.current = getOrderedCellsVertices(trackedCreatedGhostCells, false, true);
+
+            std::vector<long> deletedGhostVertices = getOrderedCellsVertices(trackedCreatedGhostCells, false, true);
+            if (!deletedGhostVertices.empty()) {
+                partitioningData.emplace_back();
+                adaption::Info &vertexCreationInfo = partitioningData.back();
+                vertexCreationInfo.entity  = adaption::ENTITY_VERTEX;
+                vertexCreationInfo.type    = adaption::TYPE_CREATION;
+                vertexCreationInfo.rank    = getRank();
+                vertexCreationInfo.current = std::move(deletedGhostVertices);
+            }
         }
 
         // Delete frame cells that are not ghosts
@@ -3216,11 +3258,43 @@ std::vector<adaption::Info> PatchKernel::_partitioningAlter_sendCells(const std:
         // Prune stale interfaces
         pruneStaleInterfaces();
 
+        // Identify orphan vertices
+        std::vector<long> orphanVertices = findOrphanVertices();
+
+        // Track ghost vertices deletion
+        //
+        // Only ghost vertices need to be tracked, all orphan internal vertex
+        // have already been tracked among the vertices that have been send.
+        if (trackPartitioning && !orphanVertices.empty()) {
+            partitioningData.emplace_back();
+            adaption::Info &vertexDeletionInfo = partitioningData.back();
+            vertexDeletionInfo.entity = adaption::ENTITY_VERTEX;
+            vertexDeletionInfo.type   = adaption::TYPE_DELETION;
+            vertexDeletionInfo.rank   = getRank();
+            for (long vertexId : orphanVertices) {
+                const Vertex &vertex = getVertex(vertexId);
+                if (vertex.isInterior()) {
+                    continue;
+                }
+
+                vertexDeletionInfo.current.push_back(vertexId);
+            }
+        }
+
         // Delete orphan vertices
-        deleteOrphanVertices();
+        deleteVertices(orphanVertices);
     } else {
         // All ghost cells will be deleted
         if (trackPartitioning) {
+            partitioningData.emplace_back();
+            adaption::Info &partitioningVertexInfo = partitioningData.back();
+            partitioningVertexInfo.entity= adaption::ENTITY_VERTEX;
+            partitioningVertexInfo.type  = adaption::TYPE_DELETION;
+            partitioningVertexInfo.current.reserve(getGhostVertexCount());
+            for (VertexConstIterator itr = ghostVertexConstBegin(); itr != ghostVertexConstEnd(); ++itr) {
+                partitioningVertexInfo.current.push_back(itr.getId());
+            }
+
             partitioningData.emplace_back();
             adaption::Info &partitioningCellInfo = partitioningData.back();
             partitioningCellInfo.entity = adaption::ENTITY_CELL;
@@ -3876,6 +3950,13 @@ std::vector<adaption::Info> PatchKernel::_partitioningAlter_receiveCells(const s
         // Track changes
         if (trackPartitioning) {
             if (!trackedReceivedInteriorCells.empty()) {
+                partitioningData.emplace_back();
+                adaption::Info &vertexRecvInfo = partitioningData.back();
+                vertexRecvInfo.entity  = adaption::ENTITY_VERTEX;
+                vertexRecvInfo.type    = adaption::TYPE_PARTITION_RECV;
+                vertexRecvInfo.rank    = sendRank;
+                vertexRecvInfo.current = getOrderedCellsVertices(trackedReceivedInteriorCells, true, false);
+
                 partitioningData.emplace_back();
                 adaption::Info &cellRecvInfo = partitioningData.back();
                 cellRecvInfo.entity = adaption::ENTITY_CELL;
@@ -3886,6 +3967,14 @@ std::vector<adaption::Info> PatchKernel::_partitioningAlter_receiveCells(const s
             }
 
             if (!trackedCreatedGhostCells.empty()) {
+                partitioningData.emplace_back();
+                adaption::Info &vertexCreationInfo = partitioningData.back();
+                vertexCreationInfo.entity  = adaption::ENTITY_VERTEX;
+                vertexCreationInfo.type    = adaption::TYPE_CREATION;
+                vertexCreationInfo.rank    = patchRank;
+                vertexCreationInfo.current = getOrderedCellsVertices(trackedCreatedGhostCells, false, true);
+
+
                 partitioningData.emplace_back();
                 adaption::Info &cellCreationInfo = partitioningData.back();
                 cellCreationInfo.entity  = adaption::ENTITY_CELL;

src/volcartesian/volcartesian.cpp

@@ -1023,6 +1023,17 @@ std::vector<adaption::Info> VolCartesian::_spawn(bool trackSpawn)
 			cellId = cell.getId();
 		}
 
+		adaptionData.emplace_back();
+		adaption::Info &adaptionVertexInfo = adaptionData.back();
+		adaptionVertexInfo.type   = adaption::TYPE_CREATION;
+		adaptionVertexInfo.entity = adaption::ENTITY_VERTEX;
+		adaptionVertexInfo.current.reserve(m_cells.size());
+		for (auto &vertex : m_vertices) {
+			adaptionVertexInfo.current.emplace_back();
+			long &vertexId = adaptionVertexInfo.current.back();
+			vertexId = vertex.getId();
+		}
+
 		adaptionData.emplace_back();
 		adaption::Info &adaptionInterfaceInfo = adaptionData.back();
 		adaptionInterfaceInfo.type   = adaption::TYPE_CREATION;

src/voloctree/voloctree.cpp

@@ -1381,15 +1381,23 @@ std::vector<adaption::Info> VolOctree::sync(bool trackChanges)
 	if (!importFromScratch) {
 		log::cout() << "  Deleting stale elements..." << std::endl;
 
-		stitchInfo = deleteCells(synchronizationData);
+		if (trackChanges) {
+			stitchInfo = deleteCells(synchronizationData, &synchronizationData);
+		} else {
+			stitchInfo = deleteCells(synchronizationData, nullptr);
+		}
 
 		log::cout() << "  Stale element successfully deleted." << std::endl;
 	}
 
 	// Import added cells
 	log::cout() << "  Creating new elements..." << std::endl;
 
-	createCells(stitchInfo, nullptr, &synchronizationData);
+	if (trackChanges) {
+		createCells(stitchInfo, nullptr, &synchronizationData, &synchronizationData);
+	} else {
+		createCells(stitchInfo, nullptr, &synchronizationData, nullptr);
+	}
 
 	log::cout() << "  New elements successfully created." << std::endl;
 
@@ -1476,9 +1484,12 @@ void VolOctree::renumberCells(const adaption::InfoCollection &cellAdaptionData)
 	that need to be performed to the cells, on output the dummy ids contained in
 	the current statuses will be replaced with the actual ids of the cells that
 	have been created
+	\param vertexAdaptionData if a valid pointer is provided, on output will
+	contain the changes applied to the vertices
 */
 void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream,
-							adaption::InfoCollection *cellAdaptionData)
+							adaption::InfoCollection *cellAdaptionData,
+							adaption::InfoCollection *vertexAdaptionData)
 {
 	// Tree information
 	long nOctants       = m_tree->getNumOctants();
@@ -1537,6 +1548,11 @@ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream,
 #endif
 					}
 
+					// Track vertex creation
+					if (vertexAdaptionData) {
+						trackedCreatedVertices.push_back(vertexId);
+					}
+
 					// Add the vertex to the stitching info
 					stitchInfo[vertexTreeKey] = vertexId;
 				}
@@ -1657,23 +1673,71 @@ void VolOctree::createCells(StitchInfo &stitchInfo, std::istream *restoreStream,
 	if (!restoreStream) {
 		updateInterfaces(false);
 	}
+
+#if BITPIT_ENABLE_MPI==1
+	// Track internal vertices received from other partitions
+	std::unordered_set<long> recvVertices;
+	if (vertexAdaptionData) {
+		for (const adaption::Info &adaptionInfo : *cellAdaptionData) {
+			if (adaptionInfo.type != adaption::TYPE_PARTITION_RECV) {
+				continue;
+			}
+
+			std::size_t vertexRecvInfoId = vertexAdaptionData->insert(adaption::TYPE_PARTITION_RECV, adaption::ENTITY_VERTEX, adaptionInfo.rank);
+			adaption::Info &vertexRecvInfo = vertexAdaptionData->at(vertexRecvInfoId);
+			vertexRecvInfo.current = getOrderedCellsVertices(adaptionInfo.current, true, false);
+			recvVertices.insert(vertexRecvInfo.current.begin(), vertexRecvInfo.current.end());
+		}
+	}
+#endif
+
+	// Track vertex creation
+	if (!trackedCreatedVertices.empty()) {
+		std::size_t vertexCreationInfoId = vertexAdaptionData->insert(adaption::TYPE_CREATION, adaption::ENTITY_VERTEX);
+		adaption::Info &vertexCreationInfo = vertexAdaptionData->at(vertexCreationInfoId);
+		vertexCreationInfo.current = std::move(trackedCreatedVertices);
+#if BITPIT_ENABLE_MPI==1
+		adaption::InfoCollection::removeIds(recvVertices, &(vertexCreationInfo.current));
+#endif
+		trackedCreatedVertices.clear();
+	}
 }
 
 /*!
 	Delete the specified cells.
 
 	\param cellAdaptionData are the information that describe the changes
 	that need to be performed to the cells
+	\param vertexAdaptionData if a valid pointer is provided, on output will
+	contain the changes applied to the vertices
 	\param stitchInfo if a valid pointer is provided, on output will contain
 	the stitch information that can used to stich the faces created after
 	deleting the octants
 */
-VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cellAdaptionData)
+VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cellAdaptionData,
+											 adaption::InfoCollection *vertexAdaptionData)
 {
 	// Info of the cells
 	int nCellVertices = m_cellTypeInfo->nVertices;
 	int nCellFaces    = m_cellTypeInfo->nFaces;
 
+#if BITPIT_ENABLE_MPI==1
+	// Track internal vertices sent to other partitions
+	std::unordered_set<long> trackedSentVertices;
+	if (vertexAdaptionData) {
+		for (const adaption::Info &adaptionInfo : cellAdaptionData) {
+			if (adaptionInfo.type != adaption::TYPE_PARTITION_SEND) {
+				continue;
+			}
+
+			std::size_t vertexSendInfoId = vertexAdaptionData->insert(adaption::TYPE_PARTITION_SEND, adaption::ENTITY_VERTEX, adaptionInfo.rank);
+			adaption::Info &vertexSendInfo = vertexAdaptionData->at(vertexSendInfoId);
+			vertexSendInfo.previous = getOrderedCellsVertices(adaptionInfo.previous, true, false);
+			trackedSentVertices.insert(vertexSendInfo.previous.begin(), vertexSendInfo.previous.end());
+		}
+	}
+#endif
+
 	// Delete cells
 	std::vector<long> deadCells;
 	std::unordered_set<long> deadVertices;
@@ -1812,6 +1876,16 @@ VolOctree::StitchInfo VolOctree::deleteCells(const adaption::InfoCollection &cel
 	log::cout() << "    " << deadVertices.size() << " vertices will be removed." << std::endl;
 	PatchKernel::deleteVertices(deadVertices);
 
+	// Track vertex deletion
+	if (vertexAdaptionData && !deadVertices.empty()) {
+		std::size_t deletionInfoId = vertexAdaptionData->insert(adaption::TYPE_DELETION, adaption::ENTITY_VERTEX);
+		adaption::Info &deletionInfo = vertexAdaptionData->at(deletionInfoId);
+		deletionInfo.previous = std::vector<long>(deadVertices.begin(), deadVertices.end());
+#if BITPIT_ENABLE_MPI==1
+		adaption::InfoCollection::removeIds(trackedSentVertices, &(deletionInfo.previous));
+#endif
+	}
+
 	return stitchInfo;
 }
 

src/voloctree/voloctree.hpp

@@ -267,9 +267,11 @@ class VolOctree : public VolumeKernel {
 	void renumberCells(const adaption::InfoCollection &treeAdaptionData);
 
 	void createCells(StitchInfo &stitchInfo, std::istream *stream,
-					 adaption::InfoCollection *cellAdaptionData);
+					 adaption::InfoCollection *cellAdaptionData,
+	                 adaption::InfoCollection *vertexAdaptionData = nullptr);
 
-	StitchInfo deleteCells(const adaption::InfoCollection &cellAdaptionData);
+	StitchInfo deleteCells(const adaption::InfoCollection &cellAdaptionData,
+                           adaption::InfoCollection *vertexAdaptionData = nullptr);
 
 	std::vector<adaption::Info> sync(bool trackChanges);
 

src/voloctree/voloctree_parallel.cpp

@@ -166,6 +166,13 @@ std::vector<adaption::Info> VolOctree::_partitioningPrepare(const std::unordered
 				long &cellId = partitioningCellInfo.previous.back();
 				cellId = getOctantId(octantInfo);
 			}
+
+			partitioningData.emplace_back();
+			adaption::Info &partitioningVertexInfo = partitioningData.back();
+			partitioningVertexInfo.entity   = adaption::ENTITY_VERTEX;
+			partitioningVertexInfo.type     = adaption::TYPE_PARTITION_SEND;
+			partitioningVertexInfo.rank     = receiver;
+			partitioningVertexInfo.previous = getOrderedCellsVertices(partitioningCellInfo.previous, true, false);
 		}
 	}