76 idx_t *eind, *eptr, *xadj, *adjncy, *vwgt, *vsize;
79 int ndofman, idofman, numflag, ncommonnodes, options [ 4 ], ie, nproc;
80 int edgecut, wgtflag, ncon;
81 real_t ubvec [ 1 ], itr;
83 MPI_Comm communicator = MPI_COMM_WORLD;
93 for ( i = 1; i <= nelem; i++ ) {
102 eind =
new idx_t [ eind_size ];
103 eptr =
new idx_t [ nlocalelems + 1 ];
104 if ( ( eind == NULL ) || ( eptr == NULL ) ) {
105 OOFEM_ERROR(
"failed to allocate eind and eptr arrays");
109 int eind_pos = 0, eptr_pos = 0;
110 for ( i = 1; i <= nelem; i++ ) {
113 eptr [ eptr_pos ] = eind_pos;
115 for ( idofman = 1; idofman <= ndofman; idofman++ ) {
124 eptr [ nlocalelems ] = eind_pos;
131 ParMETIS_V3_Mesh2Dual(
elmdist, eptr, eind, & numflag, & ncommonnodes, & xadj, & adjncy, & communicator);
133 #ifdef ParmetisLoadBalancer_DEBUG_PRINT 136 fprintf(stderr,
"[%d] xadj:", myrank);
137 for ( i = 0; i <= nlocalelems; i++ ) {
138 fprintf(stderr,
" %d", xadj [ i ]);
141 fprintf(stderr,
"\n[%d] adjncy:", myrank);
142 for ( i = 0; i < xadj [ nlocalelems ]; i++ ) {
143 fprintf(stderr,
" %d", adjncy [ i ]);
146 fprintf(stderr,
"\n");
162 if ( (
tpwgts =
new real_t [ nproc ] ) == NULL ) {
167 for ( i = 0; i < nproc; i++ ) {
168 tpwgts [ i ] = _procweights(i);
179 if ( ( vwgt =
new idx_t [ nlocalelems ] ) == NULL ) {
183 if ( ( vsize =
new idx_t [ nlocalelems ] ) == NULL ) {
187 for ( ie = 0, i = 0; i < nelem; i++ ) {
198 if ( ( part =
new idx_t [ nlocalelems ] ) == NULL ) {
203 ParMETIS_V3_AdaptiveRepart(
elmdist, xadj, adjncy, vwgt, vsize, NULL, & wgtflag, & numflag, & ncon, & nproc,
204 tpwgts, ubvec, & itr, options, & edgecut, part, & communicator);
211 for ( i = 1; i <= nelem; i++ ) {
226 #ifdef ParmetisLoadBalancer_DEBUG_PRINT 228 fprintf(stderr,
"[%d] edgecut: %d elementPart:", myrank, edgecut);
229 for ( i = 1; i <= nelem; i++ ) {
233 fprintf(stderr,
"\n");
256 elmdist =
new idx_t [ nproc + 1 ];
266 for ( i = 1; i <= nelem; i++ ) {
272 procElementCounts(myrank) = nlocelem;
274 MPI_Allgather(& nlocelem, 1, MPI_INT, procElementCounts.
givePointer(), 1, MPI_INT, MPI_COMM_WORLD);
276 for ( i = 0; i < nproc; i++ ) {
277 elmdist [ i + 1 ] = elmdist [ i ] + procElementCounts(i);
283 for ( i = 0; i < myrank; i++ ) {
294 for ( i = 1; i <= nelem; i++ ) {
314 std :: set< int, std :: less< int > >__dmanpartitions;
326 #ifdef ParmetisLoadBalancer_DEBUG_PRINT 328 fprintf(stderr,
"[%d] DofManager labels:\n", myrank);
332 for ( idofman = 1; idofman <= ndofman; idofman++ ) {
337 __dmanpartitions.
clear();
338 for ( ie = 1; ie <= dofmanconntable->
giveSize(); ie++ ) {
347 npart = __dmanpartitions.size();
350 for (
auto &dm: __dmanpartitions ) {
366 com.finishExchange();
369 for ( idofman = 1; idofman <= ndofman; idofman++ ) {
382 #ifdef ParmetisLoadBalancer_DEBUG_PRINT 383 for ( idofman = 1; idofman <= ndofman; idofman++ ) {
384 fprintf(stderr,
" | %d: ", idofman);
386 fprintf(stderr,
"NULL ");
388 fprintf(stderr,
"Local ");
390 fprintf(stderr,
"Shared");
392 fprintf(stderr,
"Remote");
394 fprintf(stderr,
"Unknown");
401 if ( ( ( ++_cols % 4 ) == 0 ) || ( idofman == ndofman ) ) {
402 fprintf(stderr,
"\n");
416 if ( ( npart == 1 ) && ( dofManPartitions->
at(1) == myrank ) ) {
419 }
else if ( npart == 1 ) {
491 int ndofman, idofman;
494 if ( iproc == myrank ) {
502 for ( idofman = 1; idofman <= ndofman; idofman++ ) {
523 int _globnum, _locnum;
526 if ( iproc == myrank ) {
535 pcbuff->
read(_globnum);
542 OOFEM_ERROR(
"internal error, unknown global dofman %d", _globnum);
551 pcbuff->
read(_globnum);
560 for (
int part: _partitions ) {
570 int __i, __j, __partition, _master;
584 for ( idofman = 1; idofman <= ndofman; idofman++ ) {
591 for ( __i = 1; __i <= slaveMastersDofMans.
giveSize(); __i++ ) {
593 _master = slaveMastersDofMans.
at(__i);
596 for ( __j = 1; __j <=
dofManPartitions [ idofman - 1 ].giveSize(); __j++ ) {
int unpackSharedDmanPartitions(ProcessCommunicator &pc)
contextIOResultType storeYourself(DataStream &stream) const
Stores array to output stream.
int determineDofManState(int idofman, int myrank, int npart, IntArray *dofManPartitions)
DofManMode
Describes the state of dofmanager after load balancing on the local partition.
virtual IntArray * giveDofManPartitions(int idofman)
Returns the partition list of given dofmanager after load balancing.
int giveGlobalNumber() const
virtual bool giveMasterDofMans(IntArray &masters)
Returns true if the receiver is linked (its slave DOFs depend on master values) to some other dof man...
int giveNumberOfDofManagers() const
Returns number of dof managers in domain.
int packAllData(T *ptr, int(T::*packFunc)(ProcessCommunicator &))
Pack all problemCommunicators data to their send buffers.
real_t * tpwgts
Partition weights (user input).
void zero()
Sets all component to zero.
Shared dofman that remains shared.
void initGlobalDofManMap(bool forceinit=false)
Initializes global dof man map according to domain dofman list.
ConnectivityTable * giveConnectivityTable()
Returns receiver's associated connectivity table.
const IntArray * givePartitionList()
Returns partition list of receiver.
virtual void calculateLoadTransfer()
ParmetisLoadBalancer(Domain *d)
int packSharedDmanPartitions(ProcessCommunicator &pc)
EngngModel * giveEngngModel()
Returns engineering model to which receiver is associated.
std::vector< IntArray > dofManPartitions
Array of dof man partitions.
Undefined (undetermined) state, if assigned means internal error.
Abstract base class for all finite elements.
virtual DofManMode giveDofManState(int idofman)
Returns the label of dofmanager after load balancing.
Base class for dof managers.
int giveNumberOfProcesses() const
Returns the number of collaborating processes.
const int * givePointer() const
Breaks encapsulation.
int giveNumberOfElements() const
Returns number of elements in domain.
The ProcessCommunicator and corresponding buffers (represented by this class) are separated in order ...
int giveRank()
Returns corresponding rank of associated partition.
virtual bool hasAnySlaveDofs()
Class implementing an array of integers.
int & at(int i)
Coefficient access function.
void addSharedDofmanPartitions(int _locnum, IntArray _partitions)
virtual int giveNumberOfDofManagers() const
dofManagerParallelMode
In parallel mode, this type indicates the mode of DofManager.
Local dofman that became remote (became local on remote partition).
IntArray elementPart
Partition vector of the locally-stored elements.
const FloatArray & giveProcessorWeights()
Returns processor weights; the larger weight means more powerful node, sum of weights should equal to...
void initGlobalParmetisElementNumbering()
Element * giveElement(int n)
Service for accessing particular domain fe element.
virtual int write(const int *data, int count)
Writes count integer values from array pointed by data.
void clear()
Clears the array (zero size).
Local dofman that remains local.
ProcessCommunicatorBuff * giveProcessCommunicatorBuff()
Returns communication buffer.
IntArray dofManState
Array of DofManMode(s).
#define SHARED_DOFMAN_PARTITIONS_TAG
Class representing connectivity table.
Class representing process communicator for engineering model.
#define PARMETISLB_END_DATA
End-of-data marker, used to identify end of data stream received.
void resize(int n)
Checks size of receiver towards requested bounds.
REGISTER_LoadBalancer(ParmetisLoadBalancer)
contextIOResultType restoreYourself(DataStream &stream)
Restores array from image on stream.
IntArray gToLMap
Element numbering maps.
Abstract base class representing general load balancer.
(Dynamic) In this case the communication pattern and the amount of data sent between nodes is not kno...
Class representing vector of real numbers.
virtual int giveElementPartition(int ielem)
Returns the new partition number assigned to local element after LB.
elementParallelMode giveParallelMode() const
Return elementParallelMode of receiver.
int dofmanGlobal2Local(int _globnum)
Element is local, there are no contributions from other domains to this element.
const IntArray * giveDofManConnectivityArray(int dofman)
void labelDofManagers()
Label local partition nodes (the nodes that are local or shared).
Class representing communicator.
virtual double predictRelativeComputationalCost()
Returns the weight representing relative computational cost of receiver The reference element is tria...
void handleMasterSlaveDofManLinks()
int giveRank() const
Returns domain rank in a group of collaborating processes (0..groupSize-1)
virtual int read(int *data, int count)
Reads count integer values into array pointed by data.
virtual ~ParmetisLoadBalancer()
The Communicator and corresponding buffers (represented by this class) are separated in order to allo...
the oofem namespace is to define a context or scope in which all oofem names are defined.
DofManager * giveDofManager(int i) const
DofManager * giveDofManager(int n)
Service for accessing particular domain dof manager.
DofManager is local, there are no contribution from other domains to this DofManager.
virtual LoadBalancerMonitor * giveLoadBalancerMonitor()
Returns reference to receiver's load balancer monitor.
DofManager is shared by neighboring partitions, it is necessary to sum contributions from all contrib...
dofManagerParallelMode giveParallelMode() const
Return dofManagerParallelMode of receiver.
int findFirstIndexOf(int value) const
Finds index of first occurrence of given value in array.
Abstract base class representing general load balancer monitor.