!------------------------------------------------------------------------!         NASA/GSFC, Data Assimilation Office, Code 910.3, GEOS/DAS!------------------------------------------------------------------------      MODULE ghostmodule!BOP!! !MODULE: ghostmodule!! !USES:      USE decompmodule, ONLY : DecompType#include "debug.h"#include "pilgrim.h"      IMPLICIT NONE!! !DESCRIPTION:!!      This module provides the basic support for "ghost regions".  In!      reality the ghost region just subset of the global domain !      described by a decomposition (pro memoria: a decomposition !      describes a partition of a global index space over a number !      of PEs; this is inherently non-overlapping).  !!      It contains the following public types and routines.!      \begin{center}!      \begin{tabular}{|l|l|} \hline \hline!         GhostType   & Type to describe ghosted local vector \\ \hline!         GhostFree   & Destroy a ghost definition \\ \hline!         GhostCreate & Copy ghost definition to newly created one\\ \hline !         GhostInfo   & Returns some information about the region \\ \hline !         \hline  \hline!      \end{tabular}!      \end{center}!!      GhostCreate is overloaded to support different types of domains:!!      \begin{center}!      \begin{tabular}{|l|l|} \hline \hline!         GhostRegular1D & Define a subset of a 1D domain \\ \hline!         GhostRegular2D & Define a subset of a 2D domain \\ \hline!         GhostRegular3D & Define a subset of a 3D domain \\ \hline!         GhostIrregular & Define a subset of an irregular domain \\ \hline!         \hline  \hline!      \end{tabular}!      \end{center}!!      Generally one will use the GhostCreate routine which corresponds!      to the underlying decomposition; e.g., if the decomposition was!      defined with DecompRegular3D you would probably use GhostRegular3D!      to define the ghost region.  But since decompositions and ghost!      regions are generic, i.e., one-size-fits-all, this is not a requirement.!      Be very careful if you use non-corresponding routines!!!      The ghost type contains a decomposition which describes the!      {\it non-overlapping} distribution of the global domain !      (this is a replicated data structure with complete information!      about all data structures on all PEs).  Its other components are!      a list of the global indices of the on the boundary !      (not replicated), and a description of the mapping of the ghosted!      local region to global indices.!!      This module is communication-free and is a foundation!      for ParUtilitiesModule.  Since GhostType is local to the!      PE, the modules routines can and should be called with !      non-replicated data structures.  Before boundary communication!      takes place, the communication pattern derived from the ghost regions!      must be created (see ParUtilitiesModule).  !       ! !REVISION HISTORY:!   00.11.10   Sawyer     Creation!   01.02.07   Sawyer     Improvements; added Border to GhostType!   01.02.12   Sawyer     Converted to free format!! !PUBLIC TYPES:      PUBLIC GhostType      PUBLIC GhostFree      PUBLIC GhostCreate      PUBLIC GhostInfo      INTERFACE     GhostCreate        MODULE PROCEDURE GhostIrregular        MODULE PROCEDURE GhostRegular1D        MODULE PROCEDURE GhostRegular2D        MODULE PROCEDURE GhostRegular3D      END INTERFACE ! Decomposition info       TYPE GhostType         TYPE(DecompType) :: Decomp ! Decomposition of global partition         TYPE(DecompType) :: Local  ! Decomposition of local region         TYPE(DecompType) :: Border ! Decomposition of local segment       END TYPE GhostType!EOP      CONTAINS!-----------------------------------------------------------------------!BOP! !IROUTINE: GhostFree --- Free a ghosted region!! !INTERFACE:      SUBROUTINE GhostFree ( Ghost )! !USES:      USE decompmodule, ONLY : DecompFree      IMPLICIT NONE! !INPUT/OUTPUT PARAMETERS:      TYPE(GhostType), INTENT( INOUT ):: Ghost  ! Ghost information!! !DESCRIPTION:!     Free the ghost decomposition -- deallocate the data structures.!! !SYSTEM ROUTINES:!     ASSOCIATED, DEALLOCATE!! !REVISION HISTORY:!   00.11.12   Sawyer     Creation!!EOP!-----------------------------------------------------------------------!BOC!!      CPP_ENTER_PROCEDURE( "GHOSTFREE" )      CALL DecompFree( Ghost%Border )      CALL DecompFree( Ghost%Local  )      CALL DecompFree( Ghost%Decomp )       CPP_LEAVE_PROCEDURE( "GHOSTFREE" )      RETURN!EOC      END SUBROUTINE GhostFree!-----------------------------------------------------------------------!-----------------------------------------------------------------------!BOP! !IROUTINE: GhostCopy --- Copy one decomposition to another!! !INTERFACE:      SUBROUTINE GhostCopy ( GhostIn, GhostOut )! !USES:      USE decompmodule, ONLY : DecompCopy      IMPLICIT NONE!! !INPUT PARAMETERS:      TYPE(GhostType), INTENT( IN )   :: GhostIn  ! Ghost information!! !OUTPUT PARAMETERS:      TYPE(GhostType), INTENT( OUT )  :: GhostOut ! Ghost information!! !DESCRIPTION:!!   Creates an output ghost definition and copies GhostIn to it !! !SYSTEM ROUTINES:!     ALLOCATE!! !REVISION HISTORY:!   00.11.12   Sawyer     Creation!!EOP!-----------------------------------------------------------------------!BOC! !LOCAL VARIABLES:      INTEGER  :: I, Nsize      CPP_ENTER_PROCEDURE( "GHOSTCOPY" )      CALL DecompCopy( GhostIn%Decomp, GhostOut%Decomp )      CALL DecompCopy( GhostIn%Local,  GhostOut%Local  )      CALL DecompCopy( GhostIn%Border, GhostOut%Border )      CPP_LEAVE_PROCEDURE( "GHOSTCOPY" )      RETURN!EOC      END SUBROUTINE GhostCopy!-----------------------------------------------------------------------!-----------------------------------------------------------------------!BOP! !IROUTINE: GhostIrregular --- Create a ghost definition for 1-D grid!! !INTERFACE:      SUBROUTINE GhostIrregular( Decomp, Id, LocalSize, Tags, Ghost )! !USES:      USE decompmodule, ONLY : DecompCreate, DecompCopy,                 &     &                         DecompGlobalToLocal, DecompInfo      IMPLICIT NONE!! !INPUT PARAMETERS:      TYPE(DecompType), INTENT( IN ) :: Decomp     ! Decomp information      INTEGER, INTENT( IN )          :: Id         ! Local PE identifer      INTEGER, INTENT( IN )          :: LocalSize  ! Size of local segment      INTEGER, INTENT( IN )          :: Tags(:)    ! Global tags!! !OUTPUT PARAMETERS:      TYPE(GhostType), INTENT( OUT ) :: Ghost  ! Ghost definition!!! !DESCRIPTION:!     Creates a ghost definition for a ghosted array given by!     the PEs and Tags of the local points.  Note that none of the !     array bounds can be outside the global domain!!! !SYSTEM ROUTINES:!     ALLOCATE, DEALLOCATE!! !REVISION HISTORY:!   00.11.12   Sawyer     Creation!! !BUGS:!   None of the array bounds can be outside of the global domain!!   This is significant if the local region is on the edge of the!   domain, and, in other words, the ghost region cannot cover!   empty space.  This limitation may be relaxed in the future.!!EOP!-----------------------------------------------------------------------!BOC! !LOCAL VARIABLES:      INTEGER :: I, NPEs, GlobalSize, Local, Count, Ipe      INTEGER, ALLOCATABLE :: Pe(:), Other(:)!!      CPP_ENTER_PROCEDURE( "GHOSTIRREGULAR" )!! Allocate the basic data structures!      CALL DecompInfo( Decomp, Npes, GlobalSize )      ALLOCATE( Pe( LocalSize ) )      ALLOCATE( Other( LocalSize ) )!! Use decompmodule to create global and local portions of Ghost! The local version is only on the local processor "0"       Other = Id      CALL DecompCreate( Npes, Other, LocalSize, Tags, Ghost%Local )!! Perform over all points local segment!      Count = 0      DO I= 1, LocalSize        CALL DecompGlobalToLocal( Decomp, Tags(I), Local, Ipe )        CPP_ASSERT_F90( (Local .GT. 0) .AND. (ipe .GE. 0) )        IF ( Ipe .ne. id ) THEN          Count = Count + 1          Other( Count ) = Tags(I)          Pe( Count )    = Ipe        ENDIF      ENDDO!! Define the border regions.  Presumably Count << LocalSize!      CALL DecompCreate( Npes, Pe, Count, Other, Ghost%Border )!! Copy the decomposition too!      CALL DecompCopy( Decomp, Ghost%Decomp )! Clean up      DEALLOCATE( Pe )      DEALLOCATE( Other )      CPP_LEAVE_PROCEDURE( "GHOSTIRREGULAR" )      RETURN!EOC      END SUBROUTINE GhostIrregular!-----------------------------------------------------------------------!-----------------------------------------------------------------------!BOP! !IROUTINE: GhostRegular1D --- Create a ghost definition for 1-D grid!! !INTERFACE:      SUBROUTINE GhostRegular1D( Decomp, Id, Xglobal, Xfrom, Xto, Xwrap, &     &                           Ghost )! !USES:      USE decompmodule, ONLY : DecompCreate, DecompCopy,                 &     &                         DecompGlobalToLocal, DecompInfo      IMPLICIT NONE!! !INPUT PARAMETERS:      TYPE(DecompType), INTENT( IN )   :: Decomp ! Decomp information      INTEGER, INTENT( IN )            :: Id     ! Local PE identifer      INTEGER, INTENT( IN )            :: Xglobal! Total in X      INTEGER, INTENT( IN )            :: Xfrom  ! Low index in X      INTEGER, INTENT( IN )            :: Xto    ! High index in X      LOGICAL, INTENT( IN )            :: Xwrap  ! Wrap in X?!! !OUTPUT PARAMETERS:p      TYPE(GhostType), INTENT( OUT )   :: Ghost  ! Ghost definition!!! !DESCRIPTION:!     Creates a ghost definition for a regular 1-D array with the!     array bounds Xfrom:Xto.!!     If the array bounds are outside of the global domain they may!     be wrapped around back into the global domain (variable Xwrap).  !     If the region is not wrapped, it is advisable that the ghost !     region end at the boundary (which usually requires!     special case treatment depending on the PE number). If !     it does not end at the boundary, undefined points are !     introduced.!! !SYSTEM ROUTINES:!     ALLOCATE, DEALLOCATE!! !REVISION HISTORY:!   00.11.12   Sawyer     Creation!! !BUGS:!!   There are certain limitations to ghost regions which can be!   avoided by clean programming practices.  If the ghosted region!   wraps back onto core regions of the same PE, problems can arise.  !   The simple case -- a ghosted region on 1 PE -- is supported in!   most cases.  However, if it wraps back onto the local PE !   in such a way that more than one ghost points is mapped to!   one core domain global index, then the code may fail.  Note!   that this is rarely the case if the ghost regions are small!   and enough processors are used to avoid wrapping back on the!   local one.!   !EOP!-----------------------------------------------------------------------!BOC! !LOCAL VARIABLES:      INTEGER :: I, L, NPEs, GlobalSize, LocalSize, Count, Local, Ipe      INTEGER :: Global      INTEGER, ALLOCATABLE :: Pe(:), Tags(:), Other(:)!!      CPP_ENTER_PROCEDURE( "GHOSTREGULAR1D" )!! Allocate the basic data structures!      CALL DecompInfo( Decomp, NPEs, GlobalSize )      CPP_ASSERT_F90( GlobalSize .EQ. Xglobal )      LocalSize = Xto - Xfrom + 1      CPP_ASSERT_F90( LocalSize .GE. 0 )      ALLOCATE( Pe( LocalSize ) )      ALLOCATE( Tags( LocalSize ) )      ALLOCATE( Other( LocalSize ) )!! Perform over all points local segment!      Count = 0      L = 0      DO I = Xfrom, Xto        L = L + 1        Global = MODULO(I-1,Xglobal)+1  ! Wrap around condition        IF (Xwrap .OR. Global==I) THEN          Tags(L) = Global                 ! Global Tags          CALL DecompGlobalToLocal( Decomp, Global, Local, Ipe )          IF ( Ipe .ne. Id .AND. Ipe .GE. 0 ) THEN            Count = Count + 1            Other( Count ) = Global        ! Local Tags            Pe( Count )    = Ipe          ENDIF!! Special case: the domain wraps-around onto the same PE.  This is! very tricky:  the ghost points are distinguished from their true! local core domain counterparts by a minus sign.  This makes the! address space in both Ghost%Border and Ghost%Local unique!          IF ( Ipe .eq. Id .AND. I .ne. Global ) THEN            Count = Count + 1            Other( Count ) = -Global       ! Local Tags            Pe( Count )    = Ipe            Tags(L) = -Global              ! Global Tags (mark ghost region!)          ENDIF        ELSE          Tags(L) = 0        ENDIF      ENDDO!! Perform over all points local segment!      CALL DecompCreate( Npes, Pe, Count, Other, Ghost%Border )!! Use decompmodule to create global and local portions of Ghost! The local version is only on the local PE!      Other = Id