/*==================================================================== * ------------------------ * | CVS File Information | * ------------------------ * * $RCSfile: az_comm.c,v $ * * $Author: tuminaro $ * * $Date: 2000/06/02 16:46:55 $ * * $Revision: 1.38 $ * * $Name:  $ *====================================================================*/#ifndef lintstatic char rcsid[] = "$Id: az_comm.c,v 1.38 2000/06/02 16:46:55 tuminaro Exp $";#endif/******************************************************************************* * Copyright 1995, Sandia Corporation.  The United States Government retains a * * nonexclusive license in this software as prescribed in AL 88-1 and AL 91-7. * * Export of this program may require a license from the United States         * * Government.                                                                 * ******************************************************************************//* System Include files */#include <stdlib.h>#include <stdio.h>#include <math.h>#include "az_aztec.h"#ifndef TRUE#define TRUE  1#define FALSE 0#endifint AZ_sys_msg_type = AZ_MSG_TYPE;/******************************************************************************//******************************************************************************//******************************************************************************/void AZ_exchange_bdry(double x[], int data_org[], int proc_config[])/*******************************************************************************  Routine to locally exchange the components of the vector "x". This routine  gathers the necessary components of the vector and then sends the required  "num_neighbors" messages. The messages which are received are placed  contiguously in the external_nodes part of x.  Author:          John N. Shadid, SNL, 1421  =======  Return code:     void  ============  Parameter list:  ===============  x:               Vector of unknowns defined on the current processor.                   Indirect addressing will be used to gather those unknowns                   that other processors need.  data_org:        Array containing information on the distribution of the                   matrix to this processor as well as communication parameters                   (see Aztec User's Guide).*******************************************************************************/{  /* local variables */  char *message_send_add[AZ_MAX_NEIGHBORS];  /* message_send_add[i] points to the beginning of the list of values to be     sent to the ith neighbor (i.e. data_org[AZ_neighbors+i] or sometimes     locally defined as proc_num_neighbor[i]). That is, *(message_send_add[i]+j)     is the jth value to be sent to the ith neighbor. */  unsigned int  message_send_length[AZ_MAX_NEIGHBORS];  /* message_send_length[i] is the number of bytes to be sent to the ith     neighbor (i.e. data_org[AZ_neighbors+i] or sometimes locally defined as     proc_num_neighbor[i]). */  char *message_recv_add[AZ_MAX_NEIGHBORS];  /* message_recv_add[i] points to the beginning of the list of locations which     are to receive values sent by the ith neighbor (i.e.     data_org[AZ_neighbors+i] or sometimes locally defined as     proc_num_neighbor[i]). That is, *(message_recv_add[i] + j) is the location     where the jth value sent from the ith neighbor will be stored. */  unsigned int  message_recv_length[AZ_MAX_NEIGHBORS];  /* message_recv_length[i] is the number of bytes to be sent to the ith     neighbor (i.e. data_org[AZ_neighbors+i] or sometimes locally defined as     proc_num_neighbor[i]). */  int              n;  double          *ptr_send_list, *ptr_recv_list;  register double *ptrd;  register int    *ptr_int, i;  int              size, num_send, num_recv;  int              type;  int              num_neighbors, *proc_num_neighbor, total_num_send_unknowns;  int             *num_unknowns_send_neighbor, *list_send_unknowns;  int              external_index, *num_unknowns_recv_neighbor;  /**************************** execution begins ******************************/  type            = AZ_sys_msg_type;  AZ_sys_msg_type = (AZ_sys_msg_type+1-AZ_MSG_TYPE) % AZ_NUM_MSGS + AZ_MSG_TYPE;#ifdef AZ_MPI   if ( proc_config[AZ_Comm_Set] != AZ_Done_by_User) {      printf("Error: Communicator not set. Use AZ_set_comm()\n");      printf("       (e.g. AZ_set_comm(proc_config,MPI_COMM_WORLD)).\n");      exit(1);  }#endif  num_neighbors              = data_org[AZ_N_neigh];  if (num_neighbors == 0) return;  proc_num_neighbor          = &data_org[AZ_neighbors];  total_num_send_unknowns    = data_org[AZ_total_send];  num_unknowns_send_neighbor = &data_org[AZ_send_length];  list_send_unknowns         = &data_org[AZ_send_list];  external_index             = data_org[AZ_N_internal] + data_org[AZ_N_border];  num_unknowns_recv_neighbor = &data_org[AZ_rec_length];  /* single processor case */  if (num_neighbors == 0) return;  /* Set up send messages: Gather send unknowns from "x" vector */  ptrd = (double *) AZ_manage_memory(data_org[AZ_total_send]*sizeof(double),                                     AZ_ALLOC, AZ_SYS, "comm buff", &n);  ptr_send_list = ptrd;  ptr_int = list_send_unknowns;  for (i = total_num_send_unknowns; i > 0; i--) {    *ptrd++ = x[*ptr_int++];  }  /* Define arrays for message passing */  ptr_recv_list = &x[external_index];  size = sizeof(double);  for (n = 0; n < num_neighbors; n++) {    num_send               = num_unknowns_send_neighbor[n];    num_recv               = num_unknowns_recv_neighbor[n];    message_send_add[n]    = (char *) ptr_send_list;    message_recv_add[n]    = (char *) ptr_recv_list;    message_send_length[n] = size * num_send;    message_recv_length[n] = size * num_recv;    ptr_send_list         += num_send;    ptr_recv_list         += num_recv;  }  AZ_exchange_local_info(num_neighbors, proc_num_neighbor, message_send_add,                         message_send_length, message_recv_add,                         message_recv_length, type, proc_config);} /* AZ_exchange_bdry *//******************************************************************************//******************************************************************************//******************************************************************************/void AZ_exchange_local_info(int num_neighbors, int proc_num_neighbor[],                            char *message_send_add[],                             unsigned int message_send_length[],                            char *message_recv_add[],                             unsigned int message_recv_length[],                            int type, int proc_config[])/*******************************************************************************  Routine to communicate between a small number of processors by first writing  all messages and then reading all messages. This generic routine can be called  with the standard addresses, message lengths and message types for a wide  variety of uses.  Author:          John N. Shadid, SNL, 1421  =======  Return code:     void  ============  Parameter list:  ===============  num_neighbors:             Total number of neighboring processors.  proc_num_neighbor:         Array containing the processor id for each of the                             current processor's neighbors.  message_send_add:          message_send_add[i] points to the beginning of                             the list of values to be sent to the ith neighbor                             (i.e. data_org[AZ_neighbors+i] or sometimes locally                             defined as proc_num_neighbor[i]). That is,                             *(message_send_add[i] + j) is the jth value to be                             sent to the ith neighbor.  message_send_length:       message_send_length[i] is the number of bytes to                             be sent to the ith neighbor (i.e.                             data_org[AZ_neighbors+i] or sometimes locally                             defined as proc_num_neighbor[i]).  message_recv_add:          message_recv_add[i] points to the beginning of the                             list of locations which are to receive values sent                             by the ith neighbor (i.e.  data_org[AZ_neighbors+i]                             or sometimes locally defined as                             proc_num_neighbor[i]). That is,                             *(message_recv_add[i] + j) is the location where                             the jth value sent from the ith neighbor will be                             stored.  message_recv_length:       message_recv_length[i] is the number of bytes to                             be sent to the ith neighbor (i.e.                             data_org[AZ_neighbors+i] or sometimes locally                             defined as proc_num_neighbor[i]).  type:                      message type used when exchanging information.*******************************************************************************/{  /* local declarations */  register int n;  int          rtype, st;  MPI_AZRequest request[AZ_MAX_NEIGHBORS];  /* Message handle */  /*********************** first executable statment *****************/  /* post receives for all messages */  for (n = 0; n < num_neighbors; n++) {    rtype = type;    (void) mdwrap_iread((void *) *(message_recv_add+n),                         *(message_recv_length+n), proc_num_neighbor+n, &rtype,                         request+n);  }  /* write out all messages */  for (n = 0; n < num_neighbors; n++) {    (void) mdwrap_write((void *) *(message_send_add+n),                         *(message_send_length+n), *(proc_num_neighbor+n),                         type, &st);  }  /* wait for all messages */  for (n = 0; n < num_neighbors; n++) {    rtype = type;    (void) mdwrap_wait((void *) *(message_recv_add+n),                        *(message_recv_length+n), proc_num_neighbor+n, &rtype,                        &st, request+n);  }} /* AZ_exchange_local_info *//******************************************************************************//******************************************************************************//******************************************************************************/void AZ_gather_mesg_info(double x[] ,int data_org[], char *message_recv_add[],                         char *message_send_add[], int message_recv_length[],                         int message_send_length[])/*******************************************************************************  Routine to locally exchange the components of the vector "x". This routine  gathers the necessary components of the vector and then sends the required  "num_neighbors" messages. The messages which are received are placed  contiguously in the external_nodes part of x.  Author:          John N. Shadid, SNL, 1421  =======  Return code:     void  ============  Parameter list:  ===============  x:                         Vector of unknowns defined on the current                             processor. Indirect addressing will be used to                             gather those unknowns that other processors need.  data_org:                  Array containing information on the distribution of                             the matrix to this processor as well as                             communication parameters (see Aztec User's Guide).  message_send_add:          message_send_add[i] points to the beginning of                             the list of values to be sent to the ith neighbor                             (i.e. data_org[AZ_neighbors+i] or sometimes locally                             defined as proc_num_neighbor[i]). That is,                             *(message_send_add[i] + j) is the jth value to be                             sent to the ith neighbor.  message_send_length:       message_send_length[i] is the number of bytes to                             be sent to the ith neighbor (i.e.                             data_org[AZ_neighbors+i] or sometimes locally                             defined as proc_num_neighbor[i]).  message_recv_add:          message_recv_add[i] points to the beginning of the                             list of locations which are to receive values sent                             by the ith neighbor (i.e.  data_org[AZ_neighbors+i]                             or sometimes locally defined as                             proc_num_neighbor[i]). That is,                             *(message_recv_add[i] + j) is the location where                             the jth value sent from the ith neighbor will be                             stored.  message_recv_length:       message_recv_length[i] is the number of bytes to                             be sent to the ith neighbor (i.e.                             data_org[AZ_neighbors+i] or sometimes locally                             defined as proc_num_neighbor[i]).  num_neighbors:             Total number of neighboring processors.  Important external definitions:  ===============================  Num_Neighbors:             Total number of neighboring processors.                             (type - int value)  Proc_Num_Neighbor[]:       Array containing the processor id for each of the                             current processor's neighbors.                             (type - int vector with fixed length                             AZ_MAX_NEIGHBORS)  Total_Num_Send_Unknowns:   Total number of unknowns which are sent to                             neighboring processors (size of mesg buff).                             (type - int value)  Num_Unknowns_Send_Neighbor[]:                             Vector containing the number of unknowns to be sent                             to each processor.                             (type - int vector with fixed length                             AZ_MAX_NEIGHBORS)  List_Send_Unknowns[]:      Vector of local node numbers for the unknowns to be                             sent to each neighbor.                             (type - pointer to variable length int vector)  Num_Unknowns_Recv_Neighbor[]:                             Array of number of unknowns to be received from                             each of the neighboring processors.                             (type - int vector with fixed length                             AZ_MAX_NEIGHBORS)*******************************************************************************/{  /* local variables */  double          *ptr_send_list, *ptr_recv_list;  register double *ptrd;  register int    *ptr_int, i;  int              size, num_send, num_recv;  int              n;  int              Num_Neighbors;  int             *Num_Unknowns_Recv_Neighbor, *Num_Unknowns_Send_Neighbor;  /*********************** first executable statement *****************/  Num_Neighbors              =  data_org[AZ_N_neigh];  Num_Unknowns_Recv_Neighbor = &data_org[AZ_rec_length];  Num_Unknowns_Send_Neighbor = &data_org[AZ_send_length];  /* single processor case */  if (Num_Neighbors == 0) return;  /* Set up send messages:  Gather send unknowns from "x" vector */  ptrd = (double *) AZ_manage_memory(data_org[AZ_total_send]*sizeof(double),                                     AZ_ALLOC, AZ_SYS, "ptrd", &n);  ptr_send_list = ptrd;  ptr_int = &data_org[AZ_send_list];  for (i = data_org[AZ_total_send]; i > 0; i--) {    *ptrd++ = x[*ptr_int++];  }  /* Define arrays for message passing */