/* smPktLib.c - VxWorks shared packet protocol library *//* Copyright 1984-2002 Wind River Systems, Inc. *//*modification history--------------------02m,03may02,mas  cache flush and volatile fix (SPR 68334); bridge flush fix		 (SPR 68844)02l,24oct01,mas  doc update (SPR 71149)02k,09oct01,mas  smPktSend: added error checking (SPR 6004), limited interrupt		 retries (SPR 25341), removed all FAST modifiers, code cleanup02j,17oct00,sn   replaced NULL with 0 so htonl works02i,21feb99,jdi  doc: listed errnos.02h,26aug93,kdl  smPktBroadcast() now checks smPktSllPut() status before 		 interrupting destination CPU  (SPR #2441).02g,13nov92,jcf  removed potential bus error during attach.02f,02aug92,kdl  Uncommented include of "copyright_wrs.h".02e,24jul92,elh  Moved heartbeat from anchor to header.02d,23jun92,elh  general cleanup and documentation.02c,02jun92,elh  the tree shuffle02b,27may92,elh	 Made completely independant, changed to use		 offsets, general cleanup.02a,14may92,pme  Split to keep only packet passing routines.01e,20feb92,elh  Added in ntohl, and htonl for 960, Added USE_OFFSET.		 Extracted and renamed OS specific calls & moved to		 smUtilLib.  Also removed function pointers		 shMemIntGenFunc and shMemHwTasFunc.		 made TAS_CHECKS 10 TAS_TRIES 5000.		 modified parameters passed to intGen function.01d,10feb92,kdl+ Changed shMemDetach to flush input queue.	    elh  Removed references to OK_WAS{EMPTY, NT_EMPTY}.		 Made shMemSend return an ERROR if packet too large.		 Changed shMemSend to return silently (no		 interrupts generated) if sending to self.		 Misc code review changes.01c,05feb92,elh  ansified.01b,27jan92,elh	 added masterCpu to shared memory anchor.		 changed shMemBeat to take pAnchor as argument.		 changed arguments to shMemIsAlive.		 changed shMemSetup to probe memory.		 changed copyright.01a,15aug90,kdl	 written.*//*DESCRIPTIONThis library contains routines which allow multiple processorsto communicate over a backplane using shared memory.  Allservices for initializing and managing the shared memory areprovided.Data is sent between CPU's via shared memory "packets".  A packetis simply a data buffer (of configurable size) with a headerthat allows it to be properly identified and manipulated as a nodein a linked list of packets.SHARED MEMORY MASTER CPUOne CPU node acts as the shared memory master.  This CPU initializesthe shared memory area and sets up the shared memory anchor.  Thesesteps are performed by the master calling the smPktSetup() routine.This routine should be called only by the shared memory master CPU,and it should only be called once for a given shared memory region.(It is, however, possible to maintain and use multiple separateshared memory regions.)Once smPktSetup() has completed successfully, there is little functionaldifference between the master CPU and other CPU's using shared memory,except that the master is responsible for maintaining the heartbeat inthe shared memory packet region header.SHARED MEMORY ANCHORThe shared memory anchor is a small data structure which is ata predetermined location, to allow all CPU's using shared memory tofind it.  The shared memory anchor contains the base offset to theactual shared memory region which will be used for inter-processormessage passing.  (This allows the master CPU to dynamically allocatethe shared memory region.)The shared memory anchor does not have to be defined as part of theshared memory region which will be used for message passing.  However,it must be located in a similar address space; the address translationconstants which CPU boards use to convert local addresses to bus addressesmust apply equally to the anchor and the regular shared memory region.ATTACHING TO SHARED MEMORYEach CPU, master or non-master, which will use the shared memory regionmust attach itself to the shared memory.  The shared memory region musthave already been initialized by the master CPU calling smPktSetup().The first step in attaching to shared memory is for each CPU to allocate andinitialize a shared memory packet descriptor (SM_PKT_DESC).  This structure describes the individual CPU's attachment to the shared memory region and is used in all subsequent shared memory calls to identify which shared memory region is being used.  Since the shared memory descriptor is used only by the local CPU, it is not necessary for the descriptor itself to be located in shared memory.  In fact, it is preferable for the descriptor to be allocated from the CPU's local memory, since local memory is usually more efficient to access.The shared memory packet descriptor is initialized by calling the smPktInit() routine.  This routine takes a number of parameters which specify the characteristics of this CPU and its access to shared memory.After the shared memory packet descriptor has been initialized, the CPU mayattach itself to the shared memory region.  This is done by calling thesmPktAttach() routine.When smPktAttach() is called, it checks that the shared memory anchor contains the special ready value and that the heartbeat is incrementing.If either of these conditions is not met, the routine will check periodically until either the ready value and incrementing heartbeat are recognized or a time limit is reached.  For non-master CPU's, this limit may be set by changing the global variable, smAliveTimeout.  The limit is expressed in seconds, with 600 seconds (10 minutes) the default.  The master CPU will only wait 5 beat periods for a recognized heartbeat, since it is the CPU responsible for initializing shared memory and incrementing the heartbeat and therefore should not have to wait.  If the time limit is reached before a valid ready value and heartbeat is seen, ERROR is returned and errno is set to S_smPktLib_DOWN .Once the CPU has attached itself to the shared memory region, it maysend packets to other CPU's or receive packets which have been sent to it.(Attempts to perform any shared memory operations without first attachingsuccessfully will return ERROR, with errno set to S_smPktLib_NOT_ATTACHED .)SENDING PACKETSTo send a packet to another CPU, an application task must first obtaina shared memory packet from the pool of free packets.  This isdone by calling smPktFreeGet(), as follows:\cs	status = smPktFreeGet (pSmPktDesc, &pPkt);\ceIn this example, <pSmPktDesc> is the address of this CPU's shared memory packetdescriptor.  If status value returned by smPktFreeGet is OK, the addressof the obtained packet will be placed in <pPkt>.  If the packet address isNULL, no free packets were available.Once a packet has been obtained, the application task must copy the datato be sent into the "data" field of the packet structure.  The maximumnumber of bytes which may be copied to the packet's data buffer is<maxPktBytes>, as specified by the master CPU during smPktSetup.  Thislimit may be determined by reading a field in the shared memory packet descriptor.The application task may set the "type" field in the packet header (SM_PKT_HDR)to indicate the specific type of packet being sent.  This field is not usedby smPktLib and may therefore be given any value.To send the completed packet, the application task calls smPktSend(),as follows:\cs	status = smPktSend (pSmPktDesc, destCpu, pPkt);\ceHere, <destCpu> is the number of the destination CPU, and <pPkt> is theaddress of the packet to be sent.  If smPktSend() returns a status of OK,the packet was successfully queued for the destination CPU.If the destination CPU did not previously have any input packets queued toit smPktSend() will call the user-provided routine, smUtilIntGen(), to interrupt the destination CPU to notify it that a packet is available.  See "Interrupts," below, for more information.If the destination CPU is not attached, ERROR is returned and errno is setto S_smPktLib_DEST_NOT_ATTACHED .  If the specified destination cpunumber is out of range (i.e. less than zero or greater than the maximumspecified during smPktSetup), ERROR is returned and errno is set toS_smPktLib_INVALID_CPU_NUMBER .If the first interrupt fails, a delay of one tick is made and anotherinterrupt is attempted.  This is repeated up to smPktMaxIntRetries attempts oruntil the interrupt is successful.  If not successful, ERROR is returnedand errno is set to S_smPktLib_INCOMPLETE_BROADCAST so that the callerdoes not try to remove the packet and place it on the free list.  Doingso would put the packet in both an input list and the free list!BROADCAST PACKETSIn some circumstances, it may be desirable to send the same data toall CPU nodes using the shared memory region.  This may be accomplishedby using a special "broadcast" mode of smPktSend().  This option sendsa copy of the same packet data to each attached CPU (except the sender).To send a broadcast message, a CPU must first obtain a free shared memorypacket using smPktFreeGet() as usual.  The data area of the packetis then filled with the message to be sent, again the same as for sendingto a single destination.Broadcast mode is indicated during smPktSend() by a special value of the<destCpu> (destination CPU) parameter.  This parameter is set toSM_BROADCAST, rather than a particular CPU number.When a broadcast message is sent, a separate packet is obtained for eachattached CPU, the data from the original packet is copied to it, andthe packet is queued to a particular CPU.  Therefore, there must besufficient free packets to send one to each CPU.  Broadcast packets aresent to destination CPU's in cpu-number order.If there are not enough free packets to send a copy to each CPU,as many as possible will be sent, but ERROR is returned and errno isset to S_smPktLib_INCOMPLETE_BROADCAST .  (If there are insufficientfree packets, the original packet passed during smPktSend() will besent to a destination CPU, to provide as complete a broadcast aspossible.)Broadcast packets are received in the same manner as any other packets.RECEIVING PACKETSPackets are received by calling the smPktRecv() routine.  This routinewill normally be called by an interrupt handler, in response to aan interrupt generated by a remote CPU sending a packet.  The smPktRecv()routine may also be called periodically, in a polling fashion, to checkfor received packets in systems which do not use interrupts to notifythe receiving CPU.To receive a packet, smPktRecv is called as follows:\cs	status = smPktRecv (pSmPktDesc, &pPkt);\ceIf the returned status is OK, <pPkt> will contain either the address ofa received packet, or NULL if no input packets were available.  A returnedstatus of ERROR indicates that an error occurred while attempting toobtain received packets.A sending CPU will interrupt the destination CPU only if therewere no packets previously queued for the destination CPU.  It istherefore important that an interrupt handler which receives packetscall smPktRecv again after each packet is received, to check foradditional packets.After a packet has been received, it must be explicitly freed by callingsmPktFreePut().DETACHING FROM SHARED MEMORYThe attachment of a CPU to shared memory may be ended by callingsmPktDetach().  This routine will mark the calling CPU as no longerattached.  After this, other CPU's may no longer send packets to it.The CPU may re-attach itself to the shared memory region by latercalling smPktAttach().  When re-attaching, the original sharedmemory descriptor may be re-used if the CPU's configuration remainsthe same, or new values may be specified via smPktInit().INTERRUPTSWhen a packet is sent to a CPU, there must be some method for that CPUto be informed that an input packet is available.  The preferred methodis for the sending CPU to be able to interrupt the receiving CPU.  Thiswill be highly dependent on the specific hardware being used.  Two types of interrupts are supported, mailbox interrupts and vmebus interrupts.  Mailbox interrupts are the first preferredmethod (SM_INT_MAILBOX), followed by vme bus interrupts (SM_INT_BUS).If interrupts cannot be used, a polling scheme may be employed(SM_INT_NONE), but this is much less efficient.When a CPU initailizes its shared memory packet descriptor via thesmPktInit() call, it passes in an interrupt type as well as threeinterrupt arguments.  This describes how the cpu wishes to be notifiedof incomming packets.  The interrupt types recognized by this libraryare listed in smLib.h.  These values may be obtained for any attached CPU bycalling smPktCpuInfoGet().The default interrupt method for a particular target is defined by theconfiguration parameters: SM_INT_TYPE, SM_INT_ARG1, SM_INT_ARG2, SM_INT_ARG3 .When a CPU sends a packet to a destination CPU which did not previouslyhave any input packets queued to it, the sending CPU will interrupt the destination CPU. A handler routine which executes in response to such an interruptmust call smPktRecv() to obtain the input packet.  Since the interruptis generated only for the first packet queued when the input list waspreviously empty, it is important that the handler routine then callsmPktRecv additional times until no more packets are available.If it is not possible to use interrupts to notify receiving CPU's, a polling method may be used.  The simplest method is for the recieving CPU torepeatedly call smPktRecv() to check for input packets.  In this case,no notification routine is used.OBTAINING STATUS INFORMATIONTwo routines are provided to obtain current status information aboutthe shared memory region and individual CPU's:The smPktInfoGet() routine gets status information which applies to thethe shared memory region as a whole.  It takes as a parameter a pointerto a special information structure (SM_PKT_INFO), which it fills beforereturning.  The smPktCpuInfoGet() routine obtains status information for a singleCPU.  The number of the CPU must be specified during the call tosmPktCpuInfoGet().  A CPU number of NONE (-1) indicates that informationon the calling CPU should be returned.  The routine takes as a parametera pointer to a special CPU information structure (SM_PKT_CPU_INFO), which it fills before returning.  INTERNALThis file runs under only vxWorks.  SunOS is no longer supported.Pointers into shared memory are declared 'volatile' so that compileroptimization does not disrupt ordering of I/O operations.Reading of locking semaphores after updates to shared memory but beforerelease of the semaphores is to cause the flushing of any external busbridge read and write FIFOs.  Failure to do so can result in erroneousreads of shared memory and subsequent deadlock conditions.*/#ifdef UNIX#undef INET#endif/* includes */#include "vxWorks.h"#include "sysLib.h"#include "taskLib.h"#include "cacheLib.h"#include "smPktLib.h"#include "smLib.h"#include "smUtilLib.h"/* defines */#define	SM_LOCK_TAKE(lockLocalAdrs,tasRoutine,numTries,pOldLvl) \		(smLockTake (lockLocalAdrs, tasRoutine, numTries, pOldLvl))#define	SM_LOCK_GIVE(lockLocalAdrs,tasClearRoutine, oldLvl) \		(smLockGive (lockLocalAdrs, tasClearRoutine, oldLvl))#define DEFAULT_TAS_TRIES	5000	/* default tries for test-and-set */#ifndef SM_PKT_MAX_INT_RETRIES# define SM_PKT_MAX_INT_RETRIES 5   /* max #retries of interrupt per pkt */#endif/* Globals */int	smPktMemSizeDefault 	= DEFAULT_MEM_SIZE;	/* memory size */int 	smPktMaxBytesDefault 	= DEFAULT_PKT_SIZE; 	/* max pkt size */int 	smPktMaxInputDefault	= DEFAULT_PKTS_MAX;	/* max input pkts */int 	smPktMaxCpusDefault     = DEFAULT_CPUS_MAX;	/* max num CPU's */int	smPktTasTries 	  	= DEFAULT_TAS_TRIES;				/* times to try test-and-set */int     smPktMaxIntRetries      = SM_PKT_MAX_INT_RETRIES;                                /* max #int retries to attempt per packet *//* Forward References */#ifdef __STDC__LOCAL STATUS smPktSllGet    (    SM_SLL_LIST volatile * listLocalAdrs,	/* local addr of packet list */    int                    baseAddr,		/* addr conversion constant */    FUNCPTR                tasRoutine,		/* test-and-set routine addr */    FUNCPTR                tasClearRoutine,	/* test-and-set routine addr */    SM_SLL_NODE **	   pNodeLocalAdrs	/* location to put node addr */    );LOCAL STATUS smPktSllPut    (    SM_SLL_LIST volatile * listLocalAdrs,	/* local addr of packet list */    int			   base,		/* base address */    FUNCPTR		   tasRoutine,		/* test-and-set routine addr */    FUNCPTR		   tasClearRoutine,	/* test-and-set routine addr */    SM_SLL_NODE volatile * nodeLocalAdrs,	/* local addr of node */    BOOL *		   pListWasEmpty	/* set to true if adding to                                                 * empty list */    );LOCAL STATUS smPktBroadcast    (    SM_PKT_DESC *	pSmPktDesc,	/* sh mem pkt descriptor */    volatile SM_PKT *	pPktOrig	/* ptr to original packet */    );#else	/* __STDC__ */LOCAL STATUS smPktSllGet ();LOCAL STATUS smPktSllPut ();LOCAL STATUS smPktBroadcast ();