/* may be lost, hence it may in theory	*/
    				/* be much too low.			*/
    				/* The count may also be too high if	*/
    				/* multiple mark threads mark the	*/
    				/* same object due to a race.		*/
    				/* Without parallel marking, the count	*/
    				/* is accurate.				*/
#   ifdef USE_MARK_BYTES
      union {
        char _hb_marks[MARK_BITS_SZ];
			    /* The i'th byte is 1 if the object 	*/
			    /* starting at granule i or object i is	*/
			    /* marked, 0 o.w.				*/
			    /* The mark bit for the "one past the	*/
			    /* end" object is always set to avoid a 	*/
			    /* special case test in the marker.		*/
	word dummy;	/* Force word alignment of mark bytes. */
      } _mark_byte_union;
#     define hb_marks _mark_byte_union._hb_marks
#   else
      word hb_marks[MARK_BITS_SZ];
#   endif /* !USE_MARK_BYTES */
};

# define ANY_INDEX 23	/* "Random" mark bit index for assertions */

/*  heap block body */

# define HBLK_WORDS (HBLKSIZE/sizeof(word))
# define HBLK_GRANULES (HBLKSIZE/GRANULE_BYTES)

/* The number of objects in a block dedicated to a certain size.	*/
/* may erroneously yield zero (instead of one) for large objects.	*/
# define HBLK_OBJS(sz_in_bytes) (HBLKSIZE/(sz_in_bytes))

struct hblk {
    char hb_body[HBLKSIZE];
};

# define HBLK_IS_FREE(hdr) (((hdr) -> hb_flags & FREE_BLK) != 0)

# define OBJ_SZ_TO_BLOCKS(sz) divHBLKSZ(sz + HBLKSIZE-1)
    /* Size of block (in units of HBLKSIZE) needed to hold objects of	*/
    /* given sz (in bytes).						*/

/* Object free list link */
# define obj_link(p) (*(void  **)(p))

# define LOG_MAX_MARK_PROCS 6
# define MAX_MARK_PROCS (1 << LOG_MAX_MARK_PROCS)

/* Root sets.  Logically private to mark_rts.c.  But we don't want the	*/
/* tables scanned, so we put them here.					*/
/* MAX_ROOT_SETS is the maximum number of ranges that can be 	*/
/* registered as static roots. 					*/
# ifdef LARGE_CONFIG
#   define MAX_ROOT_SETS 4096
# else
    /* GCJ LOCAL: MAX_ROOT_SETS increased to permit more shared */
    /* libraries to be loaded.                                  */ 
#   define MAX_ROOT_SETS 1024
# endif

# define MAX_EXCLUSIONS (MAX_ROOT_SETS/4)
/* Maximum number of segments that can be excluded from root sets.	*/

/*
 * Data structure for excluded static roots.
 */
struct exclusion {
    ptr_t e_start;
    ptr_t e_end;
};

/* Data structure for list of root sets.				*/
/* We keep a hash table, so that we can filter out duplicate additions.	*/
/* Under Win32, we need to do a better job of filtering overlaps, so	*/
/* we resort to sequential search, and pay the price.			*/
struct roots {
	ptr_t r_start;
	ptr_t r_end;
#	if !defined(MSWIN32) && !defined(MSWINCE)
	  struct roots * r_next;
#	endif
	GC_bool r_tmp;
	  	/* Delete before registering new dynamic libraries */
};

#if !defined(MSWIN32) && !defined(MSWINCE)
    /* Size of hash table index to roots.	*/
#   define LOG_RT_SIZE 6
#   define RT_SIZE (1 << LOG_RT_SIZE) /* Power of 2, may be != MAX_ROOT_SETS */
#endif

/* Lists of all heap blocks and free lists	*/
/* as well as other random data structures	*/
/* that should not be scanned by the		*/
/* collector.					*/
/* These are grouped together in a struct	*/
/* so that they can be easily skipped by the	*/
/* GC_mark routine.				*/
/* The ordering is weird to make GC_malloc	*/
/* faster by keeping the important fields	*/
/* sufficiently close together that a		*/
/* single load of a base register will do.	*/
/* Scalars that could easily appear to		*/
/* be pointers are also put here.		*/
/* The main fields should precede any 		*/
/* conditionally included fields, so that	*/
/* gc_inl.h will work even if a different set	*/
/* of macros is defined when the client is	*/
/* compiled.					*/

struct _GC_arrays {
  word _heapsize;		/* Heap size in bytes.			*/
  word _max_heapsize;
  word _requested_heapsize;	/* Heap size due to explicit expansion */
  ptr_t _last_heap_addr;
  ptr_t _prev_heap_addr;
  word _large_free_bytes;
	/* Total bytes contained in blocks on large object free */
	/* list.						*/
  word _large_allocd_bytes;
  	/* Total number of bytes in allocated large objects blocks.	*/
  	/* For the purposes of this counter and the next one only, a 	*/
  	/* large object is one that occupies a block of at least	*/
  	/* 2*HBLKSIZE.							*/
  word _max_large_allocd_bytes;
  	/* Maximum number of bytes that were ever allocated in		*/
  	/* large object blocks.  This is used to help decide when it	*/
  	/* is safe to split up a large block.				*/
  word _bytes_allocd_before_gc;
		/* Number of words allocated before this	*/
		/* collection cycle.				*/
# ifndef SEPARATE_GLOBALS
    word _bytes_allocd;
  	/* Number of words allocated during this collection cycle */
# endif
  word _bytes_finalized;
  	/* Approximate number of bytes in objects (and headers)	*/
  	/* That became ready for finalization in the last 	*/
  	/* collection.						*/
  word _non_gc_bytes_at_gc;
  	/* Number of explicitly managed bytes of storage 	*/
  	/* at last collection.					*/
  word _bytes_freed;
  	/* Number of explicitly deallocated bytes of memory	*/
  	/* since last collection.				*/
  word _finalizer_bytes_freed;
  	/* Bytes of memory explicitly deallocated while 	*/
  	/* finalizers were running.  Used to approximate mem.	*/
  	/* explicitly deallocated by finalizers.		*/
  ptr_t _scratch_end_ptr;
  ptr_t _scratch_last_end_ptr;
	/* Used by headers.c, and can easily appear to point to	*/
	/* heap.						*/
  GC_mark_proc _mark_procs[MAX_MARK_PROCS];
  	/* Table of user-defined mark procedures.  There is	*/
	/* a small number of these, which can be referenced	*/
	/* by DS_PROC mark descriptors.  See gc_mark.h.		*/

# ifndef SEPARATE_GLOBALS
    void *_objfreelist[MAXOBJGRANULES+1];
			  /* free list for objects */
    void *_aobjfreelist[MAXOBJGRANULES+1];
			  /* free list for atomic objs 	*/
# endif

  void *_uobjfreelist[MAXOBJGRANULES+1];
			  /* uncollectable but traced objs 	*/
			  /* objects on this and auobjfreelist  */
			  /* are always marked, except during   */
			  /* garbage collections.		*/
# ifdef ATOMIC_UNCOLLECTABLE
    void *_auobjfreelist[MAXOBJGRANULES+1];
# endif
			  /* uncollectable but traced objs 	*/

    word _composite_in_use;
   		/* Number of words in accessible composite	*/
		/* objects.					*/
    word _atomic_in_use;
   		/* Number of words in accessible atomic		*/
		/* objects.					*/
# ifdef USE_MUNMAP
    word _unmapped_bytes;
# endif

    size_t _size_map[MAXOBJBYTES+1];
    	/* Number of words to allocate for a given allocation request in */
    	/* bytes.							 */

# ifdef STUBBORN_ALLOC
    ptr_t _sobjfreelist[MAXOBJGRANULES+1];
# endif
  			  /* free list for immutable objects	*/
# ifdef MARK_BIT_PER_GRANULE
    short * _obj_map[MAXOBJGRANULES+1];
                       /* If not NIL, then a pointer to a map of valid  */
    		       /* object addresses.				*/
		       /* _obj_map[sz_in_granules][i] is 		*/
  		       /* i % sz_in_granules.				*/
  		       /* This is now used purely to replace a 		*/
  		       /* division in the marker by a table lookup.	*/
    		       /* _obj_map[0] is used for large objects and	*/
    		       /* contains all nonzero entries.  This gets us	*/
    		       /* out of the marker fast path without an extra 	*/
    		       /* test.						*/
#   define MAP_LEN BYTES_TO_GRANULES(HBLKSIZE)
# endif
#   define VALID_OFFSET_SZ HBLKSIZE
  char _valid_offsets[VALID_OFFSET_SZ];
				/* GC_valid_offsets[i] == TRUE ==> i 	*/
				/* is registered as a displacement.	*/
  char _modws_valid_offsets[sizeof(word)];
				/* GC_valid_offsets[i] ==>		  */
				/* GC_modws_valid_offsets[i%sizeof(word)] */
# ifdef STUBBORN_ALLOC
    page_hash_table _changed_pages;
        /* Stubborn object pages that were changes since last call to	*/
	/* GC_read_changed.						*/
    page_hash_table _prev_changed_pages;
        /* Stubborn object pages that were changes before last call to	*/
	/* GC_read_changed.						*/
# endif
# if defined(PROC_VDB) || defined(MPROTECT_VDB) || \
     defined(GWW_VDB) || defined(MANUAL_VDB)
    page_hash_table _grungy_pages; /* Pages that were dirty at last 	   */
				     /* GC_read_dirty.			   */
# endif
# if defined(MPROTECT_VDB) || defined(MANUAL_VDB)
    volatile page_hash_table _dirty_pages;	
			/* Pages dirtied since last GC_read_dirty. */
# endif
# if defined(PROC_VDB) || defined(GWW_VDB)
    page_hash_table _written_pages;	/* Pages ever dirtied	*/
# endif
# ifdef LARGE_CONFIG
#   if CPP_WORDSZ > 32
#     define MAX_HEAP_SECTS 4096 	/* overflows at roughly 64 GB	   */
#   else
#     define MAX_HEAP_SECTS 768		/* Separately added heap sections. */
#   endif
# else
#   ifdef SMALL_CONFIG
#     define MAX_HEAP_SECTS 128		/* Roughly 256MB (128*2048*1K)	*/
#   else
#     define MAX_HEAP_SECTS 384		/* Roughly 3GB			*/
#   endif
# endif
  struct HeapSect {
      ptr_t hs_start; size_t hs_bytes;
  } _heap_sects[MAX_HEAP_SECTS];
# if defined(MSWIN32) || defined(MSWINCE)
    ptr_t _heap_bases[MAX_HEAP_SECTS];
    		/* Start address of memory regions obtained from kernel. */
# endif
# ifdef MSWINCE
    word _heap_lengths[MAX_HEAP_SECTS];
    		/* Commited lengths of memory regions obtained from kernel. */
# endif
  struct roots _static_roots[MAX_ROOT_SETS];
# if !defined(MSWIN32) && !defined(MSWINCE)
    struct roots * _root_index[RT_SIZE];
# endif
  struct exclusion _excl_table[MAX_EXCLUSIONS];
  /* Block header index; see gc_headers.h */
  bottom_index * _all_nils;
  bottom_index * _top_index [TOP_SZ];
#ifdef ENABLE_TRACE
  ptr_t _trace_addr;
#endif
#ifdef SAVE_CALL_CHAIN
  struct callinfo _last_stack[NFRAMES];	/* Stack at last garbage collection.*/
  					/* Useful for debugging	mysterious  */
  					/* object disappearances.	    */
  					/* In the multithreaded case, we    */
  					/* currently only save the calling  */
  					/* stack.			    */
#endif
};

GC_API GC_FAR struct _GC_arrays GC_arrays; 

# ifndef SEPARATE_GLOBALS
#   define GC_objfreelist GC_arrays._objfreelist
#   define GC_aobjfreelist GC_arrays._aobjfreelist
#   define GC_bytes_allocd GC_arrays._bytes_allocd
# endif
# define GC_uobjfreelist GC_arrays._uobjfreelist
# ifdef ATOMIC_UNCOLLECTABLE
#   define GC_auobjfreelist GC_arrays._auobjfreelist
# endif
# define GC_sobjfreelist GC_arrays._sobjfreelist
# define GC_valid_offsets GC_arrays._valid_offsets
# define GC_modws_valid_offsets GC_arrays._modws_valid_offsets
# ifdef STUBBORN_ALLOC
#    define GC_changed_pages GC_arrays._changed_pages
#    define GC_prev_changed_pages GC_arrays._prev_changed_pages
# endif
# ifdef MARK_BIT_PER_GRANULE
#   define GC_obj_map GC_arrays._obj_map
# endif
# define GC_last_heap_addr GC_arrays._last_heap_addr
# define GC_prev_heap_addr GC_arrays._prev_heap_addr
# define GC_large_free_bytes GC_arrays._large_free_bytes
# define GC_large_allocd_bytes GC_arrays._large_allocd_bytes
# define GC_max_large_allocd_bytes GC_arrays._max_large_allocd_bytes
# define GC_bytes_finalized GC_arrays._bytes_finalized
# define GC_non_gc_bytes_at_gc GC_arrays._non_gc_bytes_at_gc
# define GC_bytes_freed GC_arrays._bytes_freed
# define GC_finalizer_bytes_freed GC_arrays._finalizer_bytes_freed
# define GC_scratch_end_ptr GC_arrays._scratch_end_ptr
# define GC_scratch_last_end_ptr GC_arrays._scratch_last_end_ptr
# define GC_mark_procs GC_arrays._mark_procs
# define GC_heapsize GC_arrays._heapsize
# define GC_max_heapsize GC_arrays._max_heapsize
# define GC_requested_heapsize GC_arrays._requested_heapsize
# define GC_bytes_allocd_before_gc GC_arrays._bytes_allocd_before_gc
# define GC_heap_sects GC_arrays._heap_sects
# define GC_last_stack GC_arrays._last_stack
#ifdef ENABLE_TRACE
#define GC_trace_addr GC_arrays._trace_addr
#endif
# ifdef USE_MUNMAP
#   define GC_unmapped_bytes GC_arrays._unmapped_bytes
# endif
# if defined(MSWIN32) || defined(MSWINCE)
#   define GC_heap_bases GC_arrays._heap_bases
# endif
# ifdef MSWINCE
#   define GC_heap_lengths GC_arrays._heap_lengths
# endif
# define GC_static_roots GC_arrays._static_roots
# define GC_root_index GC_arrays._root_index
# define GC_excl_table GC_arrays._excl_table
# define GC_all_nils GC_arrays._all_nils
# define GC_top_index GC_arrays._top_index
# if defined(PROC_VDB) || defined(MPROTECT_VDB) || \
     defined(GWW_VDB) || defined(MANUAL_VDB)
#   define GC_grungy_pages GC_arrays._grungy_pages
# endif
# if defined(MPROTECT_VDB) || defined(MANUAL_VDB)
#   define GC_dirty_pages GC_arrays._dirty_pages
# endif
# if defined(PROC_VDB) || defined(GWW_VDB)
#   define GC_written_pages GC_arrays._written_pages
# endif
# define GC_composite_in_use GC_arrays._composite_in_use
# define GC_atomic_in_use GC_arrays._atomic_in_use
# define GC_size_map GC_arrays._size_map

# define beginGC_arrays ((ptr_t)(&GC_arrays))
# define endGC_arrays (((ptr_t)(&GC_arrays)) + (sizeof GC_arrays))

#define USED_HEAP_SIZE (GC_heapsize - GC_large_free_bytes)

/* Object kinds: */
# define MAXOBJKINDS 16

extern struct obj_kind {