{   bi_count *ptr;   int       space_used;   if ((ng[0]>=max_words) || (ng[1]>=max_words)) {      /* Something's gone wrong */      HError(17093, "bigram_add: Found a word id higher than the base+number of words - all word ids are expected to be allocated in an unbroken chunk.\n[Current bigram is (%d,%d). Number of words is %d]", ng[0], ng[1], max_words);   }   /* Keep backward count */   backward[ng[1]].size++;   if (ng[0] == last_word) {      /* Make sure there's room in the buffer */      if (store_idx >= curr_bistore_size) {         /* Expand bigram buffer store to cope */         curr_bistore_size += bigram_buffer_grow;         if (trace & T_MEM) {            printf("Expanding bigram read buffer to %d entries\n", curr_bistore_size);         }         store = realloc(store, curr_bistore_size*sizeof(bi_count));      }      /* Store in buffer */      store[store_idx].id = ng[1];      store[store_idx].count = count;      store_idx++;      return;   }   /* Otherwise we must have just gone on to a new word, so keep the old      details */   forward[last_word].size = store_idx;   space_used = store_idx*sizeof(bi_count);   ptr = get_space(space_used);   memcpy(ptr, store, space_used);   forward[last_word].bi = ptr;   /* And go on to the next entry */   last_word = ng[0];   store[0].id = ng[1];   store[0].count = count;   store_idx = 1;}/* Call when all bigrams have been passed in */void bigram_added_all(void){   bi_count *ptr;   int       space_used;   int       i, j, backward_id;   /* Store last set of details */   forward[last_word].size = store_idx;   space_used = store_idx*sizeof(bi_count);   ptr = get_space(space_used);   memcpy(ptr, store, space_used);   forward[last_word].bi = ptr;   free(store);   sum_of_all_bigram_counts = 0;   /* Generate backward lookup table */   if (trace & T_EXTRA) {      printf("Building bigram backward lookup table...");      fflush(stdout);   }   /* Allocate required storage space */   for (i=0; i<max_words; i++) {      backward[i].bi = get_space(backward[i].size * sizeof(bi_count));      backward[i].size = 0; /* Reset to use as counter when building data */   }   /* Run through all forward data, copying into backward array */   for (i=0; i<max_words; i++) {      for (j=0; j<forward[i].size; j++) {         backward_id = forward[i].bi[j].id;         backward[backward_id].bi[backward[backward_id].size].id = i;         backward[backward_id].bi[backward[backward_id].size].count                                            = forward[i].bi[j].count;         backward[backward_id].size++;         sum_of_all_bigram_counts += forward[i].bi[j].count;      }   }   if (trace & T_EXTRA) {      printf(" done\n");   }}/* Must be called before almost any other function in this file will work */void bigram_init(int words) /* Pass ->used field from word-map */{   max_words = words;   forward = CNew(&global_stack, words * sizeof(bigrams));   backward = CNew(&global_stack, words * sizeof(bigrams));   if (trace & T_MEM) {      printf("Bigram store for %d words created\n", words);   }   last_word = 0;   store_idx = 0;   curr_bistore_size = initial_bigram_buffer;   store = calloc(initial_bigram_buffer, sizeof(bi_count));   if (trace & T_MEM) {      printf("Bigram read buffer of %d entries created\n", initial_bigram_buffer);   }}/* Main program control function */int main(int argc, char *argv[]){   char *s;   float weight;   /* used when loading gram files */   char *filename; /* used when loading gram files */   int   iterations=1, loop;   char *init_cmap = NULL;   char *recover_from = NULL;   char *write_classprobs = NULL;   char *write_classcounts = NULL;   Boolean read_gram_files=FALSE; /* Has the user passed any gram files? */   Boolean set_classes = FALSE, loaded_map = FALSE; /* Check for -c and -l */   Boolean keep_unk_sep = FALSE; /* Was -k passed? */   Boolean passed_unk = FALSE; /* Unknown word was passed in */   int start_word_id, end_word_id, unknown_word_id;   int numb_classes, min_classes;   char *ptr, *ptr2; /* temp results */   /* Initialise HTK/HLM modules */   InitShell(argc, argv, Cluster_version, Cluster_vc_id);   InitMem();   InitMath();   InitWave();   InitLabel();   InitLUtil();   InitWMap();   InitGBase();   SetConfParms();   /* Default start, end and unknown words */   strcpy(sent_start, DEF_STARTWORD);   strcpy(sent_end, DEF_ENDWORD);   strcpy(unknown_w, DEF_UNKNOWNNAME);   /* Default number of classes */   numb_classes = classes_get_default();   /* Parse command line */   if (!InfoPrinted() && NumArgs() == 0)      ReportUsage();   if (NumArgs() == 0)      Exit(EXIT_FAILURE);   /* Create a global stack and heap */   CreateHeap(&global_stack, "Clusterer stack", MSTAK, 1, 0.0, 8192, 8192);   CreateHeap(&global_heap, "Clusterer heap", MHEAP, block_grab_size, 0.0, 1, 1);   while (NextArg() == SWITCHARG) {      s = GetSwtArg();      if (strlen(s) !=1 )         HError(17019, "Cluster: Bad switch %s; must be single letter",s);      switch(s[0]) {         case 'c':            if (NextArg()!=INTARG)               HError(17019,"Cluster: number of categories expected for -c");	    numb_classes = GetIntArg();            classes_set_number(numb_classes);	    set_classes = TRUE;            break;         case 'i':            if (NextArg()!=INTARG)               HError(17019,"Cluster: number of iterations expected for -i");            iterations = GetIntArg();            break;          case 'r':            if (NextArg()!=INTARG)               HError(17019,"Cluster: recovery export frequency expected for -r");            rec_freq = GetIntArg();            break;         case 'm':	    classes_showMLV(1);	    break;         case 'o':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: output filename prefix expected for -o");            set_output_prefix(GetStrArg());            break;         case 'p':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: output filename expected for -p");            write_classprobs = GetStrArg();            break;         case 'q':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: output filename expected for -q");            write_classcounts = GetStrArg();            break;         case 'l':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: output filename prefix expected for -l");            init_cmap = GetStrArg();	    loaded_map = TRUE;            break;         case 's':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: sentence start word expected for -s");            strcpy(sent_start, GetStrArg());            break;         case 't':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: sentence end word expected for -t");            strcpy(sent_end, GetStrArg());            break;         case 'u':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: unknown word token expected for -u");            strcpy(unknown_w, GetStrArg());	    passed_unk = TRUE;            break;         case 'x':            if (NextArg()!=STRINGARG)               HError(17019,"Cluster: recovery filename expected for -x");            recover_from = GetStrArg();            break;         case 'w':            if (NextArg()!=STRINGARG)               HError(17019, "Cluster: wordmap sort order expected for -w");            strcpy(tmp, GetStrArg());            for (ptr=tmp; *ptr!=0; *ptr=toupper(*ptr), ptr++);            if (strcmp(tmp, "WMAP")==0) {               sort_order = SORT_WMAP;            }            else if (strcmp(tmp, "FREQ")==0) {               sort_order = SORT_FREQ;            }            else {               HError(17019, "Cluster: -w expects either WMAP or FREQ");            }            break;         case 'k':	    classes_keep_unk_separate(TRUE);	    keep_unk_sep = TRUE;            break;         case 'v':	    verbose = TRUE;            break;         case 'n':	    write_logfile = !write_logfile;            break;         case 'T':            trace = GetChkedInt(0,017,s); break;         default:            HError(17019,"Cluster: Unknown switch %s",s);      }   }   if (NextArg()!=STRINGARG)      HError(17019, "Cluster: word map file name expected");   CreateWordMap(GetStrArg(), &wmap, 0);   min_classes = 4 + (keep_unk_sep?1:0); /* Minimum number of classes */   if (loaded_map && set_classes) {      HError(-17019, "Ignoring -c option: when combined with -l the number of classes in the existing map must be used");   }   else if (numb_classes < min_classes) {      HError(17019, "It doesn't make sense to specify less than %d classes -\n    %d classes are reserved, and you need at least 2 more", min_classes, min_classes-2);   }   /* See if start and end word occur in the data */   if (!GetLabId(sent_start, FALSE)) {      HError(17051, "Sentence start token '%s' not in word list");   }   if (!GetLabId(sent_end, FALSE)) {      HError(17051, "Sentence end token '%s' not in word list");   }   /* We can't keep the unknown word in its own class if one wasn't passed */   if (!GetLabId(unknown_w, FALSE) && keep_unk_sep) {      HError(17051, "Unknown word token '%s' not in word list and -k passed", unknown_w);   }   /* And generate a sensible warning if necessary: */   if (!GetLabId(unknown_w, FALSE) && passed_unk) {      HError(-17051, "Unknown word token '%s' was explicitly given with -u, but does not occur in the word map", unknown_w);   }   start_word_id = GetMEIndex(&wmap, (((MapEntry *)(GetLabId(sent_start, FALSE)->aux))->ndx));   end_word_id = GetMEIndex(&wmap, (((MapEntry *)(GetLabId(sent_end, FALSE)->aux))->ndx));   if (keep_unk_sep) {      unknown_word_id = GetMEIndex(&wmap, (((MapEntry *)(GetLabId(unknown_w, FALSE)->aux))->ndx));   }   else {      unknown_word_id = 0;   }   set_ids(start_word_id, end_word_id, unknown_word_id);   /* If we're doing no iterations we want to ignore the given filename      prefix and use the one from the classmap - this way we'll write the      correct information into the saved probabilities file header */   if (iterations==0 && init_cmap) {      ptr = strrchr(init_cmap, '.');      if (ptr) {         *ptr = '\0';         ptr2 = strrchr(init_cmap, '.');         if (ptr2) {            *ptr2 = '\0';            set_output_prefix(init_cmap);            *ptr2 = '.';         }         else            set_output_prefix(init_cmap);         *ptr = '.';      }      else {         set_output_prefix(init_cmap);      }   }   if (trace & T_FILE) {      printf("Wordmap loaded - %d words\n", wmap.used);   }   unigram_init(wmap.used);   bigram_init(wmap.used);   /* Add input gram files to input set */   if (trace & T_TOP)      printf("Preparing input gram set\n");   CreateHeap(&imem, "inputset", MSTAK, 1, 0.0, 1000, 1000);   CreateHeap(&imem2, "inputset2", MSTAK, 1, 0.0, 1000, 1000);   CreateInputSet(&imem, &wmap, &inset);   CreateInputSet(&imem2, &wmap, &inset2);   weight = 1.0;   while (NextArg() == STRINGARG || NextArg() == FLOATARG) {      if (NextArg() == FLOATARG) {         weight = GetFltArg();      }      if (weight==0.0 || weight<-10000.0 || weight>10000.0) {         HError(17019, "Improbable gram file weight (%.4f)", weight);      }      if (NextArg()!=STRINGARG) {         HError(17019,"Gram file name expected");      }      filename = GetStrArg();      AddInputGFile(&inset, filename, weight);      AddInputGFile(&inset2, filename, weight);      read_gram_files = TRUE;      if (trace & T_TOP)         printf("Input gram file %s added (weight=%f)\n", filename, weight);   }   if (!read_gram_files) {      HError(17019, "No gram files passed");   }   LoadBiGrams();   LoadUniGrams();   bigram_added_all();   DeleteHeap(&imem);   DeleteHeap(&imem2);   if (init_cmap) {      import_classmap(init_cmap, wmap.used);   }   else if (recover_from) {      do_recovery(recover_from, wmap.used);   }   /* Allocate memory and compute bigram pair arrays */   if (!recover_from) {      classes_init(wmap.used);      /* Perform default initial clustering */      if (!init_cmap) {         initial_cluster();      }      /* Calculate initial counts required */      setup_all_counts();   }   /* Run clustering algorithm */   for (loop=0; loop<iterations; loop++) {      cluster_words(1);      export_classes(0);   }   if (write_classprobs) {      write_word_probs(write_classprobs);   }   if (write_classcounts) {      write_word_counts(write_classcounts);   }   if (trace &