#!/bin/sh## make-batch-counts --#	generate n-gram counts in batches## A list of data files is partitioned into batches, results from each of# which are deposited in a separate ngram-count file.## usage: make-batch-count file-list [batch-size [filter \#		[countdir [options]]]]## file-list	is a file containing a list of data files#		(lines starting with # are ignored)# batch-size	is the number of input files per batch# filter	is preprocessor filter to condition the data # countdir	is the directory where count files are deposited# options	are arguments passed on to ngram-count## $Header: /home/srilm/devel/utils/src/RCS/make-batch-counts,v 1.3 2004/03/01 05:45:57 stolcke Exp $#if [ $# -lt 1 ]; then	echo "usage: $0 file-list [batch-size [filter [countdir [options]]]]" >&2	exit 2fifilelist=$1batchsize=${2-10}filter=${3-/bin/cat}countdir=${4-./counts}case $# in1)	shift;;2)	shift; shift;;3)	shift; shift; shift;;4)	shift; shift; shift; shift;;esacoptions="$@"what=`basename $filelist .files`statsfile=$countdir/$what.statsinfiles=$countdir/$what.filesset -eif [ ! -d $countdir ]; then	mkdir $countdirfigrep -v '^#' $filelist | xargs -l$batchsize echo > $infilestrap 'rm -f $newfile; exit 1' 1 2 15fileno=1> $statsfilewhile read datafiles; do	newfile=$countdir/$what-$fileno.ngrams.gz	echo "counting in $newfile sources $datafiles" >&2		$filter $datafiles | \	ngram-count -text - \		-tag $newfile \		-sort \		-write-order 0 \		-write $newfile \		$options \		2>> $statsfile	fileno=`expr $fileno + 1`done < $infiles