%CLEVALS Classifier evaluation (feature size/learning curve), bootstrap possible% % 	E = CLEVALS(A,CLASSF,FEATSIZE,TRAINSIZES,NREPS,T,FID)%% INPUT%   A          Training dataset%   CLASSF     Classifier to evaluate%   FEATSIZE   Vector of feature sizes%                (default: 1:K, where K is the number of features in A)%   TRAINSIZES Vector of class sizes, used to generate subsets of A% 	           	 (default [2,3,5,7,10,15,20,30,50,70,100])%   NREPS 		 Number of repetitions (default 1)%   T      		 Tuning set, or 'bootstrap' (default [], i.e. use remaining%             		objects in A)%		FID				 File ID to write progress to (default 0, i.e. no report)%% OUTPUT%   E       	 Error structure (see PLOTR)%% DESCRIPTION % Generates at random, for all feature sizes defined in FEATSIZES or all% class sizes defined in TRAINSIZES, training sets out of the dataset A and% uses these for training the untrained classifier CLASSF. CLASSF may also% be a cell array of untrained classifiers; in this case the routine will be% run for all of them. The resulting trained classifiers are tested on all% objects in A. This procedure is then repeated N times.%% Training set generation is done "with replacement" and such that for each% run the larger training sets include the smaller ones and that for all% classifiers the same training sets are used.% % If CLASSF is fully deterministic, this function uses the RAND random% generator and thereby reproduces if its seed is reset (see RAND). % If CLASSF uses RANDN, its seed may have to be set as well.%% Use FID = 1 to report progress to the command window.% % EXAMPLES% See PREX_CLEVAL.%% SEE ALSO% MAPPINGS, DATASETS, CLEVALB, TESTC, PLOTR% Copyright: R.P.W. Duin, duin@ph.tn.tudelft.nl% Faculty of Applied Sciences, Delft University of Technology% P.O. Box 5046, 2600 GA Delft, The Netherlands% $Id: clevals.m,v 1.3 2003/10/19 14:42:30 bob Exp $function e = clevals(a,classf,featsizes,learnsizes,nreps,t,fid) 	prtrace(mfilename);  if (nargin < 7)		fid = 0; 	end;	if (nargin < 6)		prwarning(2,'no tuning set supplied, bootstrapping');		t = [];	end;  if (nargin < 5)		prwarning(2,'number of repetitions not specified, assuming NREPS = 1');		nreps = 1; 	end;	% If a single mapping is given, convert it to a 1 x 1 cell array.  if (ismapping(classf)), classf = {classf}; end	% Correct for old argument order.  if (isdataset(classf)) & (ismapping(a))   	tmp = a; a = classf; classf = {tmp};  end  if (isdataset(classf)) & (iscell(a)) & (ismapping(a{1}))   	tmp = a; a = classf; classf = tmp;  end  if ~iscell(classf), classf = {classf}; end	% Assert that all is right.  isdataset(a); ismapping(classf{1});		% Remove requested class sizes that are larger than the size of the	% smallest class.  mc = classsizes(a); [m,k,c] = getsize(a);	% Defaults for size arrays.	if (nargin < 4) 		prwarning(2,'vector of training set class sizes not specified, assuming [2,3,5,7,10,15,20,30,50,70,100]'); 		learnsizes = [2,3,5,7,10,15,20,30,50,70,100]; 	end;  if (nargin < 3)		prwarning(2,'vector of feature sizes not specified, assuming K'); 		featsizes = k;	end;	learncurve = 0; featcurve = 0;	if (isempty(featsizes))									% Learning curve.		toolarge = find(learnsizes >= min(mc));		if (~isempty(toolarge))			prwarning(2,['training set class sizes ' num2str(learnsizes(toolarge)) ...									 ' larger than the minimal class size in A; removed them']);		  learnsizes(toolarge) = [];		end		learnsizes = learnsizes(:)';		featsizes  = k;		sizes = learnsizes;		learncurve = 1;	else																		% Feature size curve.		toolarge = find(featsizes > k);		if (~isempty(toolarge))			prwarning(2,['feature sizes ' num2str(featsizes(toolarge)) ...									 ' larger than number of features in A; removed them']);		  featsizes(toolarge) = [];		end		if (max(size(learnsizes)) > 1)			error('For a feature size curve, specify a scalar LEARNSIZE.');		end;	  featsizes = featsizes(:)';		sizes = featsizes;		featcurve = 1;	end;	% Fill the error structure.  nw = length(classf(:));  datname = getname(a);  e.error   = zeros(nw,length(sizes));  e.std     = zeros(nw,length(sizes));  e.xvalues = sizes(:)';  e.n       = nreps;  e.names   = [];  if (nreps > 1)  	e.ylabel= ['Averaged error (' num2str(nreps) ' experiments)'];  elseif (nreps == 1)  	e.ylabel = 'Error';  else		error('Number of repetitions NREPS should be >= 1.');	end;  if (~isempty(datname))		if (isempty(t))			 	e.title = ['Bootstrapped learning curve on ' datname];		else			 	e.title = ['Learning curve on ' datname];	 	end	end; 	if (learncurve)   	e.xlabel = 'Training set size'; 	else 		e.xlabel = 'Feature size'; 	end;  if (sizes(end)/sizes(1) > 20)  	e.plot = 'semilogx'; 				% If range too large, use a log-plot for X.  end  	% Report progress.	if (isempty(t))		if (learncurve)		  fprintf(fid,['\nBootstrapped classifier evaluation (learning curve): \n' ...  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));		else		  fprintf(fid,['\nBootstrapped classifier evaluation (feature size curve): \n' ...  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));		end	else		if (featcurve)		  fprintf(fid,['\nClassifier evaluation (learning curve): \n' ...  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));		else		  fprintf(fid,['\nClassifier evaluation (feature size curve): \n' ...  		    '    %i classifiers, %i repetitions, %i sizes ['],nw,nreps,length(sizes));		end	end  fprintf(fid,' %i ',sizes)  fprintf(fid,']\n');	% Store the seed, to reset the random generator later for different	% classifiers.  seed = rand('state');	% Loop over all classifiers (with index WI).  for wi = 1:nw		% Assert that CLASSF{WI} is an untrained mapping.  	isuntrained(classf{wi});  	name = getname(classf{wi});    fprintf(fid,'Classifier: %s\n',name); e.names = char(e.names,name);		% E1 will contain the error estimates.  	e1 = zeros(nreps,length(sizes));		% Take care that classifiers use same training set.  	rand('state',seed); seed2 = seed;		% For N repetitions...  	for i = 1:nreps			if (isempty(t))  			% Bootstrap. Store the randomly permuted indices of samples of class  			% CI to use in this training set in JR(CI,:).    		JR = zeros(c,max(learnsizes));    		for ci = 1:c    			JC = findnlab(a,ci);  				  				% Necessary for reproducable training sets: set the seed and store  				% it after generation, so that next time we will use the previous one.    			rand('state',seed2);		    			R = ceil(rand(1,max(learnsizes))*length(JC));    			JR(ci,:) = JC(R)';    			seed2 = rand('state');     		end					t = a;			end  		% Either the outer loop or the inner loop will be traversed just once,			% depending on whether we want to find a learning curve or feature			% size curve.  		ii = 0;    	for li = 1:length(learnsizes)  			for fi = 1:length(featsizes)  				ii = ii + 1;    			% Ja will contain the indices for this training set, Jt for					% the tuning set.    			Ja = [];       		for ci = 1:c      			Ja = [Ja;JR(ci,1:learnsizes(li))'];       		end;										Jt = setdiff(1:m,Ja);    				    			% Train classifier CLASSF{WI} on this training set and   				% calculate error on tuning set.					if (learncurve)	        	e1(i,ii) = testc(a, ...														 a(Ja,1:featsizes(fi))*classf{wi});					else	        	e1(i,ii) = testc(t(Jt,1:featsizes(fi)), ...														 a(Ja,1:featsizes(fi))*classf{wi});					end;     			fprintf(fid,'.');    		end  		end      fprintf(fid,'\n');  	end		% Calculate average error and standard deviation for this classifier		% (or set the latter to zero if there's been just 1 repetition).  	e.error(wi,:) = mean(e1,1);  	if (nreps == 1)  		e.std(wi,:) = zeros(1,size(e.std,2));  	else  		e.std(wi,:) = std(e1)/sqrt(nreps);  	end  end  % The first element is the empty string [], remove it.  e.names(1,:) = [];return