/*	Copyright (C) 2006, Mike Gashler	This library is free software; you can redistribute it and/or	modify it under the terms of the GNU Lesser General Public	License as published by the Free Software Foundation; either	version 2.1 of the License, or (at your option) any later version.	see http://www.gnu.org/copyleft/lesser.html*/#include "GNeuralNet.h"#include "GArff.h"#include "GMath.h"#include "GMacros.h"#include "GArray.h"#include "GBits.h"class GNeuron;#define INIT_THRESH .15#define OUTPUT_MIN .1#define OUTPUT_MIDDLE .5#define OUTPUT_RANGE .8#define INPUT_MIN -.7#define INPUT_RANGE 1.4struct GSynapse{	double m_dWeight; // Synapse	double m_dWeightDelta;	GNeuron* m_pInput;	GNeuron* m_pOutput;	GSynapse()	{		m_dWeight = (GBits::GetRandomDouble() * INIT_THRESH) - (INIT_THRESH / 2);		m_dWeightDelta = 0;		m_pInput = NULL;		m_pOutput = NULL;	}};class GNeuron{public:	GNeuron() {}	virtual ~GNeuron() {}	virtual double PullEvalDownStream() = 0;	virtual double PullErrorBackUpStream() = 0;	virtual double GetOutput() = 0;	virtual void SetOutput(double d) = 0;	virtual void SetError(double d) = 0;	virtual void AjustWeights(double dLearningRate, double dMomentum) = 0;	virtual int SerializeWeights(double* pBuffer) = 0;	virtual int DeserializeWeights(double* pBuffer) = 0;	virtual void AddInput(GNeuron* pNeuron) = 0;	virtual void Print() = 0;	virtual void BatchUpdateDeltas(double dLearningRate) = 0;	virtual void BatchUpdateWeights() = 0;	// For internal use only	virtual void ConnectOutputToSynapse(GSynapse* pSynapse) = 0;	virtual void RemapSynapse(GSynapse* pOld, GSynapse* pNew) = 0;};class GStandardNeuron : public GNeuron{protected:	double m_dOutput; // Axon	double m_dError;	int m_nInputs;	int m_nInputSpace;	struct GSynapse* m_pInputs;	int m_nOutputs;	int m_nOutputSpace;	struct GSynapse** m_pOutputs;public:	GStandardNeuron() : GNeuron()	{		m_dOutput = 1e50;		m_dError = 1e50;		m_nInputs = 0;		m_nInputSpace = 0;		m_pInputs = NULL;		AddInput(NULL); // Add the constant 1 (for bias) input		m_nOutputs = 0;		m_nOutputSpace = 0;		m_pOutputs = NULL;	}	virtual ~GStandardNeuron()	{		delete [] m_pInputs;		delete(m_pOutputs);	}	virtual void Print()	{		int n;		for(n = 0; n < m_nInputs; n++)			printf("\t%lf\n", m_pInputs[n].m_dWeight);	}	virtual double PullEvalDownStream()	{		if(m_dOutput == 1e50)		{			// Sum up the weighted inputs			double dSum = m_pInputs[0].m_dWeight;			int n;			for(n = 1; n < m_nInputs; n++)				dSum += (m_pInputs[n].m_dWeight * m_pInputs[n].m_pInput->PullEvalDownStream());			// Squash the sum			m_dOutput = GMath::sigmoid(dSum, 1);		}		return m_dOutput;	}	virtual double PullErrorBackUpStream()	{		GAssert(m_dOutput != 1e50, "output was not calculated");		if(m_dError == 1e50)		{			// Sum up the errors from each output			double dSum = 0;			int n;			for(n = 0; n < m_nOutputs; n++)				dSum += (m_pOutputs[n]->m_dWeight * m_pOutputs[n]->m_pOutput->PullErrorBackUpStream());			// Multiply by derivative of squashing function			m_dError = dSum * m_dOutput * ((double)1 - m_dOutput);		}		return m_dError;	}	virtual double GetOutput()	{		GAssert(m_dOutput != 1e50, "output was not calculated");		return m_dOutput;	}	virtual void SetOutput(double d)	{		m_dOutput = d;	}	virtual void SetError(double d)	{		m_dError = d;	}	virtual void AjustWeights(double dLearningRate, double dMomentum)	{		GAssert(m_dError != 1e50, "output was not calculated");		GSynapse* pNeuronRef;		pNeuronRef = &m_pInputs[0];		pNeuronRef->m_dWeightDelta *= dMomentum;		pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError);		pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;		int n;		for(n = 1; n < m_nInputs; n++)		{			pNeuronRef = &m_pInputs[n];			pNeuronRef->m_dWeightDelta *= dMomentum;			pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError * pNeuronRef->m_pInput->GetOutput());			pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;		}	}	virtual void BatchUpdateDeltas(double dLearningRate)	{		GAssert(m_dError != 1e50, "output was not calculated");		GSynapse* pNeuronRef;		pNeuronRef = &m_pInputs[0];		pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError);		int n;		for(n = 1; n < m_nInputs; n++)		{			pNeuronRef = &m_pInputs[n];			pNeuronRef->m_dWeightDelta += (dLearningRate * m_dError * pNeuronRef->m_pInput->GetOutput());		}	}	virtual void BatchUpdateWeights()	{		GSynapse* pNeuronRef;		pNeuronRef = &m_pInputs[0];		pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;		pNeuronRef->m_dWeightDelta = 0;		int n;		for(n = 1; n < m_nInputs; n++)		{			pNeuronRef = &m_pInputs[n];			pNeuronRef->m_dWeight += pNeuronRef->m_dWeightDelta;			pNeuronRef->m_dWeightDelta = 0;		}	}	virtual int SerializeWeights(double* pBuffer)	{		if(pBuffer)		{			int n;			for(n = 0; n < m_nInputs; n++)				pBuffer[n] = m_pInputs[n].m_dWeight;		}		return m_nInputs;	}	virtual int DeserializeWeights(double* pBuffer)	{		int n;		for(n = 0; n < m_nInputs; n++)			m_pInputs[n].m_dWeight = pBuffer[n];		return m_nInputs;	}	virtual void AddInput(GNeuron* pNeuron)	{		if(m_nInputs >= m_nInputSpace)		{			// Reallocate input space			int nInputSpace = MAX(4, m_nInputSpace * 2);			GSynapse* pInputs = new GSynapse[nInputSpace];			memcpy(pInputs, m_pInputs, sizeof(GSynapse) * m_nInputSpace);			// Remap everything			int n;			for(n = 0; n < m_nInputs; n++)			{				if(m_pInputs[n].m_pInput)					m_pInputs[n].m_pInput->RemapSynapse(&m_pInputs[n], &pInputs[n]);			}			delete(m_pInputs);			m_pInputs = pInputs;			m_nInputSpace = nInputSpace;		}		m_pInputs[m_nInputs].m_pInput = pNeuron;		m_pInputs[m_nInputs].m_pOutput = this;		if(pNeuron)			pNeuron->ConnectOutputToSynapse(&m_pInputs[m_nInputs]);		else		{			GAssert(m_nInputs == 0, "only the first input should be NULL");		}		m_nInputs++;	}	virtual void RemapSynapse(GSynapse* pOld, GSynapse* pNew)	{		int n;		for(n = 0; n < m_nOutputs; n++)		{			if(m_pOutputs[n] == pOld)			{				m_pOutputs[n] = pNew;				break;			}		}	}protected:	virtual void ConnectOutputToSynapse(GSynapse* pSynapse)	{		if(m_nOutputs >= m_nOutputSpace)		{			// Reallocate output space			int nOutputSpace = MAX(4, m_nOutputSpace * 2);			GSynapse** pOutputs = new GSynapse*[nOutputSpace];			memcpy(pOutputs, m_pOutputs, sizeof(GSynapse*) * m_nOutputSpace);			delete(m_pOutputs);			m_pOutputs = pOutputs;			m_nOutputSpace = nOutputSpace;		}		m_pOutputs[m_nOutputs++] = pSynapse;	}};// ----------------------------------------------------------------------GNeuralNet::GNeuralNet(GArffRelation* pRelation): GSupervisedLearner(pRelation){	m_pInternalRelation = NULL;	m_pNeurons = new GPointerArray(64);	m_pBestSet = NULL;	m_nWeightCount = 0;	m_nInputStart = 0;	m_nLayerStart = 0;	m_nLayerSize = 0;	m_pMinAndRanges = NULL;	MakeInternalRelationAndOutputLayer();	// Default settings	m_dLearningRate = .215;	m_dLearningDecay = 1;	m_dMomentum = .9;	m_nRunEpochs = 4000;	m_nMaximumEpochs = 50000;	m_nEpochsPerValidationCheck = 5;	m_dAcceptableMeanSquareError = 0.000001;	m_dTrainingPortion = .65;	// Step training	m_pTrainingDataInternal = NULL;	m_pValidationDataInternal = NULL;}GNeuralNet::~GNeuralNet(){	int nCount = m_pNeurons->GetSize();	int n;	for(n = 0; n < nCount; n++)		delete((GNeuron*)m_pNeurons->GetPointer(n));	delete(m_pNeurons);	delete(m_pBestSet);	delete(m_pMinAndRanges);	delete(m_pInternalRelation);	ReleaseInternalData();}void GNeuralNet::ReleaseInternalData(){	if(m_pValidationDataInternal != m_pTrainingDataInternal)		delete(m_pValidationDataInternal);	delete(m_pTrainingDataInternal);	m_pTrainingDataInternal = NULL;	m_pValidationDataInternal = NULL;}void GNeuralNet::MakeInternalRelationAndOutputLayer(){	// Make the internal relation	GAssert(m_pInternalRelation == NULL, "already created the internal relation");	m_pInternalRelation = new GArffRelation();	// Add the internal input nodes	GArffAttribute* pAttr;	int nValueCount;	int nInputCount = m_pRelation->GetInputCount();	int n, i;	for(n = 0; n < nInputCount; n++)	{		pAttr = m_pRelation->GetAttribute(m_pRelation->GetInputIndex(n));		if(pAttr->IsContinuous())			m_pInternalRelation->AddAttribute(new GArffAttribute(true, 0, NULL));		else		{			nValueCount = pAttr->GetValueCount();			if(nValueCount <= 2)				m_pInternalRelation->AddAttribute(new GArffAttribute(true, 0, NULL));			else			{				for(i = 0; i < nValueCount; i++)					m_pInternalRelation->AddAttribute(new GArffAttribute(true, 0, NULL));			}		}	}	// Add the internal output nodes	int nOutputCount = m_pRelation->GetOutputCount();	for(n = 0; n < nOutputCount; n++)	{		pAttr = m_pRelation->GetAttribute(m_pRelation->GetOutputIndex(n));		if(pAttr->IsContinuous())			m_pInternalRelation->AddAttribute(new GArffAttribute(false, 0, NULL));		else		{			nValueCount = pAttr->GetValueCount();			if(nValueCount <= 2)				m_pInternalRelation->AddAttribute(new GArffAttribute(false, 0, NULL));			else			{				for(i = 0; i < nValueCount; i++)					m_pInternalRelation->AddAttribute(new GArffAttribute(false, 0, NULL));			}		}	}	// Make the output layer	AddLayer(m_pInternalRelation->GetOutputCount());}void GNeuralNet::MakeInputLayer(){	GAssert(m_nInputStart == 0, "already made the input layer");	m_nInputStart = m_pNeurons->GetSize();	AddLayer(m_pInternalRelation->GetInputCount());}void GNeuralNet::InputsToInternal(double* pExternal, double* pInternal){	GAssert(m_pMinAndRanges, "min and ranges not calculated yet");	GArffAttribute* pAttr;	int nValueCount;	int nInputCount = m_pRelation->GetInputCount();	int nInternalIndex = 0;	int n, i, nExternalIndex;	for(n = 0; n < nInputCount; n++)	{		nExternalIndex = m_pRelation->GetInputIndex(n);		pAttr = m_pRelation->GetAttribute(nExternalIndex);		if(pAttr->IsContinuous())			pInternal[nInternalIndex++] = GArffData::Normalize(pExternal[nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex + 1], INPUT_MIN, INPUT_RANGE);		else		{			nValueCount = pAttr->GetValueCount();			if(nValueCount <= 2)				pInternal[nInternalIndex++] = (pExternal[nExternalIndex] < .5 ? INPUT_MIN : INPUT_MIN + INPUT_RANGE);			else			{				for(i = 0; i < nValueCount; i++)					pInternal[nInternalIndex + i] = INPUT_MIN;				GAssert((int)pExternal[nExternalIndex] >= 0 && (int)pExternal[nExternalIndex] < nValueCount, "out of range");				pInternal[nInternalIndex + (int)pExternal[nExternalIndex]] = INPUT_MIN + INPUT_RANGE;				nInternalIndex += nValueCount;			}		}	}	GAssert(nInternalIndex == m_pInternalRelation->GetInputCount(), "error");}void GNeuralNet::OutputsToInternal(double* pExternal, double* pInternal){	GAssert(m_pMinAndRanges, "min and ranges not calculated yet");	GArffAttribute* pAttr;	int nValueCount;	int nOutputCount = m_pRelation->GetOutputCount();	int nInternalIndex = m_pInternalRelation->GetOutputIndex(0);	int n, i, nExternalIndex;	for(n = 0; n < nOutputCount; n++)	{		nExternalIndex = m_pRelation->GetOutputIndex(n);		pAttr = m_pRelation->GetAttribute(nExternalIndex);		if(pAttr->IsContinuous())			pInternal[nInternalIndex++] = GArffData::Normalize(pExternal[nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex], m_pMinAndRanges[nExternalIndex + nExternalIndex + 1], OUTPUT_MIN, OUTPUT_RANGE);		else		{			nValueCount = pAttr->GetValueCount();			if(nValueCount <= 2)				pInternal[nInternalIndex++] = (pExternal[nExternalIndex] < .5 ? OUTPUT_MIN : OUTPUT_MIN + OUTPUT_RANGE);			else			{				for(i = 0; i < nValueCount; i++)					pInternal[nInternalIndex + i] = OUTPUT_MIN;				GAssert((int)pExternal[nExternalIndex] >= 0 && (int)pExternal[nExternalIndex] < nValueCount, "out of range");				pInternal[nInternalIndex + (int)pExternal[nExternalIndex]] = OUTPUT_MIN + OUTPUT_RANGE;				nInternalIndex += nValueCount;			}		}	}