#include "Stdafx.h"
#include "Backpropagation Neural Network.h"
#include <string>
#include "math.h"

using namespace std;
Neuron::Neuron()
{
	Set_Default_Values();
}
Neuron::~Neuron()
{
	delete [] pPosterior_Neuron;
	delete [] pAnterior_Neuron;
	delete [] pWeight;
	delete [] pWeight_Adjustment;
	this->Set_Default_Values();
}
void Neuron::Set_Default_Values()
{
	pPosterior_Neuron = NULL;
	pAnterior_Neuron = NULL;
	Num_Of_Posterior_Neurons = 0;
	Num_Of_Anterior_Neurons = 0;
	pWeight = NULL;
	pWeight_Adjustment = NULL;
	Bias = 0;
	Bias_Adjustment = 0;
	Total_Input = 0;
	Output = 0;
	Error_Signal = 0;
}
Layer::Layer()
{
	this->Set_Default_Values();
}
Layer::~Layer()
{
	delete [] pNeuron;
	this->Set_Default_Values();
}
void Layer::Set_Default_Values()
{
	pNeuron = NULL;
	Num_Of_Neurons = 0;
}
Backpropagation_Neural_Network::Backpropagation_Neural_Network()
{
	this->Set_Default_Values();
}
Backpropagation_Neural_Network::~Backpropagation_Neural_Network()
{
	delete [] pLayer;
	this->Set_Default_Values();
}
void Backpropagation_Neural_Network::Set_Default_Values()
{
	Learning_Rate = 1;
	Mean_Squared_Error = 0;
	Num_Of_Layers = 0;
	pLayer = NULL;
}

void Backpropagation_Neural_Network::Init(int * p,double learnrate)
{
	/*
	p is a pointer to an array or ints the first of which tells the number of layers
	and the rest define the neurons in those layers p[1]is the input layer
	*/
	Learning_Rate = learnrate;
	//allocate all the neurons in the layers
	Num_Of_Layers = p[0];
	if (Num_Of_Layers < 0)
	{
		MessageBox(NULL,"Negative number of layers?!??? WTF r u smokin?","",MB_OK);
		return;
	}
	pLayer = new Layer[Num_Of_Layers];
	for (int i = 1;i <= Num_Of_Layers; i++)
	{
		pLayer[i-1].Num_Of_Neurons = p[i];
		pLayer[i-1].pNeuron = new Neuron[p[i]];
	}
	Assign_Posterior_Neurons();
	Assign_Anterior_Neurons();
	Assign_Weights();
}
void Backpropagation_Neural_Network::Train_Once(double * in,double * out)
{
	Forward_Propagate(in);
	Back_Propagate(out);
	Apply_Weight_Adjustments();
}
void Backpropagation_Neural_Network::Forward_Propagate(double * d)
{
	/*
	This function propagates the values forward in the network starting at the input layer.  Each
	neuron outputs a value to every neuron in the next layer where it is weighted and added to the
	weighted values from other neurons.  It then goes through an activation function which is an infinitly
	derivable function and is outputted to the next layer.
	d is a pointer to an array of doubles that are the inputs for the first layer of the network and 
	should be normalized values in the interval -1,1 
	*/
	//load the input vector into the nodes in the input layer
	for (int f = 0; f <= this->pLayer[0].Num_Of_Neurons-1; f++)
	{
		this->pLayer[0].pNeuron[f].Total_Input = d[f];
	}
	//for every layer
	for (int i = 0; i <= this->Num_Of_Layers-1;i++)
	{
		//for every neuron
		for (int j = 0; j <= this->pLayer[i].Num_Of_Neurons-1;j++)
		{
			//clear input to zero first except if its the first layer
			if (!(i == 0))
			{
				this->pLayer[i].pNeuron[j].Total_Input = 0;
			}
			//add the outputs of all anterior neurons times their weightings
			for (int k = 0; k <=  this->pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons-1;k++)
			{
				this->pLayer[i].pNeuron[j].Total_Input = this->pLayer[i].pNeuron[j].Total_Input + (pLayer[i].pNeuron[j].pAnterior_Neuron[k]->Output * pLayer[i].pNeuron[j].pWeight[k]);
			}
			//add the bias (its just the bias weight multiplied by one so its the same thing as adding)
			this->pLayer[i].pNeuron[j].Total_Input = this->pLayer[i].pNeuron[j].Total_Input + this->pLayer[i].pNeuron[j].Bias;
			//figure out what the output is
			pLayer[i].pNeuron[j].Output = Activation_Function(pLayer[i].pNeuron[j].Total_Input);		
		}

	}
}
void Backpropagation_Neural_Network::Back_Propagate(double * d)
{
	/*
	This is the part of the network that does the learning.  After the network has been forward propagated
	we check what we got for output values based on the input we gave the network.  We then look at what
	we were suppoused to get (the desired output) and then compute the difference.  We can use the function
	that tells us this difference, called the error function to compute how we should change the network weights.
	the error is the desired output minus the observed output but the observed output can be expressed as all
	the math that went on inside the network which also inclues the weights for the neurons.  We determine how
	to adjust the weights by taking the partial derivative of the error function with respect to each individual 
	weight.  This tells us the instantaneous slope of the error function for that exact weight value.  If the
	derivative (slope) is positive (error increasing) we decrease the weight if it is negative (error decreasing)
	we increase the weight.

	d is a double pointer to an array of doubles that are the expected outputs
	the array should be the length of the number of output neurons and the values should be normalized
	in the range -1,1
	*/
	//first calculate the output error
	//find the mse
	Mean_Squared_Error = 0;
	//mse = sum k=0 to (num of output neurons) of (1/2*(desired_output_k - observed_output_k)^2)
	for (int i = 0; i <= this->pLayer[Num_Of_Layers-1].Num_Of_Neurons-1;i++)
	{
		Mean_Squared_Error = Mean_Squared_Error + ((double)1 / (double)2 * pow((d[i]-pLayer[Num_Of_Layers-1].pNeuron[i].Output),2));
	}
	//find the error signal for every neuron in the output layer
	for (int i = 0; i <= this->pLayer[Num_Of_Layers-1].Num_Of_Neurons-1;i++)
	{
		//(desired output - observed output)
		pLayer[Num_Of_Layers-1].pNeuron[i].Error_Signal = (d[i]-pLayer[Num_Of_Layers-1].pNeuron[i].Output);
	}
	//find the error signal for hidden layer neurons (we dont need to know the error for layer 1 because it does not weight anything)
	for (int i = this->Num_Of_Layers-2; i >= 1 ;i--)
	{
		for (int j = 0; j <= this->pLayer[i].Num_Of_Neurons-1;j++)
		{
			double Propagated_Error = 0;
			for (int k = 0; k <= this->pLayer[i].pNeuron[j].Num_Of_Posterior_Neurons-1;k++)
			{
				//for every posterior neuron multiply 				
				//the weighting of the current neuron output from posterior neuron k times the error of posterior neuron k
				Propagated_Error = Propagated_Error + 
				(pLayer[i].pNeuron[j].pPosterior_Neuron[k]->pWeight[j] *
				pLayer[i].pNeuron[j].pPosterior_Neuron[k]->Error_Signal);
			}
			//derivative of activation function of input for current neuron times propagated error
			pLayer[i].pNeuron[j].Error_Signal = Derivative_Activation_Function(pLayer[i].pNeuron[j].Total_Input) * Propagated_Error;
		}
	}
	//find weight adjustments
	for (int i = 1; i <= this->Num_Of_Layers-1;i++)
	{
		for (int j = 0; j <= this->pLayer[i].Num_Of_Neurons-1;j++)
		{
			//the weight adjustment for weight(destination,source) is the destination node error signal times the source node output
 			for (int k = 0; k <= this->pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons-1;k++)
			{
				pLayer[i].pNeuron[j].pWeight_Adjustment[k] =
				Learning_Rate *
				(pLayer[i].pNeuron[j].Error_Signal) *
				(pLayer[i].pNeuron[j].pAnterior_Neuron[k]->Output);
			}
			//the bias unit has an input of one all the time so its just error signal times 1
			pLayer[i].pNeuron[j].Bias_Adjustment = Learning_Rate * pLayer[i].pNeuron[j].Error_Signal;
		}
	}
}
void Backpropagation_Neural_Network::Apply_Weight_Adjustments()
{
	/*
	Apply the adjustments that are due to be made to the weights and biases
	*/
	for (int i = 1; i <= this->Num_Of_Layers-1;i++)
	{
		for (int j = 0; j <= this->pLayer[i].Num_Of_Neurons-1;j++)
		{
			for (int k = 0; k <= this->pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons-1;k++)
			{
				pLayer[i].pNeuron[j].pWeight[k] = pLayer[i].pNeuron[j].pWeight[k] + pLayer[i].pNeuron[j].pWeight_Adjustment[k];
				pLayer[i].pNeuron[j].pWeight_Adjustment[k] = 0;
			}
			pLayer[i].pNeuron[j].Bias = pLayer[i].pNeuron[j].Bias + pLayer[i].pNeuron[j].Bias_Adjustment;
			pLayer[i].pNeuron[j].Bias_Adjustment = 0;
		}
	}
}
void Backpropagation_Neural_Network::Assign_Posterior_Neurons()
{
	/*
	Assign pointers to all the posterior neurons so the neurons know their neighbors
	*/
	//the last layer of neurons does not have any posterior nodes
	for (int i = 0; i <= Num_Of_Layers-2;i++)
	{
		for (int j = 0; j <= pLayer[i].Num_Of_Neurons-1;j++)
		{
			pLayer[i].pNeuron[j].Num_Of_Posterior_Neurons = pLayer[i+1].Num_Of_Neurons;
			pLayer[i].pNeuron[j].pPosterior_Neuron = new Neuron*[pLayer[i+1].Num_Of_Neurons];
			//take every neuron and give the address of every neuron in the next layer
			for (int k = 0; k <= pLayer[i+1].Num_Of_Neurons-1;k++)
			{
				pLayer[i].pNeuron[j].pPosterior_Neuron[k] = &(pLayer[i+1].pNeuron[k]);
			}
		}
	}
}
void Backpropagation_Neural_Network::Assign_Anterior_Neurons()
{
	/*
	Assign pointers to all the anterior neurons so the neurons know their neighbors
	*/
	//the first layer of neurons does not have any anterior nodes
	for (int i = 1; i <= Num_Of_Layers-1;i++)
	{
		for (int j = 0; j <= pLayer[i].Num_Of_Neurons-1;j++)
		{
			pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons = pLayer[i-1].Num_Of_Neurons;
			pLayer[i].pNeuron[j].pAnterior_Neuron = new Neuron*[pLayer[i-1].Num_Of_Neurons];
			//take every neuron and give the address of every neuron in the layer before
			for (int k = 0; k <= pLayer[i-1].Num_Of_Neurons-1;k++)
			{
				pLayer[i].pNeuron[j].pAnterior_Neuron[k] = &(pLayer[i-1].pNeuron[k]);
			}
		}
	}
}
void Backpropagation_Neural_Network::Assign_Weights()
{
	/*
	Assign weights and initial values for the weights
	the number of weights is the number of anterior neurons for that neuron
	a weight for a neuron is the weight of output from the anterior neuron to the current neuron
	*/
	//the first layer of neurons does not have any anterior nodes and therefore no weights
	for (int i = 1; i <= Num_Of_Layers-1;i++)
	{
		//go throught every neuron
		for (int j = 0; j <= pLayer[i].Num_Of_Neurons-1;j++)
		{
			pLayer[i].pNeuron[j].pWeight = new double[pLayer[i-1].Num_Of_Neurons];
			pLayer[i].pNeuron[j].pWeight_Adjustment = new double[pLayer[i-1].Num_Of_Neurons];
			//assign the weights random values in the interval -r,r where r is the 1/sqrt(number of anterior neurons)
			//this normalizes the input to the neuron so that it doesent get saturated which will leave it with error 
			//but no gradient to correct the error.
			for (int k = 0; k <= pLayer[i-1].Num_Of_Neurons-1;k++)
			{
				pLayer[i].pNeuron[j].pWeight[k] = Get_Rand_In_Range((double)1/pow(pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons,(double)0.5));
				pLayer[i].pNeuron[j].pWeight_Adjustment[k] = 0;
			}
			pLayer[i].pNeuron[j].Bias = Get_Rand_In_Range((double)1/pow(pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons,(double)0.5));
			pLayer[i].pNeuron[j].Bias_Adjustment = 0;
		}
	}
}
string Backpropagation_Neural_Network::Give_Network_Info_String()
{
	char * Buffer = new char[1000];
	string strTemp = "";
	sprintf(Buffer,"Network has %i layers \n",this->Num_Of_Layers);
	strTemp = strTemp + Buffer;
	sprintf(Buffer,"Mean squared error is currently %f  \n",this->Mean_Squared_Error);
	strTemp = strTemp + Buffer;
	sprintf(Buffer,"Learning rate %f  \n",this->Learning_Rate);
	strTemp = strTemp + Buffer;
	/*
	for (int i = 0; i <= Num_Of_Layers-1;i++)
	{
		sprintf(Buffer,">>>>>>>>>>>> start of layer number %i which has %i Neurons  <<<<<<<<<<<< \n",i,this->pLayer[i].Num_Of_Neurons);
		strTemp = strTemp + Buffer;		
		for (int j = 0; j <= pLayer[i].Num_Of_Neurons-1;j++)
		{
			sprintf(Buffer,"____Info for neuron number: %i  \n",j);
			strTemp = strTemp + Buffer;	
			sprintf(Buffer,"________Total_input: %f  \n",this->pLayer[i].pNeuron[j].Total_Input);
			strTemp = strTemp + Buffer;	
			sprintf(Buffer,"________Bias: %f  \n",this->pLayer[i].pNeuron[j].Bias);
			strTemp = strTemp + Buffer;	
			sprintf(Buffer,"________Bias yet to be applied: %f  \n",this->pLayer[i].pNeuron[j].Bias_Adjustment);
			strTemp = strTemp + Buffer;	
			sprintf(Buffer,"________Output: %f  \n",this->pLayer[i].pNeuron[j].Output);
			strTemp = strTemp + Buffer;
			sprintf(Buffer,"________Num_Of_Anterior_Neurons: %i  \n",this->pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons);
			strTemp = strTemp + Buffer;
			sprintf(Buffer,"________Num_Of_Posterior_Neurons: %i  \n",this->pLayer[i].pNeuron[j].Num_Of_Posterior_Neurons);
			strTemp = strTemp + Buffer;
			sprintf(Buffer,"________Error signal: %f  \n",this->pLayer[i].pNeuron[j].Error_Signal);
			strTemp = strTemp + Buffer;
			for (int k = 0; k  <= pLayer[i].pNeuron[j].Num_Of_Anterior_Neurons-1; k++)
			{
				sprintf(Buffer,"____________Weighting of anterior neuron %i is: %f \n",k,this->pLayer[i].pNeuron[j].pWeight[k]);
				strTemp = strTemp + Buffer;	
				sprintf(Buffer,"____________Adjustment of neuron %i that is yet to be applied: %f \n",k,this->pLayer[i].pNeuron[j].pWeight_Adjustment[k]);
				strTemp = strTemp + Buffer;	
			}
		}
		sprintf(Buffer,">>>>>>>>>>>> end of layer number %i which has %i Neurons  <<<<<<<<<<<< \n",i,this->pLayer[i].Num_Of_Neurons);
		strTemp = strTemp + Buffer;	
	}
	*/

	for (int i = Num_Of_Layers-1; i <= Num_Of_Layers-1;i++)
	{
		for (int j = 0; j <= pLayer[i].Num_Of_Neurons-1;j++)
		{
			sprintf(Buffer,"____Info for neuron number: %i  \n",j);
			strTemp = strTemp + Buffer;	
			sprintf(Buffer,"________Total_output: %f  \n",this->pLayer[i].pNeuron[j].Output);
			strTemp = strTemp + Buffer;	
			sprintf(Buffer,"________Error signal: %f  \n",this->pLayer[i].pNeuron[j].Error_Signal);
			strTemp = strTemp + Buffer;
		}
	}

	delete [] Buffer;
	return strTemp;
}
double Activation_Function(double d)
{
	//hyperbolic tangent
	return (((pow((double)NUMBER_E,(double)2*d)-(double)1)  /  (pow((double)NUMBER_E,(double)2*d)+(double)1)));
}
double Derivative_Activation_Function(double d)
{
	//derivative hyperbolic tangent
	return (1 - pow(Activation_Function(d),(double)2));
}
double Get_Rand_In_Range(double d)
{
	//returns a random number in the interval -d,d
	return ((((double)rand() / (double)RAND_MAX * ((double)2*d))) -d);
}