ANN.cpp

Go to the documentation of this file.

/* -*- Mode: C++; -*- */
/* VER: $Id$ */
// copyright (c) 2004 by Christos Dimitrakakis <dimitrak@idiap.ch>
/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#include <cstring>
#include <learning/ANN.h>
#include <learning/string_utils.h>
#include <learning/Distribution.h>


#undef ANN_DBUG

//==========================================================
// NewANN
//----------------------------------------------------------
ANN *NewANN(int n_inputs, int n_outputs)
{
    ANN *ann = NULL;

    if (!(ann = AllocM(ANN, 1))) {
        Serror("Could not allocate ANN\n");
        return NULL;
    }
    ann->x = NULL;
    ann->y = NULL;
    ann->t = NULL;
    ann->d = NULL;
    ann->error = NULL;
    ann->c = NULL;
    ann->a = 0.1f;
    ann->lambda = 0.9f;
    ann->zeta = 0.9f;
    ann->n_inputs = n_inputs;
    ann->n_outputs = n_outputs;
    ann->batch_mode = false;


    /* outputs are not allocated */
    //logmsg ("Creating ANN with %d inputs and %d outputs\n", n_inputs, n_outputs);
    if (!(ann->error = AllocM(real, n_outputs))) {
        Serror("Could not allocate errors\n");
        DeleteANN(ann);
        return NULL;
    }

    if (!(ann->d = AllocM(real, n_outputs))) {
        Serror("Could not allocate derivatives\n");
        DeleteANN(ann);
        return NULL;
    }

    if (!(ann->c = List())) {
        Serror("Could not allocate list\n");
        DeleteANN(ann);
        return NULL;
    }
#ifdef ANN_DBUG
    message("Creating ANN with %d inputs and %d outputs", n_inputs,
        n_outputs);
#endif
    return ann;
}

//==========================================================
// DeleteANN
//----------------------------------------------------------
int DeleteANN(ANN * ann)
{
    if (!ann) {
        Swarning("Attempting to delete NULL ANN\n");
        return DEC_ARG_INVALID;
    }

    if (ann->error) {
        FreeM(ann->error);
    }
    
    //if (ann->x) {
    //  FreeM (ann->x);
    //}

    if (ann->d) {
        FreeM(ann->d);
    }

    /* We must clear all allocations in the list */
    if (ann->c) {
        ClearList(ann->c);
        ann->c = NULL;
    }

    FreeM(ann);
    return 0;
}


//==========================================================
// ANN_AddHiddenLayer()
//----------------------------------------------------------
int ANN_AddHiddenLayer(ANN * ann, int n_nodes)
{
#ifdef ANN_DBUG
    message("Adding Hidden layer with %d nodes", n_nodes);
#endif

    LISTITEM *item = LastListItem(ann->c);
    if (item) {
        Layer *p = (Layer *) item->obj;
        ANN_AddLayer(ann, p->n_outputs, n_nodes, p->y);
    } else {
        ANN_AddLayer(ann, ann->n_inputs, n_nodes, ann->x);
    }
    return 0;
}

//==========================================================
// ANN_AddRBFHiddenLayer()
//----------------------------------------------------------
int ANN_AddRBFHiddenLayer(ANN * ann, int n_nodes)
{
#ifdef ANN_DBUG
    message("Adding Hidden layer with %d nodes", n_nodes);
#endif
    LISTITEM *item = LastListItem(ann->c);
    if (item) {
        Layer *p = (Layer *) item->obj;
        ANN_AddRBFLayer(ann, p->n_outputs, n_nodes, p->y);
    } else {
        ANN_AddRBFLayer(ann, ann->n_inputs, n_nodes, ann->x);
    }
    return 0;
}



//==========================================================
// ANN_AddLayer()
//----------------------------------------------------------
Layer *ANN_AddLayer(ANN * ann, int n_inputs, int n_outputs, real * x)
{
    Layer *l = NULL;
    if ((x == NULL) && (ann->c->n)) {
        Swarning
            ("Layer connects to null but layer list is not empty\n");
    }


    if (!(l = AllocM(Layer, 1))) {
        Serror("Could not allocate layer structure\n");
        return NULL;
    }

    assert(n_inputs > 0);
    assert(n_outputs > 0);

    l->n_inputs = n_inputs;
    l->n_outputs = n_outputs;
    l->x = x;
    l->a = ann->a;
    l->zeta = ann->zeta;
    l->lambda = ann->lambda;
    l->forward = &ANN_CalculateLayerOutputs;
    l->backward = &ANN_Backpropagate;
    l->f = &htan;
    l->f_d = &htan_d;
    //l->f = &dtan;
    //  l->f_d = &dtan_d;
    l->batch_mode = false;
    if (!(l->y = AllocM(real, n_outputs))) {
        Serror("Could not allocate layer outputs\n");
        ANN_FreeLayer(l);
        return NULL;
    }
    int i;
    for (i=0; i<n_outputs; i++) {
        l->y[i] = 0.0;
    }

    if (!(l->z = AllocM(real, n_outputs))) {
        Serror("Could not allocate layer activations\n");
        ANN_FreeLayer(l);
        return NULL;
    }
    for (i=0; i<n_outputs; i++) {
        l->z[i] = 0.0;
    }
    if (!(l->d = AllocM(real, n_inputs + 1 /*bias */ ))) {
        Serror("Could not allocate layer outputs\n");
        ANN_FreeLayer(l);
        return NULL;
    }
    for (i=0; i<n_inputs+1; i++) {
        l->d[i] = 0.0;
    }

    if (!
        (l->c =
         AllocM(Connection, (n_inputs + 1 /*bias */ ) * n_outputs))) {
        Serror("Could not allocate connections\n");
        ANN_FreeLayer(l);
        return NULL;
    }

    l->rbf = NULL;

    real bound = 2.0f / sqrt((real) n_inputs);
    for (i = 0; i < n_inputs + 1 /*bias */ ; i++) {
        Connection *c = &l->c[i * n_outputs];
        for (int j = 0; j < n_outputs; j++) {
            c->w = (urandom() - 0.5f)* bound;;
            c->c = 1;
            c->e = 0.0f;
            c->dw = 0.0f;
            c->v = 1.0;
            c++;
        }
    }
    ListAppend(ann->c, (void *) l, &ANN_FreeLayer);
    return l;
}


//==========================================================
// ANN_AddRBFLayer()
//----------------------------------------------------------
Layer *ANN_AddRBFLayer(ANN * ann, int n_inputs, int n_outputs, real * x)
{
    Layer *l = NULL;
    if ((x == NULL) && (ann->c->n)) {
        Swarning
            ("Layer connects to null and layer list not empty\n");
    }

    if (!(l = AllocM(Layer, 1))) {
        Serror("Could not allocate layer structure\n");
        return NULL;
    }

    assert(n_inputs > 0);
    assert(n_outputs > 0);

    l->n_inputs = n_inputs;
    l->n_outputs = n_outputs;
    l->x = x;
    l->a = ann->a;
    l->forward = &ANN_RBFCalculateLayerOutputs;
    l->backward = &ANN_RBFBackpropagate;
    l->f = &Exp;
    l->f_d = &Exp_d;
    l->batch_mode = false;

    if (!(l->y = AllocM(real, n_outputs))) {
        Serror("Could not allocate layer outputs\n");
        ANN_FreeLayer(l);
        return NULL;
    }

    if (!(l->z = AllocM(real, n_outputs))) {
        Serror("Could not allocate layer activations\n");
        ANN_FreeLayer(l);
        return NULL;
    }

    if (!(l->d = AllocM(real, n_inputs + 1 /*bias */ ))) {
        Serror("Could not allocate layer outputs\n");
        ANN_FreeLayer(l);
        return NULL;
    }

    if (!
        (l->rbf =
         AllocM(RBFConnection,
            (n_inputs + 1 /*bias */ ) * n_outputs))) {
        Serror("Could not allocate connections\n");
        ANN_FreeLayer(l);
        return NULL;
    }

    l->c = NULL;

    real bound = 2.0f / sqrt((real) n_inputs);
    for (int i = 0; i < n_inputs + 1 /*bias */ ; i++) {
        RBFConnection *c = &l->rbf[i * n_outputs];
        for (int j = 0; j < n_outputs; j++) {
            c->w = (urandom() - 0.5f) * bound;;
            c->m = (urandom() - 0.5f) * 2.0f;
            c++;
        }
    }
    ListAppend(ann->c, (void *) l, &ANN_FreeLayer);
    return l;
}


//==========================================================
// ANN_FreeLayer()
//----------------------------------------------------------
void ANN_FreeLayer(void *l)
{
    ANN_FreeLayer((Layer *) l);
}

//==========================================================
// ANN_FreeLayer()
//----------------------------------------------------------
void ANN_FreeLayer(Layer * l)
{
    FreeM(l->y);
    if (l->z) {
        FreeM(l->z);
    }
    if (l->c) {
        FreeM(l->c);
    }
    if (l->rbf) {
        FreeM(l->rbf);
    }
    FreeM(l->d);
    FreeM(l);

}

//==========================================================
// ANN_Init()
//----------------------------------------------------------

int ANN_Init(ANN * ann)
{
    // Add output layer
    LISTITEM *item = LastListItem(ann->c);
    Layer *l = NULL;
#ifdef ANN_DBUG
    message("Initialising");
#endif
    if (item) {
        Layer *p = (Layer *) item->obj;
        l = ANN_AddLayer(ann, p->n_outputs, ann->n_outputs, p->y);
    } else {
        l = ANN_AddLayer(ann, ann->n_inputs, ann->n_outputs,
                 ann->x);
    }
    if (l == NULL) {
        Serror("Could not create final layer\n");
        DeleteANN(ann);
        return -1;
    }
    ann->y = l->y;
    l->f = &linear;
    l->f_d = &linear_d;
    //  ann->t = l->t;
    return 0;
}



//==========================================================
// ANN_Reset()
//----------------------------------------------------------
void ANN_Reset(ANN * ann)
{
    LISTITEM *p = FirstListItem(ann->c);

    while (p) {
        Layer *l = (Layer *) p->obj;
        for (int i = 0; i < l->n_inputs + 1 /* bias */; i++) {
            Connection *c = &l->c[i * l->n_outputs];
            for (int j = 0; j < l->n_outputs; j++) {
                c->e = 0.0;
                c->dw = 0.0;
                c++;
            }
        }
        p = NextListItem (ann->c);
    }
}


//==========================================================
// ANN_Input()
//----------------------------------------------------------

real ANN_Input(ANN * ann, real * x)
{
    LISTITEM *p = FirstListItem(ann->c);
    Layer *first_layer = (Layer *) p->obj;
    ann->x = x;
    first_layer->x = x; // Setup input of first layer
    //  printf ("II: %f\n", ann->x[0]);
    while (p) {
        Layer *current_layer = (Layer *) p->obj;
        //    printf ("\tIII: %f\n", current_layer->x[0]);
        current_layer->forward(current_layer, false);
        p = NextListItem(ann->c);
    }
    return 0.0f;
}

//==========================================================
// ANN_StochasticInput()
//----------------------------------------------------------
real ANN_StochasticInput(ANN * ann, real * x)
{
    LISTITEM *p = FirstListItem(ann->c);
    Layer *first_layer = (Layer *) p->obj;
    ann->x = x;
    first_layer->x = x; // Setup input of first layer
    //  printf ("II: %f\n", ann->x[0]);
    while (p) {
        Layer *current_layer = (Layer *) p->obj;
        //    printf ("\tIII: %f\n", current_layer->x[0]);
        current_layer->forward(current_layer, true);
        p = NextListItem(ann->c);
    }
    return 0.0f;
}

//==========================================================
// ANN_CalculateLayerOutputs()
//----------------------------------------------------------
void ANN_CalculateLayerOutputs(Layer * current_layer, bool stochastic)
{
    int i, j;
    int n_inputs = current_layer->n_inputs;
    int n_outputs = current_layer->n_outputs;
    real *x = current_layer->x;
    real *y = current_layer->y;
    real *z = current_layer->z;
    Connection *c;

    for (j = 0; j < n_outputs; j++) {
        z[j] = 0.0f;
    }
    c = current_layer->c;
    if (stochastic) {
        for (i = 0; i < n_inputs; i++) {
            for (j = 0; j < n_outputs; j++) {
                // using uniform bounded.. 
                real w = c->w + (urandom()-0.5f)*c->v ;
                z[j] += x[i] * w;
                c++;
            }
        }
        
        // bias
        for (j = 0; j < n_outputs; j++) {
            real w = c->w + (urandom()-0.5f)*c->v ;
            z[j] += w;
            c++;
        }
    } else {
        for (i = 0; i < n_inputs; i++) {
            for (j = 0; j < n_outputs; j++) {
                z[j] += x[i] * c->w;
                c++;
            }
        }
        
        // bias
        for (j = 0; j < n_outputs; j++) {
            z[j] += c->w;
            c++;
        }
    }
    
    for (j = 0; j < n_outputs; j++) {
        y[j] = current_layer->f(z[j]);
    }
}

//==========================================================
// ANN_RBFCalculateLayerOutputs()
//----------------------------------------------------------
void ANN_RBFCalculateLayerOutputs(Layer * current_layer, bool stochastic)
{
    int i, j;
    int n_inputs = current_layer->n_inputs;
    int n_outputs = current_layer->n_outputs;
    real *x = current_layer->x;
    real *y = current_layer->y;
    real *z = current_layer->z;
    RBFConnection *c;


    for (j = 0; j < n_outputs; j++) {
        z[j] = 0.0f;
    }

    c = current_layer->rbf;
    for (i = 0; i < n_inputs; i++) {
        real in = x[i];
        for (j = 0; j < n_outputs; j++, c++) {
            real o = (in - c->m) * c->w;
            z[j] += o * o;
        }
    }

    for (j = 0; j < n_outputs; j++) {
        z[j] = -0.5f * z[j];
        y[j] = current_layer->f(z[j]);
    }
}


//==========================================================
// ANN_Train()                           simple MSE training 
//----------------------------------------------------------

real ANN_Train(ANN * ann, real * x, real * t)
{
    LISTITEM *p = LastListItem(ann->c);
    Layer *l = (Layer *) p->obj;
    real sum = 0.0f;
    int j;

    ANN_Input(ann, x);

    for (j = 0; j < ann->n_outputs; j++) {
        real f = l->f_d(ann->y[j]);
        real e = t[j] - ann->y[j];
        ann->error[j] = e;
        ann->d[j] = e * f;
        sum += e * e;
    }

    l->backward(p, ann->d, ann->eligibility_traces, 0.0);

    return sum;
}


//==========================================================
// ANN_Delta_Train()                  Train with custom cost
//----------------------------------------------------------

real ANN_Delta_Train(ANN * ann, real* delta, real TD)
{
    LISTITEM *p = LastListItem(ann->c);
    Layer *l = (Layer *) p->obj;
    real sum = 0.0f;
    int j;
    //ANN_Input(ann, x);
    for (j = 0; j < ann->n_outputs; j++) {
        real f = l->f_d(ann->y[j]);
        real e = delta[j];
        ann->error[j] = e;
        ann->d[j] = e * f;
        sum += e * e;
    }

    l->backward(p, ann->d, ann->eligibility_traces, TD);

    return sum;
}



//==========================================================
// ANN_Backpropagate
//----------------------------------------------------------
real ANN_Backpropagate(LISTITEM * p, real * d, bool use_eligibility, real TD)
{
    int i, j;
    real f;
    real a;
    Layer *l = (Layer *) p->obj;
    LISTITEM *back = p->prev;
    Layer *back_layer = NULL;
    a = l->a;
    //  message ("backing with in: %d",l->x);
    if (back) {
        //message ("and to prev");
        back_layer = (Layer *) back->obj;
        for (i = 0; i < l->n_inputs; i++) {
            real der = 0.0f;

            Connection *c = &l->c[i * l->n_outputs];
            for (j = 0; j < l->n_outputs; j++) {
                der += c->w * d[j];
                c++;
            }
            f = back_layer->f_d(l->x[i]);
            der*=f;
            l->d[i] = der;
        }

        /* bias */
        i = l->n_inputs;
        l->d[i] = 0.0f;
        Connection *c = &l->c[i * l->n_outputs];
        for (j = 0; j < l->n_outputs; j++) {
            l->d[i] += c->w * d[j];
            c++;
        }
        f = back_layer->f_d(1.0f);
        l->d[i] = l->d[i] * f;

        back_layer->backward(back, l->d, use_eligibility, TD);
    }
    //update weights
    for (i = 0; i < l->n_inputs; i++) {
        Connection *c = &l->c[i * l->n_outputs];
        real dx = a * l->x[i];
        if (l->batch_mode) {
            for (j = 0; j < l->n_outputs; j++) {
                real delta;
                if (use_eligibility) {
                    c->e = c->e * l->lambda + l->x[i]* d[j];
                    delta = a * c->e * TD; //better?
                    c->v += (1.0f - l->zeta)*c->v+(l->zeta)*delta*delta;
                } else {
                    delta = dx * d[j];
                }
                c->dw += delta;
                c->v = (1.0f - l->zeta)*c->v + (l->zeta)*fabs(delta);
                if (c->v < 0.01f) c->v = 0.01f;
                c++;
            }
        } else {
            for (j = 0; j < l->n_outputs; j++) {
                real delta;
                if (use_eligibility) {
                    c->e = c->e * l->lambda + l->x[i] * d[j];
                    delta = a * c->e * TD;
                } else {
                    delta = dx * d[j];
                }
                c->w += delta;
                delta /= a;
                c->v = (1.0f - l->zeta)*c->v + (l->zeta)*fabs(delta);
                //printf("%f\n", c->v);
                if (c->v < 0.01f) c->v = 0.01f;
                c++;
            }
        }
    }
    // update bias weight
    {
        Connection *c = &l->c[l->n_inputs * l->n_outputs];
        if (l->batch_mode) {
            for (j = 0; j < l->n_outputs; j++) {
                real delta;
                if (use_eligibility) {
                    c->e = c->e * l->lambda + d[j];
                    delta = a * c->e * TD;
                } else {
                    delta = a * d[j];
                }
                c->dw += delta;
                c->v = (1.0 - l->zeta)*c->v + (l->zeta)*fabs(delta);
                if (c->v < 0.01f) c->v = 0.01f;
                c++;
            }
        } else {
            for (j = 0; j < l->n_outputs; j++) {
                real delta;
                if (use_eligibility) {
                    c->e = c->e * l->lambda + d[j];
                    delta = a * c->e * TD; //better?
                } else {
                    delta = a * d[j];
                }
                c->w += delta;
                c->v = (1.0f - l->zeta)*c->v + (l->zeta)*fabs(delta);
                if (c->v < 0.01f) c->v = 0.01f;
                c++;
            }
        }
    }
    return 0.0f;
}

//==========================================================
// ANN_RBFBackpropagate
//----------------------------------------------------------
real ANN_RBFBackpropagate(LISTITEM * p, real * d, bool use_eligibility, real TD)
{
    int i, j;
    real f;
    real a;
    Layer *l = (Layer *) p->obj;
    LISTITEM *back = p->prev;
    Layer *back_layer = NULL;
    a = l->a;

    if (back) {
        back_layer = (Layer *) back->obj;
        for (i = 0; i < l->n_inputs; i++) {
            l->d[i] = 0.0f;
            RBFConnection *c = &l->rbf[i * l->n_outputs];
            for (j = 0; j < l->n_outputs; j++) {
                real dx = l->x[i] - c->m;
                real dm = d[j] * dx * c->w * c->w;
                l->d[j] -= dm;
                c++;
            }
            f = back_layer->f_d(l->x[i]);
            l->d[i] = l->d[i] * f;
        }
        back_layer->backward(back, l->d, use_eligibility, TD);
    }

    return 0.0f;
    //update weights
    for (i = 0; i < l->n_inputs; i++) {
        RBFConnection *c = &l->rbf[i * l->n_outputs];
        real dx = l->x[i] - c->m;
        for (j = 0; j < l->n_outputs; j++) {
            real dy = d[j];
            real dx2 = a * dy * dx * c->w;
            real dm = dx2 * c->w;
            real dw = dx2 * dx;
            c->m += dm;
            c->w += dw;
            c++;
        }
    }
    return 0.0f;
}

void ANN_LayerBatchAdapt(Layer * l)
{
    int i, j;

    if (l->batch_mode == false) {
        Serror("Batch adapt yet not in batch mode!");
    }
    //update weights
    for (i = 0; i < l->n_inputs; i++) {
        Connection *c = &l->c[i * l->n_outputs];
        for (j = 0; j < l->n_outputs; j++) {
            c->w += c->dw;
            c++;
        }
    }
    // update bias weight
    {
        Connection *c = &l->c[l->n_inputs * l->n_outputs];
        for (j = 0; j < l->n_outputs; j++) {
            c->w += c->dw;
            c++;
        }
    }
}

//==========================================================
// ANN_Test()
//----------------------------------------------------------
real ANN_Test(ANN * ann, real * x, real * t)
{
    //LISTITEM *p = LastListItem(ann->c);
    //Layer *l = (Layer *) p->obj;
    real sum = 0.0f;
    int j;
    ANN_Input(ann, x);

    for (j = 0; j < ann->n_outputs; j++) {
        //real f = l->f_d(ann->y[j]);
        real e = t[j] - ann->y[j];
        ann->error[j] = e;
        ann->d[j] =0.0;// e * f;
        sum += e * e;
    }
    return sum;
}

//==========================================================
// ANN_GetOutput()
//----------------------------------------------------------
real *ANN_GetOutput(ANN * ann)
{
    return ann->y;
}

//==========================================================
// ANN_GetError()
//----------------------------------------------------------
real ANN_GetError(ANN * ann)
{
    real sum = 0.0;
    for (int i=0; i<ann->n_outputs; i++) {
    real e = ann->error[i];
    sum += e*e;
    }
    return (real) sqrt(sum);
}

//==========================================================
// ANN_GetErrorVector()
//----------------------------------------------------------
real* ANN_GetErrorVector(ANN * ann)
{
    return ann->error;
}

//==========================================================
// ANN_SetLearningRate()
//----------------------------------------------------------
void ANN_SetLearningRate(ANN * ann, real a)
{
    LISTITEM *c;

    ann->a = a;
    c = FirstListItem(ann->c);
    while (c) {
        Layer *l = (Layer *) c->obj;
        l->a = a;
        c = NextListItem(ann->c);
    }
}
//==========================================================
// ANN_SetLambda()
//----------------------------------------------------------
void ANN_SetLambda(ANN * ann, real lambda)
{
    LISTITEM *c;

    ann->lambda = lambda;
    c = FirstListItem(ann->c);
    while (c) {
        Layer *l = (Layer *) c->obj;
        l->lambda = lambda;
        c = NextListItem(ann->c);
    }
}

//==========================================================
// ANN_SetZeta()
//----------------------------------------------------------
void ANN_SetZeta(ANN * ann, real zeta)
{
    LISTITEM *c;

    ann->zeta = zeta;
    c = FirstListItem(ann->c);
    while (c) {
        Layer *l = (Layer *) c->obj;
        l->zeta = zeta;
        c = NextListItem(ann->c);
    }
}
//==========================================================
// ANN_SetBatchMode
//----------------------------------------------------------
void ANN_SetBatchMode(ANN * ann, bool batch)
{
    LISTITEM *c;

    ann->batch_mode = batch;
    c = FirstListItem(ann->c);
    while (c) {
        Layer *l = (Layer *) c->obj;
        l->batch_mode = batch;
        c = NextListItem(ann->c);
    }
}

//==========================================================
// ANN_BatchAdapt
//----------------------------------------------------------
void ANN_BatchAdapt(ANN * ann)
{
    LISTITEM *c;

    c = FirstListItem(ann->c);
    while (c) {
        Layer *l = (Layer *) c->obj;
        ANN_LayerBatchAdapt(l);
        c = NextListItem(ann->c);
    }
}



//==========================================================
// ANN_ShowWeights()
//----------------------------------------------------------
real ANN_ShowWeights(ANN * ann)
{
    LISTITEM *c;
    real sum = 0.0f;
    c = FirstListItem(ann->c);
    while (c) {
        Layer *l = (Layer *) c->obj;
        sum += ANN_LayerShowWeights(l);
        c = NextListItem(ann->c);
    }
    return sum;
}

//==========================================================
// ANN_LayerShowWeights()
//----------------------------------------------------------
real ANN_LayerShowWeights(Layer * l)
{
    int i, j;
    Connection *c = l->c;
    real sum = 0.0f;
    for (i = 0; i < l->n_inputs + 1 /*bias */ ; i++) {
        for (j = 0; j < l->n_outputs; j++) {
            sum += (c->w) * (c->w); //printf ("%f ", c->w);
            printf("%f ", c->w);
            c++;
        }
    }
    //  printf ("%f ", sum);
    return sum;
}


//==========================================================
// ANN_ShowInputs()
//----------------------------------------------------------
real ANN_ShowInputs(ANN * ann)
{
    LISTITEM *c;
    real sum = 0.0f;
    c = FirstListItem(ann->c);
    while (c) {
        Layer *l = (Layer *) c->obj;
        sum += ANN_LayerShowInputs(l);
        c = NextListItem(ann->c);
    }
    return sum;
}

//==========================================================
// ANN_LayerShowInputs()
//----------------------------------------------------------
real ANN_LayerShowInputs(Layer * l)
{
    int i;
    real sum = 0.0f;
    for (i = 0; i < l->n_inputs; i++) {
        printf("#%f ", l->x[i]);
    }
    printf("-->");
    for (i = 0; i < l->n_outputs; i++) {
        printf("#(%f)%f ", l->f(l->z[i]), l->y[i]);
    }

    printf("\n");
    return sum;
}


//==========================================================
// ANN_ShowOutputs()
//----------------------------------------------------------
void ANN_ShowOutputs(ANN * ann)
{
    int i;

    for (i = 0; i < ann->n_outputs; i++) {
        printf("%f ", ann->y[i]);
    }
    printf("\n");
}



//==========================================================
// ANN_SetOutputsToLinear()
//----------------------------------------------------------
void ANN_SetOutputsToLinear(ANN * ann)
{
    LISTITEM *c;
    c = LastListItem(ann->c);
    if (c) {
        Layer *l = (Layer *) c->obj;
        l->f = &linear;
        l->f_d = &linear_d;
    } else {
        Serror("Could not set outputs to linear\n");
    }
}

//==========================================================
// ANN_SetOutputsToTanH()
//----------------------------------------------------------
void ANN_SetOutputsToTanH(ANN * ann)
{
    LISTITEM *c;
    c = LastListItem(ann->c);
    if (c) {
        Layer *l = (Layer *) c->obj;
        l->f = &htan;
        l->f_d = &htan_d;
    } else {
        Serror("Could not set outputs to TanH\n");
    }
}


//==========================================================
// Exp()
//----------------------------------------------------------
real Exp(real x)
{
    return (real) exp(x);
}

//==========================================================
// Exp_d()
//----------------------------------------------------------
real Exp_d(real x)
{
    return x;
}

//==========================================================
// htan()
//----------------------------------------------------------
real htan(real x)
{
    return (real) tanh(x);
}

//==========================================================
// htan_d()
//----------------------------------------------------------
real htan_d(real x)
{
    real f = (real) tanh(x);
    return (real) (1.0 - f * f);
}

//==========================================================
// dtan()
//----------------------------------------------------------
real dtan(real x)
{
    if (x>1.0) {
        return 1.0;
    } else if (x<1.0) {
        return -1.0;
    }
    return x;
}

//==========================================================
// dtan_d()
//----------------------------------------------------------
real dtan_d(real x)
{
    if (x>1.0) {
        return 0.0;
    } else if (x<-1.0) {
        return 0.0;
    }
    return 1.0;
}

//==========================================================
// linear()
//----------------------------------------------------------
real linear(real x)
{
    return x;
}

//==========================================================
// linear_d()
//----------------------------------------------------------
real linear_d(real x)
{
    return 1.0f;
}

static inline bool CheckMatchingToken (const char* tag, StringBuffer* buf, FILE* f)
{
    int l = 1+strlen(tag);
    buf = SetStringBufferLength (buf, l);
    if (buf==NULL) {
        return false;
    }
    fread(buf->c, sizeof(char), l, f);

    if (strcmp(tag,buf->c)) {
        fprintf (stderr, "Expected tag <%s>, found <%s>.\n", tag, buf->c);
        return false;
    }
    return true;
}

static inline void WriteToken (const char* tag, FILE* f)
{
    fwrite (tag, sizeof(char), 1+strlen(tag), f);
}

ANN* LoadANN(char* filename)
{
    FILE* f = fopen (filename, "rb");
    if (f) {
        ANN* ann = LoadANN (f);
        fclose (f);
        return ann;
    }
    return NULL;
}
int SaveANN(ANN* ann, char* filename)
{
    FILE* f = fopen (filename, "wb");
    if (f) {
        int r = SaveANN (ann, f);
        fclose (f);
        return r;
    }
    return -1;
}

ANN* LoadANN(FILE* f)
{
    if (f==NULL) {
        return NULL;
    }
    StringBuffer* rtag = NewStringBuffer (256);
    CheckMatchingToken("VSOUND_ANN", rtag, f);
    int n_inputs;
    int n_outputs;
    fread(&n_inputs, sizeof(int), 1, f);
    fread(&n_outputs, sizeof(int), 1, f);
    ANN* ann = NewANN (n_inputs, n_outputs);
    CheckMatchingToken("Layer Data", rtag, f);
    int n_layers;
    fread(&n_layers, sizeof(int), 1, f);
    for (int i=0; i<n_layers-1; i++) {
        int layer_type;
        CheckMatchingToken("TYPE", rtag, f);
        fread(&layer_type, sizeof(int), 1, f);
        int nhu;
        CheckMatchingToken("UNITS", rtag, f);
        fread(&nhu, sizeof(int), 1, f);
        if (layer_type==0) {
            ANN_AddHiddenLayer(ann, nhu);
        } else {
            ANN_AddRBFHiddenLayer(ann, nhu);
        }
    }
    {
        int layer_type =0;
        ANN_Init(ann);
        CheckMatchingToken("Output Type", rtag, f);
        fread(&layer_type, sizeof(int), 1, f);
        if (layer_type==0) {
            ANN_SetOutputsToLinear(ann);    
        } else {
            ANN_SetOutputsToTanH(ann);
        }
    }

    LISTITEM* list_item = FirstListItem(ann->c);
    while (list_item) {
        Layer* l = (Layer*) list_item->obj;
        CheckMatchingToken("Connections", rtag, f);
        int size = (l->n_inputs + 1 /*bias*/) * l->n_outputs;
        fread(l->c, size, sizeof(Connection), f);
        list_item = NextListItem (ann->c);
    }
    CheckMatchingToken("END", rtag, f);

    FreeStringBuffer (&rtag);
    return ann;
}

int SaveANN(ANN* ann, FILE* f)
{
    if (f==NULL) {
        return -1;
    }
    
    StringBuffer* rtag = NewStringBuffer (256);

    WriteToken("VSOUND_ANN", f);
    fwrite(&ann->n_inputs, sizeof(int), 1, f);
    fwrite(&ann->n_outputs, sizeof(int), 1, f);
    WriteToken("Layer Data", f);
    int n_layers = 0;
    LISTITEM* list_item = FirstListItem(ann->c);
    while (list_item) {
        n_layers++;
        list_item = NextListItem (ann->c);
    }
    fwrite(&n_layers, sizeof(int), 1, f);
    list_item = FirstListItem(ann->c);
    for (int i=0; i<n_layers-1; i++) {
        Layer* l = (Layer*) list_item->obj;

        int layer_type = 0;
        WriteToken("TYPE", f);
        fwrite(&layer_type, sizeof(int), 1, f);

        int nhu = l->n_outputs;
        WriteToken("UNITS", f);
        fwrite(&nhu, sizeof(int), 1, f);
        list_item = NextListItem (ann->c);
    }
    WriteToken("Output Type", f);
    {
        int layer_type = 0;
        LISTITEM *c;
        c = LastListItem(ann->c);
        if (c) {
            Layer *l = (Layer *) c->obj;
            if (l->f==&linear) {
                layer_type = 0;
            } else {
                layer_type = 1;
            }
        }
        fwrite(&layer_type, sizeof(int), 1, f);
    }
    list_item = FirstListItem(ann->c); 
    while(list_item) {
        Layer* l = (Layer*) list_item->obj;
        WriteToken("Connections", f);
        int size = (l->n_inputs + 1 /*bias*/) * l->n_outputs;
        fwrite(l->c, size, sizeof(Connection), f);
        list_item = NextListItem(ann->c);
    }
    WriteToken("END", f);

    FreeStringBuffer (&rtag);
    return 0;
}