policy.h

Go to the documentation of this file.

// -*- Mode: c++ -*-
// copyright (c) 2004 by Christos Dimitrakakis <dimitrak@idiap.ch>
// $Id$


/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
#ifndef POLICY_H
#define POLICY_H

#include <cmath>
#include <cstdlib>
#include <cstdio>
#include <learning/ANN.h>
#include <learning/Distribution.h>

enum LearningMethod {QLearning, Sarsa, ForcedLearning, ELearning};
enum ConfidenceDistribution {SINGULAR, BOUNDED, GAUSSIAN, LAPLACIAN};

class DiscretePolicy
{
protected:

    enum LearningMethod learning_method; 
    int n_states; 
    int n_actions; 
    real** Q; 
    real** e; 
    real* eval; 
    real* sample; 
    real pQ; 
    int ps; 
    int pa; 
    real r; 
    real temp; 
    real tdError; 
    bool smax; 
    bool pursuit; 
    real** P; 
    real gamma; 
    real lambda; 
    real alpha; 
    real expected_r; 
    real expected_V; 
    int n_samples; 
    int min_el_state; 
    int max_el_state; 
    bool replacing_traces; 
    bool forced_learning; 
    bool confidence; 
    bool confidence_eligibility; 
    bool reliability_estimate; 
    enum ConfidenceDistribution confidence_distribution; 
    bool confidence_uses_gibbs; 
    real zeta; 
    real** vQ; 
    int confMax(real* Qs, real* vQs, real p = 1.0); 
    int confSample(real* Qs, real* vQs); 
    int softMax(real* Qs); 
    int eGreedy(real* Qs); 
    int argMax(real* Qs); 
public:
    DiscretePolicy (int n_states, int n_actions, real alpha=0.1, real gamma=0.8, real lambda=0.8, bool softmax = false, real randomness=0.1, real init_eval=0.0);
    virtual ~DiscretePolicy();
    virtual void setLearningRate (real alpha) {this->alpha = alpha;}
    virtual real getTDError () {return tdError;}
    virtual real getLastActionValue () {return Q[ps][pa];}
    virtual int SelectAction(int s, real r, int forced_a=-1);
    virtual void Reset();
    virtual void loadFile (char* f);
    virtual void saveFile (char* f);
    virtual void setQLearning();
    virtual void setELearning();
    virtual void setSarsa();
    virtual bool useConfidenceEstimates(bool confidence, real zeta=0.01, bool confidence_eligibility = false);
    virtual void setForcedLearning(bool forced);
    virtual void setRandomness (real epsilon);
    virtual void setGamma (real gamma);
    virtual void setPursuit (bool pursuit);
    virtual void setReplacingTraces (bool replacing);
    virtual void useSoftmax (bool softmax);
    virtual void setConfidenceDistribution (enum ConfidenceDistribution cd);
    virtual void useGibbsConfidence (bool gibbs);
    virtual void useReliabilityEstimate (bool ri);
    virtual void saveState (FILE* f);
};


#endif