RetraceL.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_MDP_RETRACE_L_HEADER_FILE
#define AI_TOOLBOX_MDP_RETRACE_L_HEADER_FILE
 
#include <AIToolbox/MDP/Algorithms/Utils/OffPolicyTemplate.hpp>
 
namespace AIToolbox::MDP {
    class RetraceL : public OffPolicyControl<RetraceL> {
        public:
            using Parent = OffPolicyControl<RetraceL>;
 
            RetraceL(const PolicyInterface & behaviour, const double discount = 1.0, const double alpha = 0.1,
                        const double lambda = 0.9,const double tolerance = 0.001, const double epsilon = 0.1) :
                    Parent(behaviour.getS(), behaviour.getA(), discount, alpha, tolerance, epsilon),
                    behaviour_(behaviour)
            {
                setLambda(lambda);
            }
 
            void setLambda(double l) {
                if ( l < 0.0 || l > 1.0 ) throw std::invalid_argument("Lambda parameter must be in [0,1]");
                lambda_ = l;
            }
 
            double getLambda() const { return lambda_; }
 
        private:
            friend Parent;
            double getTraceDiscount(const size_t s, const size_t a, const size_t, const double, const size_t maxA) const {
                const auto prob = epsilon_ / A + (a == maxA) * (1.0 - epsilon_);
                return lambda_ * std::min(1.0, prob / behaviour_.getActionProbability(s, a));
            }
 
            double lambda_;
            const PolicyInterface & behaviour_;
    };
 
    class RetraceLEvaluation : public OffPolicyEvaluation<RetraceLEvaluation> {
        public:
            using Parent = OffPolicyEvaluation<RetraceLEvaluation>;
 
            RetraceLEvaluation(const PolicyInterface & target, const PolicyInterface & behaviour,
                               const double discount, const double alpha, const double lambda, const double tolerance) :
                    Parent(target, discount, alpha, tolerance),
                    behaviour_(behaviour)
            {
                setLambda(lambda);
            }
 
            void setLambda(double l) {
                if ( l < 0.0 || l > 1.0 ) throw std::invalid_argument("Lambda parameter must be in [0,1]");
                lambda_ = l;
            }
 
            double getLambda() const { return lambda_; }
 
        private:
            friend Parent;
 
            double getTraceDiscount(const size_t s, const size_t a, const size_t, const double) const {
                return lambda_ * std::min(1.0, target_.getActionProbability(s, a) / behaviour_.getActionProbability(s, a));
            }
 
            double lambda_;
            const PolicyInterface & behaviour_;
    };
}
 
#endif