OffPolicyTemplate.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_MDP_OFF_POLICY_TEMPLATE_HEADER_FILE
#define AI_TOOLBOX_MDP_OFF_POLICY_TEMPLATE_HEADER_FILE
 
#include <AIToolbox/MDP/Policies/PolicyInterface.hpp>
#include <AIToolbox/MDP/Types.hpp>
 
namespace AIToolbox::MDP {
    class OffPolicyBase {
        public:
            using Trace = std::tuple<size_t, size_t, double>;
            using Traces = std::vector<Trace>;
 
            OffPolicyBase(size_t s, size_t a, double discount = 1.0, double alpha = 0.1, double tolerance = 0.001);
 
            void setLearningRate(double a);
 
            double getLearningRate() const;
 
            void setDiscount(double d);
 
            double getDiscount() const;
 
            void setTolerance(double t);
 
            double getTolerance() const;
 
            void clearTraces();
 
            const Traces & getTraces() const;
 
            void setTraces(const Traces & t);
 
            size_t getS() const;
 
            size_t getA() const;
 
            const QFunction & getQFunction() const;
 
            void setQFunction(const QFunction & qfun);
 
        protected:
            size_t S, A;
            double discount_, alpha_, tolerance_;
 
            void updateTraces(size_t s, size_t a, double error, double traceDiscount);
 
            QFunction q_;
            Traces traces_;
    };
 
    template <typename Derived>
    class OffPolicyEvaluation : public OffPolicyBase {
        public:
            using Parent = OffPolicyBase;
 
            OffPolicyEvaluation(const PolicyInterface & target, double discount = 1.0,
                    double alpha = 0.1, double tolerance = 0.001);
 
            void stepUpdateQ(const size_t s, const size_t a, const size_t s1, const double rew);
 
        protected:
            const PolicyInterface & target_;
    };
 
    template <typename Derived>
    class OffPolicyControl : public OffPolicyBase {
        public:
            using Parent = OffPolicyBase;
 
            OffPolicyControl(size_t s, size_t a, double discount = 1.0, double alpha = 0.1,
                        double tolerance = 0.001, double epsilon = 0.1);
 
            void stepUpdateQ(const size_t s, const size_t a, const size_t s1, const double rew);
 
            void setEpsilon(double e);
 
            double getEpsilon() const;
 
        protected:
            double epsilon_;
    };
 
    template <typename Derived>
    void OffPolicyEvaluation<Derived>::stepUpdateQ(const size_t s, const size_t a, const size_t s1, const double rew) {
        auto expectedQ = 0.0;
        for (size_t a = 0; a < A; ++a)
            expectedQ += q_(s1, a) * target_.getActionProbability(s1, a);
 
        const auto error = alpha_ * ( rew + discount_ * expectedQ - q_(s, a) );
        const auto traceDiscount = discount_ * static_cast<Derived*>(this)->getTraceDiscount(s, a, s1, rew);
 
        updateTraces(s, a, error, traceDiscount);
    }
 
    template <typename Derived>
    void OffPolicyControl<Derived>::stepUpdateQ(const size_t s, const size_t a, const size_t s1, const double rew) {
        // The basic idea here is that differently from the evaluation, we want
        // to do a maximization. At the same time, to work we need to "round
        // the edges" a bit, and that's why we assume an epsilon-greedy policy.
        //
        // The expected value we can compute easily since each action has the
        // same probability of being chosen, but for the greedy one which is
        // more likely.
        size_t maxA;
        double expectedQ = 0.0;
        double maxV = std::numeric_limits<double>::lowest();
        for (size_t aa = 0; aa < A; ++aa) {
            expectedQ += q_(s1, aa);
            if (maxV < q_(s1, aa)) {
                maxA = aa;
                maxV = q_(s1, aa);
            }
        }
        expectedQ *= epsilon_ / A;
        expectedQ += (1.0 - epsilon_) * maxV;
 
        const auto error = alpha_ * ( rew + discount_ * expectedQ - q_(s, a) );
        const auto traceDiscount = discount_ * static_cast<Derived*>(this)->getTraceDiscount(s, a, s1, rew, maxA);
 
        updateTraces(s, a, error, traceDiscount);
    }
 
    template <typename Derived>
    OffPolicyEvaluation<Derived>::OffPolicyEvaluation(
        const PolicyInterface & target,
        const double discount, const double alpha, const double tolerance
    ) :
        Parent(target.getS(), target.getA(), discount, alpha, tolerance),
        target_(target) {}
 
    template <typename Derived>
    OffPolicyControl<Derived>::OffPolicyControl(
        const size_t s, const size_t a, const double discount,
        const double alpha, const double tolerance, const double epsilon
    ) :
        Parent(s, a, discount, alpha, tolerance)
    {
        setEpsilon(epsilon);
    }
 
    template <typename Derived>
    void OffPolicyControl<Derived>::setEpsilon(const double e) {
        if ( e < 0.0 || e > 1.0 ) throw std::invalid_argument("Epsilon must be >= 0 and <= 1");
        epsilon_ = e;
    }
 
    template <typename Derived>
    double OffPolicyControl<Derived>::getEpsilon() const {
        return epsilon_;
    }
}
 
#endif