ImportanceSampling.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_MDP_IMPORTANCE_SAMPLING_HEADER_FILE
#define AI_TOOLBOX_MDP_IMPORTANCE_SAMPLING_HEADER_FILE
 
#include <AIToolbox/MDP/Algorithms/Utils/OffPolicyTemplate.hpp>
 
namespace AIToolbox::MDP {
    class ImportanceSampling : public OffPolicyControl<ImportanceSampling> {
        public:
            using Parent = OffPolicyControl<ImportanceSampling>;
 
            ImportanceSampling(const PolicyInterface & behaviour, const double discount = 1.0,
                     const double alpha = 0.1, const double tolerance = 0.001, const double epsilon = 0.1) :
                    Parent(behaviour.getS(), behaviour.getA(), discount, alpha, tolerance, epsilon),
                    behaviour_(behaviour) {}
 
        private:
            friend Parent;
            double getTraceDiscount(const size_t s, const size_t a, const size_t, const double, const size_t maxA) const {
                const auto prob = epsilon_ / A + (a == maxA) * (1.0 - epsilon_);
                return prob / behaviour_.getActionProbability(s, a);
            }
 
            const PolicyInterface & behaviour_;
    };
 
    class ImportanceSamplingEvaluation : public OffPolicyEvaluation<ImportanceSamplingEvaluation> {
        public:
            using Parent = OffPolicyEvaluation<ImportanceSamplingEvaluation>;
 
            ImportanceSamplingEvaluation(const PolicyInterface & target, const PolicyInterface & behaviour,
                               const double discount, const double alpha, const double tolerance) :
                    Parent(target, discount, alpha, tolerance),
                    behaviour_(behaviour) {}
 
        private:
            friend Parent;
            double getTraceDiscount(const size_t s, const size_t a, const size_t, const double) const {
                return target_.getActionProbability(s, a) / behaviour_.getActionProbability(s, a);
            }
 
            const PolicyInterface & behaviour_;
    };
}
 
#endif