Model.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_MDP_MODEL_HEADER_FILE
#define AI_TOOLBOX_MDP_MODEL_HEADER_FILE
 
#include <utility>
#include <random>
 
#include <AIToolbox/Seeder.hpp>
#include <AIToolbox/Types.hpp>
#include <AIToolbox/MDP/Types.hpp>
#include <AIToolbox/MDP/TypeTraits.hpp>
#include <AIToolbox/Utils/Core.hpp>
#include <AIToolbox/Utils/Probability.hpp>
 
namespace AIToolbox::MDP {
    class Model {
        public:
            using TransitionMatrix   = Matrix3D;
            using RewardMatrix       = Matrix2D;
 
            Model(size_t s, size_t a, double discount = 1.0);
 
            template <IsNaive3DMatrix T, IsNaive3DMatrix R>
            Model(size_t s, size_t a, const T & t, const R & r, double d = 1.0);
 
            template <IsModel M>
            Model(const M& model);
 
            Model(NoCheck, size_t s, size_t a, TransitionMatrix && t, RewardMatrix && r, double d);
 
            template <IsNaive3DMatrix T>
            void setTransitionFunction(const T & t);
 
            void setTransitionFunction(const TransitionMatrix & t);
 
            template <IsNaive3DMatrix R>
            void setRewardFunction(const R & r);
 
            void setRewardFunction(const RewardMatrix & r);
 
            void setDiscount(double d);
 
            std::tuple<size_t, double> sampleSR(size_t s, size_t a) const;
 
            size_t getS() const;
 
            size_t getA() const;
 
            double getDiscount() const;
 
            double getTransitionProbability(size_t s, size_t a, size_t s1) const;
 
            double getExpectedReward(size_t s, size_t a, size_t s1) const;
 
            const TransitionMatrix & getTransitionFunction() const;
 
            const Matrix2D & getTransitionFunction(size_t a) const;
 
            const RewardMatrix & getRewardFunction() const;
 
            bool isTerminal(size_t s) const;
 
        private:
            size_t S, A;
            double discount_;
 
            TransitionMatrix transitions_;
            RewardMatrix rewards_;
 
            mutable RandomEngine rand_;
    };
 
    template <IsNaive3DMatrix T, IsNaive3DMatrix R>
    Model::Model(const size_t s, const size_t a, const T & t, const R & r, const double d) :
            S(s), A(a), transitions_(A, Matrix2D(S, S)),
            rewards_(S, A), rand_(Seeder::getSeed())
    {
        setDiscount(d);
        setTransitionFunction(t);
        setRewardFunction(r);
    }
 
    template <IsModel M>
    Model::Model(const M& model) :
            S(model.getS()), A(model.getA()), transitions_(A, Matrix2D(S, S)),
            rewards_(S, A), rand_(Seeder::getSeed())
    {
        setDiscount(model.getDiscount());
        rewards_.setZero();
        for ( size_t a = 0; a < A; ++a )
            for ( size_t s = 0; s < S; ++s ) {
                for ( size_t s1 = 0; s1 < S; ++s1 ) {
                    transitions_[a](s, s1) = model.getTransitionProbability(s, a, s1);
                    rewards_    (s, a)     += model.getExpectedReward       (s, a, s1) * transitions_[a](s, s1);
                }
                if ( !isProbability(S, transitions_[a].row(s)) )
                    throw std::invalid_argument("Input transition matrix does not contain valid probabilities.");
            }
    }
 
    template <IsNaive3DMatrix T>
    void Model::setTransitionFunction(const T & t) {
        if (!isProbability(S, A, S, t))
            throw std::invalid_argument("Input transition matrix does not contain valid probabilities.");
 
        for ( size_t s = 0; s < S; ++s )
            for ( size_t a = 0; a < A; ++a )
                for ( size_t s1 = 0; s1 < S; ++s1 )
                    transitions_[a](s, s1) = t[s][a][s1];
    }
 
    template <IsNaive3DMatrix R>
    void Model::setRewardFunction(const R & r) {
        rewards_.setZero();
        for ( size_t s = 0; s < S; ++s )
            for ( size_t a = 0; a < A; ++a )
                for ( size_t s1 = 0; s1 < S; ++s1 )
                    rewards_(s, a) += r[s][a][s1] * transitions_[a](s, s1);
    }
}
 
#endif