OldMDPModel.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_OLD_MDP_MODEL_HEADER_FILE
#define AI_TOOLBOX_OLD_MDP_MODEL_HEADER_FILE
 
#include <utility>
#include <random>
 
#include <AIToolbox/Types.hpp>
#include <AIToolbox/TypeTraits.hpp>
#include <AIToolbox/MDP/Types.hpp>
#include <AIToolbox/MDP/TypeTraits.hpp>
#include <AIToolbox/Utils/Core.hpp>
#include <AIToolbox/Utils/Probability.hpp>
#include <AIToolbox/Seeder.hpp>
 
class OldMDPModel {
    public:
        using TransitionMatrix = AIToolbox::DumbMatrix3D;
        using RewardMatrix     = AIToolbox::DumbMatrix3D;
 
        OldMDPModel(size_t s, size_t a, double discount = 1.0);
 
        template <AIToolbox::IsNaive3DMatrix T, AIToolbox::IsNaive3DMatrix R>
        OldMDPModel(size_t s, size_t a, const T & t, const R & r, double d = 1.0);
 
        template <AIToolbox::MDP::IsModel M>
        OldMDPModel(const M& model);
 
        template <AIToolbox::IsNaive3DMatrix T>
        void setTransitionFunction(const T & t);
 
        template <AIToolbox::IsNaive3DMatrix R>
        void setRewardFunction(const R & r);
 
        void setDiscount(double d);
 
        std::tuple<size_t, double> sampleSR(size_t s, size_t a) const;
 
        size_t getS() const;
 
        size_t getA() const;
 
        double getDiscount() const;
 
        double getTransitionProbability(size_t s, size_t a, size_t s1) const;
 
        double getExpectedReward(size_t s, size_t a, size_t s1) const;
 
        const TransitionMatrix & getTransitionFunction() const;
 
        const RewardMatrix & getRewardFunction() const;
 
        bool isTerminal(size_t s) const;
 
    private:
        size_t S, A;
        double discount_;
 
        TransitionMatrix transitions_;
        RewardMatrix rewards_;
 
        mutable AIToolbox::RandomEngine rand_;
 
        friend std::istream& operator>>(std::istream &is, OldMDPModel &);
};
 
template <AIToolbox::IsNaive3DMatrix T, AIToolbox::IsNaive3DMatrix R>
OldMDPModel::OldMDPModel(size_t s, size_t a, const T & t, const R & r, double d) : S(s), A(a), transitions_(boost::extents[S][A][S]), rewards_(boost::extents[S][A][S]),
    rand_(AIToolbox::Seeder::getSeed())
{
    setDiscount(d);
    setTransitionFunction(t);
    setRewardFunction(r);
}
 
template <AIToolbox::MDP::IsModel M>
OldMDPModel::OldMDPModel(const M& model) : S(model.getS()), A(model.getA()), discount_(model.getDiscount()), transitions_(boost::extents[S][A][S]), rewards_(boost::extents[S][A][S]),
    rand_(AIToolbox::Seeder::getSeed())
{
    for ( size_t s = 0; s < S; ++s )
        for ( size_t a = 0; a < A; ++a ) {
            for ( size_t s1 = 0; s1 < S; ++s1 ) {
                transitions_[s][a][s1] = model.getTransitionProbability(s, a, s1);
                rewards_    [s][a][s1] = model.getExpectedReward       (s, a, s1);
            }
            if ( ! AIToolbox::isProbability(S, transitions_[s][a]) )
                throw std::invalid_argument("Input transition matrix does not contain valid probabilities.");
        }
}
 
template <AIToolbox::IsNaive3DMatrix T>
void OldMDPModel::setTransitionFunction(const T & t) {
    for ( size_t s = 0; s < S; ++s )
        for ( size_t a = 0; a < A; ++a )
            if ( ! AIToolbox::isProbability(S, t[s][a]) )
                throw std::invalid_argument("Input transition matrix does not contain valid probabilities.");
 
    copyDumb3D(t, transitions_, S, A, S);
}
 
template <AIToolbox::IsNaive3DMatrix R>
void OldMDPModel::setRewardFunction( const R & r ) {
    copyDumb3D(r, rewards_, S, A, S);
}
 
OldMDPModel::OldMDPModel(size_t s, size_t a, double discount) : S(s), A(a), discount_(discount), transitions_(boost::extents[S][A][S]), rewards_(boost::extents[S][A][S]),
    rand_(AIToolbox::Seeder::getSeed())
{
    // Make transition matrix true probability
    for ( size_t s = 0; s < S; ++s )
        for ( size_t a = 0; a < A; ++a )
            transitions_[s][a][s] = 1.0;
}
 
inline std::tuple<size_t, double> OldMDPModel::sampleSR(size_t s, size_t a) const {
    size_t s1 = AIToolbox::sampleProbability(S, transitions_[s][a], rand_);
 
    return std::make_tuple(s1, rewards_[s][a][s1]);
}
 
inline double OldMDPModel::getTransitionProbability(size_t s, size_t a, size_t s1) const {
    return transitions_[s][a][s1];
}
 
inline double OldMDPModel::getExpectedReward(size_t s, size_t a, size_t s1) const {
    return rewards_[s][a][s1];
}
 
inline void OldMDPModel::setDiscount(double d) {
    if ( d <= 0.0 || d > 1.0 ) throw std::invalid_argument("Discount parameter must be in (0,1]");
    discount_ = d;
}
 
inline bool OldMDPModel::isTerminal(size_t s) const {
    bool answer = true;
    for ( size_t a = 0; a < A; ++a ) {
        if ( !AIToolbox::checkEqualSmall(1.0, transitions_[s][a][s]) ) {
            answer = false;
            break;
        }
    }
    return answer;
}
 
inline size_t OldMDPModel::getS() const { return S; }
inline size_t OldMDPModel::getA() const { return A; }
inline double OldMDPModel::getDiscount() const { return discount_; }
 
inline const OldMDPModel::TransitionMatrix & OldMDPModel::getTransitionFunction() const { return transitions_; }
inline const OldMDPModel::RewardMatrix &     OldMDPModel::getRewardFunction()     const { return rewards_; }
 
#endif