AMDP.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_POMDP_AMDP_HEADER_FILE
#define AI_TOOLBOX_POMDP_AMDP_HEADER_FILE
 
#include <cmath>
 
#include <AIToolbox/Types.hpp>
#include <AIToolbox/POMDP/Types.hpp>
#include <AIToolbox/POMDP/TypeTraits.hpp>
#include <AIToolbox/POMDP/Utils.hpp>
#include <AIToolbox/MDP/Model.hpp>
#include <AIToolbox/MDP/SparseModel.hpp>
#include <AIToolbox/POMDP/Algorithms/Utils/BeliefGenerator.hpp>
 
namespace AIToolbox::POMDP {
    class AMDP {
        public:
            using Discretizer = std::function<size_t(const Belief&)>;
 
            AMDP(size_t nBeliefs, size_t entropyBuckets);
 
 
            void setBeliefSize(size_t nBeliefs);
 
            void setEntropyBuckets(size_t buckets);
 
            size_t getBeliefSize() const;
 
            size_t getEntropyBuckets() const;
 
            template <IsModel M>
            std::tuple<MDP::Model, Discretizer> discretizeDense(const M& model);
 
            template <IsModel M>
            std::tuple<MDP::SparseModel, Discretizer> discretizeSparse(const M& model);
 
        private:
            Discretizer makeDiscretizer(size_t S);
 
            size_t beliefSize_, buckets_;
    };
 
    template <IsModel M>
    std::tuple<MDP::Model, AMDP::Discretizer> AMDP::discretizeDense(const M& model) {
        const size_t S = model.getS(), A = model.getA(), O = model.getO();
        const size_t S1 = S * buckets_;
 
        BeliefGenerator bGen(model);
        const auto beliefs = bGen(beliefSize_);
 
        auto T = MDP::Model::TransitionMatrix(A, Matrix2D::Zero(S1, S1));
        auto R = MDP::Model::RewardMatrix    (S1, A);
        R.setZero();
 
        const auto discretizer = makeDiscretizer(S);
 
        Belief b1(S);
        for ( const auto & b : beliefs ) {
            const size_t s = discretizer(b);
 
            for ( size_t a = 0; a < A; ++a ) {
                const double r = beliefExpectedReward(model, b, a);
 
                for ( size_t o = 0; o < O; ++o ) {
                    updateBeliefUnnormalized(model, b, a, o, &b1);
                    const auto p = b1.sum();
                    if (checkDifferentSmall(0.0, p)) {
                        b1 /= p;
                        const size_t s1 = discretizer(b1);
 
                        T[a](s, s1) += p;
                        R(s, a)     += p * r;
                    }
                }
            }
        }
 
        for ( size_t a = 0; a < A; ++a )
            for ( size_t s = 0; s < S1; ++s ) {
                R(s, a) /= T[a].row(s).sum();
 
                const double sum = T[a].row(s).sum();
                if ( checkEqualSmall(sum, 0.0) ) T[a](s, s) = 1.0;
                else T[a].row(s) /= sum;
            }
 
        return std::make_tuple(MDP::Model(NO_CHECK, S1, A, std::move(T), std::move(R), model.getDiscount()), std::move(discretizer));
    }
 
    template <IsModel M>
    std::tuple<MDP::SparseModel, AMDP::Discretizer> AMDP::discretizeSparse(const M& model) {
        const size_t S = model.getS(), A = model.getA(), O = model.getO();
        const size_t S1 = S * buckets_;
 
        BeliefGenerator<M> bGen(model);
        const auto beliefs = bGen(beliefSize_);
 
        auto T = MDP::SparseModel::TransitionMatrix(A, SparseMatrix2D(S1, S1));
        auto R = MDP::SparseModel::RewardMatrix    (S1, A);
 
        auto discretizer = makeDiscretizer(S);
 
        Belief b1(S);
        for ( const auto & b : beliefs ) {
            const size_t s = discretizer(b);
 
            for ( size_t a = 0; a < A; ++a ) {
                const double r = beliefExpectedReward(model, b, a);
 
                for ( size_t o = 0; o < O; ++o ) {
                    updateBeliefUnnormalized(model, b, a, o, &b1);
                    const auto p = b1.sum();
                    if (checkDifferentSmall(0.0, p)) {
                        b1 /= p;
                        const size_t s1 = discretizer(b1);
 
                        T[a].coeffRef(s, s1) += p;
                        if (checkDifferentSmall(0.0, r))
                            R.coeffRef(s, a) += p * r;
                    }
                }
            }
        }
 
        for ( size_t a = 0; a < A; ++a ) {
            for ( size_t s = 0; s < S1; ++s ) {
                if (checkDifferentSmall(0.0, R.coeff(s, a)))
                    R.coeffRef(s, a) /= T[a].row(s).sum();
 
                const double sum = T[a].row(s).sum();
                if ( checkEqualSmall(sum, 0.0) ) T[a].coeffRef(s, s) = 1.0;
                else T[a].row(s) /= sum;
            }
            T[a].makeCompressed();
        }
        R.makeCompressed();
 
        return std::make_tuple(MDP::SparseModel(NO_CHECK, S1, A, std::move(T), std::move(R), model.getDiscount()), std::move(discretizer));
    }
}
 
#endif