Utils.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_MDP_UTILS_HEADER_FILE
#define AI_TOOLBOX_MDP_UTILS_HEADER_FILE
 
#include <stddef.h>
#include <AIToolbox/MDP/Types.hpp>
#include <AIToolbox/MDP/TypeTraits.hpp>
 
namespace AIToolbox::MDP {
    QFunction makeQFunction(size_t S, size_t A);
 
    ValueFunction makeValueFunction(size_t S);
 
    ValueFunction bellmanOperator(const QFunction & q);
 
    void bellmanOperatorInplace(const QFunction & q, ValueFunction * v);
 
    template <IsModel M>
    Matrix2D computeImmediateRewards(const M & model) {
        if constexpr(IsModelEigen<M>) {
            return model.getRewardFunction();
        } else {
            const auto S = model.getS();
            const auto A = model.getA();
 
            auto ir = QFunction(S, A);
            ir.setZero();
            for ( size_t s = 0; s < S; ++s )
                for ( size_t a = 0; a < A; ++a )
                    for ( size_t s1 = 0; s1 < S; ++s1 )
                        ir(s, a) += model.getTransitionProbability(s,a,s1) * model.getExpectedReward(s,a,s1);
            return ir;
        }
    }
 
    template <IsModel M>
    QFunction computeQFunction(const M & model, const Values & v, QFunction ir) {
        const auto A = model.getA();
 
        if constexpr(IsModelEigen<M>) {
            for ( size_t a = 0; a < A; ++a )
                ir.col(a).noalias() += model.getTransitionFunction(a) * v;
        } else {
            const auto S = model.getS();
            for ( size_t s = 0; s < S; ++s )
                for ( size_t a = 0; a < A; ++a )
                    for ( size_t s1 = 0; s1 < S; ++s1 )
                        ir(s, a) += model.getTransitionProbability(s,a,s1) * v[s1];
        }
        return ir;
    }
}
 
#endif