ValueIteration.hpp

Go to the documentation of this file.

#ifndef AI_TOOLBOX_MDP_VALUE_ITERATION_HEADER_FILE
#define AI_TOOLBOX_MDP_VALUE_ITERATION_HEADER_FILE
 
#include <AIToolbox/Logging.hpp>
#include <AIToolbox/MDP/Types.hpp>
#include <AIToolbox/MDP/TypeTraits.hpp>
#include <AIToolbox/MDP/Utils.hpp>
#include <AIToolbox/Utils/Probability.hpp>
 
namespace AIToolbox::MDP {
    class ValueIteration {
        public:
            ValueIteration(unsigned horizon, double tolerance = 0.001, ValueFunction v = {Values(), Actions(0)});
 
            template <IsModel M>
            std::tuple<double, ValueFunction, QFunction> operator()(const M & m);
 
            void setTolerance(double e);
 
            void setHorizon(unsigned h);
 
            void setValueFunction(ValueFunction v);
 
            double getTolerance() const;
 
            unsigned getHorizon() const;
 
            const ValueFunction & getValueFunction() const;
 
        private:
            // Parameters
            double tolerance_;
            unsigned horizon_;
            ValueFunction vParameter_;
 
            // Internals
            ValueFunction v1_;
    };
 
    template <IsModel M>
    std::tuple<double, ValueFunction, QFunction> ValueIteration::operator()(const M & model) {
        // Extract necessary knowledge from model so we don't have to pass it around
        const size_t S = model.getS();
        const size_t A = model.getA();
 
        {
            // Verify that parameter value function is compatible.
            const size_t size = vParameter_.values.size();
            if ( size != S ) {
                if ( size != 0 ) {
                    AI_LOGGER(AI_SEVERITY_WARNING, "Size of starting value function is incorrect, ignoring...");
                }
                // Defaulting
                v1_ = makeValueFunction(S);
            }
            else
                v1_ = vParameter_;
        }
 
        const auto & ir = [&]{
            if constexpr (IsModelEigen<M>) return model.getRewardFunction();
            else return computeImmediateRewards(model);
        }();
 
        unsigned timestep = 0;
        double variation = tolerance_ * 2; // Make it bigger
 
        Values val0;
        auto & val1 = v1_.values;
        QFunction q = makeQFunction(S, A);
 
        const bool useTolerance = checkDifferentSmall(tolerance_, 0.0);
        while ( timestep < horizon_ && (!useTolerance || variation > tolerance_) ) {
            ++timestep;
            AI_LOGGER(AI_SEVERITY_DEBUG, "Processing timestep " << timestep);
 
            val0 = val1;
 
            // We apply the discount directly on the values vector.
            val1 *= model.getDiscount();
            q = computeQFunction(model, val1, ir);
 
            // Compute the new value function (note that also val1 is overwritten)
            bellmanOperatorInplace(q, &v1_);
 
            // We do this only if the tolerance specified is positive, otherwise we
            // continue for all the timesteps.
            if ( useTolerance )
                variation = (val1 - val0).cwiseAbs().maxCoeff();
        }
 
        // We do not guarantee that the Value/QFunctions are the perfect ones,
        // as we stop as within the given tolerance.
        return std::make_tuple(useTolerance ? variation : 0.0, std::move(v1_), std::move(q));
    }
}
 
#endif