ftn.hpp

Go to the documentation of this file.

#ifndef MLPACK_METHODS_RL_ENVIRONMENT_FTN_HPP
#define MLPACK_METHODS_RL_ENVIRONMENT_FTN_HPP

#include <mlpack/core.hpp>
#include "ftn_param.hpp"

namespace mlpack{
namespace rl{

class FruitTreeNavigation
{
 public:
  class State
  {
   public:
    State(): data(dimension) { /* Nothing to do here. */ }

    State(const arma::colvec& data) : data(data)
    { /* Nothing to do here. */ }

    arma::vec& Data() { return data; }

    double Row() const { return data[0]; }
    double& Row() { return data[0]; }

    double Column() const { return data[1]; }
    double& Column() { return data[1]; }

    static constexpr size_t dimension = 2;

   private:
    arma::vec data;
  };

  class Action
  {
   public:
    enum actions
    {
      left,
      right,
    };
    // To store the action.
    Action::actions action;

    // Track the size of the action space.
    static const size_t size = 2;
  };

  FruitTreeNavigation(const size_t maxSteps = 500,
      const size_t depth = 6) :
      maxSteps(maxSteps),
      stepsPerformed(0)
  {
    FruitTree tree(depth);
  }

  arma::vec Sample(const State& state,
                   const Action& action,
                   State& nextState)
  {
    // Update the number of steps performed.
    stepsPerformed++;

    // Make a vector to estimate nextstate.
    arma::vec currentState {state.Row(), state.Column()};
    arma::vec direction = std::unordered_map<Action::actions, arma::vec>({
            { Action::actions::left,  arma::vec({1, currentState(1)}) },
            { Action::actions::right,  arma::vec({1, currentState(1) + 1}) }
        })[action.action];

    arma::vec currentNextState = currentState + direction;
    nextState.Row() = currentNextState[0];
    nextState.Column() = currentNextState[1];

    // Check if the episode has terminated.
    bool done = IsTerminal(nextState);

    // Do not reward the agent if time ran out.
    if (done && maxSteps != 0 && stepsPerformed >= maxSteps)
      return arma::zeros(rewardSize);

    return fruitTree.GetReward(state);
  };

  arma::vec Sample(const State& state, const Action& action)
  {
    State nextState;
    return Sample(state, action, nextState);
  }

  State InitialSample()
  {
    stepsPerformed = 0;
    return State(arma::zeros<arma::vec>(2));
  }

  bool IsTerminal(const State& state) const
  {
    if (maxSteps != 0 && stepsPerformed >= maxSteps)
    {
      Log::Info << "Episode terminated due to the maximum number of steps"
          " being taken.";
      return true;
    }
    else if (state.Row() == fruitTree.Depth())
    {
      Log::Info << "Episode terminated due to reaching leaf node.";
      return true;
    }
    return false;
  }
  
  size_t StepsPerformed() const { return stepsPerformed; }

  size_t MaxSteps() const { return maxSteps; }
  size_t& MaxSteps() { return maxSteps; }

  static constexpr size_t rewardSize = 6;

 private:

  /*
   *  A Fruit Tree is a full binary tree. Each node of this tree represents a state.
   *  Non-leaf nodes yield a null vector R^{6} reward. Leaf nodes have a pre-defined
   *  set of reward vectors such that they lie on a Convex Convergence Set (CCS).
   */
  class FruitTree
  {
    public:
      FruitTree(const size_t depth) : depth(depth)
      { 
        if (std::find(validDepths.begin(), validDepths.end(), depth) == validDepths.end())
        {
          throw std::logic_error("FruitTree()::FruitTree: Invalid depth value: " + std::to_string(depth) + " provided. "
                                 "Only depth values of: 5, 6, 7 are allowed.");
        }

        arma::mat branches = arma::zeros((rewardSize, (size_t) std::pow(2, depth - 1)));
        tree = arma::join_rows(branches, Fruits());
      }

      // Extract array index from {(row, column)} representation.
      size_t GetIndex(const State& state) const
      {
        return static_cast<size_t>(std::pow(2, state.Row() - 1) + state.Column());
      }

      // Yield reward from the current node.
      arma::vec GetReward(const State& state) const
      {
        return tree.col(GetIndex(state));
      }

      arma::mat Fruits() const { return ConvexSetMap.at(depth); }

      size_t Depth() const { return depth; }
    
    private:
      size_t depth;

      arma::mat tree;

      const std::array<size_t, 3> validDepths {5, 6, 7};
  };
  
  size_t maxSteps;

  size_t stepsPerformed;

  FruitTree fruitTree;
  };
};

} // namespace rl
} // namespace mlpack

#endif