[Reinforcement Learning / review article / c++] Policy Gradient (Two-armed Bandit)

깊은 이해를 위해 [Reinforcement Learning] Policy Gradient (Two-armed Bandit) 포스팅의 코드

즉 https://medium.com/@awjuliani/super-simple-reinforcement-learning-tutorial-part-1-fd544fab149#.ywa46v109

이 분의 코드를  c++ 를 사용하여 코딩을 해봤습니다.

그동안 tensorflow 가 해주었던 부분 특히 블랙박스같은 부분을 직접 구현해보는 일은 공부에 많은 도움이 될 것 같네요.

제 코드와 원래 코드의 유일한 차이점은 weights 를 처음 초기화 할 때 1이 아닌 2로 했다는 부분입니다. 최초 값을 1으로 초기화할 경우 ln 에 넣었을 때 0이 나와 훈련이 되지 않았습니다.

main.cpp

#include <iostream>
#include <random>
#include "jaykay/Matrix.h"
 
using namespace std;
 
double PullBandit(double InBandit)
{
    random_device rd;
    mt19937 gen(rd());
    normal_distribution<double> Rnd(0.0, 1.0);
 
    if (Rnd(gen) > InBandit)
    {
        return 1.0;
    }
    else
    {
        return -1.0;
    }
}
 
int PolicyForward(Matrix1D& Weights, float E, int NumBandits, const float* Bandits)
{
    if (rand() % 10 / 10.0f < E)
    {
        return rand() % NumBandits;
    }
    else
    {
        return Weights.ArgMax();
    }
}
 
void PolicyBackward(Matrix1D& Weights, int Action, double Reward, float LearningRate)
{
    double Loss = -1.0 * (log(Weights.Value[Action]) * Reward);
    Weights.Value[Action] += LearningRate * Loss;
}
 
int main()
{
    srand((unsigned)time(0));
 
    const int NumBandits = 4;
    const float Bandits[NumBandits] = {0.2f, 0.f, -0.2f, 5.f};
    const int AnswerIndex = 3;
    const float E = 0.1f;
    const float LearningRate = 0.001f;
 
    Matrix1D Weights(NumBandits, 2.0);
 
    const int TotalEpisodes = 1000;
    double TotalReward[NumBandits] = {0.0, };
 
    for (int i = 0 ; i < TotalEpisodes; i++)
    {
        int Action = PolicyForward(Weights, E, NumBandits, Bandits);
        double Reward = PullBandit(Bandits[Action]);
        PolicyBackward(Weights, Action, Reward, LearningRate);
        TotalReward[Action] += Reward;
 
        if (i % 50 == 0)
        {
            cout << "Running reward for the " << NumBandits << " bandits: [";
            for (int j = 0; j < NumBandits; j++)
            {
                cout << TotalReward[j] << " ";
            }
            cout << "]" << endl;
        }
    }
    int Y = Weights.ArgMax();
    cout << "The agent thinks bandit " << Y + 1 << " is the most promising...." << endl;
    if (Y == AnswerIndex)
    {
        cout << "...and it was right!" << endl;
    }
    else
    {
        cout << "...and it was wrong!" << endl;
    }
 
    return 0;
}

Matrix.h

//
// Created by JK Choi on 2017. 3. 19..
//
 
#ifndef RLBANDITS_MATRIX_H
#define RLBANDITS_MATRIX_H
 
 
class Matrix1D
{
public:
 
    Matrix1D(int InColumn);
    Matrix1D(int InColumn, double DefaultValue);
    ~Matrix1D();
 
    void Init(int InColumn);
    void Print();
    int ArgMax();
    Matrix1D Slice(int Begin, int Size);
    Matrix1D Log();
    friend Matrix1D operator*(const Matrix1D& Left, double Right);
 
    int Column;
    double *Value;
};
 
class Matrix2D
{
public:
 
    Matrix2D(int InRow, int InColumn);
    Matrix2D(int InRow, int InColumn, double DefaultValue);
    ~Matrix2D();
 
    void Init(int InRow, int InColumn);
    
    friend Matrix2D operator*(const Matrix2D& Left, const Matrix2D& Right);
    friend Matrix2D operator*(const Matrix2D& Left, double Right);
 
    void Print();
    Matrix1D ArgMax(int Axis);
    Matrix2D Slice(int Begin[2], int Size[2]);
 
    int Row;
    int Column;
    double **Value;
};
 
namespace MatrixMath
{
    Matrix1D Log(const Matrix1D& In);
    Matrix2D Log(const Matrix2D& In);
}
 
#endif //RLBANDITS_MATRIX_H
 

Matrix.cpp

//
// Created by JK Choi on 2017. 3. 19..
//
 
#include "Matrix2D.h"
#include <stdio.h>
#include <string.h>
#include <iostream>
#include <cmath>
 
using namespace std;
 
/////////////////////////////////////////////////////////////////////////
// Matrix1D implementation
 
Matrix1D::Matrix1D(int InColumn)
{
    Init(InColumn);
}
 
Matrix1D::Matrix1D(int InColumn, double DefaultValue)
{
    Init(InColumn);
    for (int c = 0; c < Column; c++)
    {
        Value[c] = DefaultValue;
    }
}
 
Matrix1D::~Matrix1D()
{
    delete [] Value;
}
 
void Matrix1D::Init(int InColumn)
{
    Column = InColumn;
    Value = new double[Column];
    memset(Value, 0, sizeof(int) * Column);
}
 
void Matrix1D::Print()
{
    for (int c = 0; c < Column; c++)
    {
        if (c != 0)
        {
            cout << " ";
        }
        cout << Value[c];
    }
    cout << endl;
}
 
int Matrix1D::ArgMax()
{
    int Ret = 0;
    double MaxV = Value[0];
    for (int i = 1; i < Column; i++)
    {
        if (Value[i] > MaxV)
        {
            MaxV = Value[i];
            Ret = i;
        }
    }
    return Ret;
}
 
Matrix1D Matrix1D::Slice(int Begin, int Size)
{
    if (Begin + Size > Column)
    {
        cout << "InValid Begin and Size" << endl;
        return Matrix1D(0);
    }
    Matrix1D Ret(Size);
    for (int i = 0; i < Size; i++)
    {
        Ret.Value[i] = Value[Begin + i];
    }
 
    return Ret;
}
 
Matrix1D Matrix1D::Log()
{
    Matrix1D Ret(Column);
    for (int c = 0; c < Column; c++)
    {
        Ret.Value[c] = log(Value[c]);
    }
    return Ret;
}
 
Matrix1D operator*(const Matrix1D &Left, double Right)
{
    Matrix1D Ret(Left.Column);
    for (int c = 0; c < Left.Column; c++)
    {
        Ret.Value[c] = Left.Value[c] * Right;
    }
    return Ret;
}
 
 
///////////////////////////////////////////////////////////////////////////
// Matrix2D implementation
 
Matrix2D::Matrix2D(int InRow, int InColumn)
{
    Init(InRow, InColumn);
}
 
Matrix2D::Matrix2D(int InRow, int InColumn, double DefaultValue)
{
    Init(InRow, InColumn);
 
    for (int r = 0; r < Row; r++)
    {
        for (int c = 0; c < Column; c++)
        {
            Value[r][c] = DefaultValue;
        }
    }
}
 
Matrix2D::~Matrix2D()
{
    for (int i = 0; i < Row; i++)
    {
        delete [] Value[i];
    }
}
 
void Matrix2D::Init(int InRow, int InColumn)
{
    Row = InRow;
    Column = InColumn;
    Value = new double*[Row];
    for (int i = 0; i < Row; i++)
    {
        Value[i] = new double[Column];
        memset(Value[i], 0, sizeof(int) * Column);
    }
}
 
Matrix2D operator*(const Matrix2D &Left, const Matrix2D &Right)
{
    if (Left.Column != Right.Row)
    {
        cout << "Can not multiply" << endl;
        return Matrix2D(0, 0);
    }
    Matrix2D Ret(Left.Row, Right.Column);
    double TempValue = 0;
    for (int r = 0; r < Left.Row; r++)
    {
        for (int c = 0; c < Right.Column; c++)
        {
            TempValue = 0;
            for(int k = 0; k < Left.Column; k++)
            {
                TempValue += Left.Value[r][k] * Right.Value[k][c];
            }
            Ret.Value[r][c] = TempValue;
        }
    }
    return Ret;
}
 
Matrix2D operator*(const Matrix2D &Left, double Right)
{
    Matrix2D Ret(Left.Row, Left.Column);
    for (int r = 0; r < Left.Row; r++)
    {
        for (int c = 0; c < Left.Column; c++)
        {
            Ret.Value[r][c] = Left.Value[r][c] * Right;
        }
    }
    return Ret;
}
 
void Matrix2D::Print()
{
    for (int r = 0; r < Row; r++)
    {
        for (int c = 0; c < Column; c++)
        {
            if (c != 0)
            {
                cout << " ";
            }
            cout << Value[r][c];
        }
        cout << endl;
    }
}
 
Matrix1D Matrix2D::ArgMax(int Axis)
{
    if (Axis != 0 && Axis != 1)
    {
        cout << "InValid Axis" << endl;
        return Matrix1D(0);
    }
 
    Matrix1D Ret(Axis == 0 ? Column : Row);
 
    if (Axis == 0)
    {
        for (int c = 0; c < Column; c++)
        {
            int Index = 0;
            double MaxV = Value[0][c];
            for (int r = 1; r < Row; r++)
            {
                if (Value[r][c] > MaxV)
                {
                    MaxV = Value[r][c];
                    Index = r;
                }
            }
            Ret.Value[c] = Index;
        }
    }
    else
    {
        for (int r = 0; r < Row; r++)
        {
            int Index = 0;
            double MaxV = Value[r][0];
            for (int c = 1; c < Column; c++)
            {
                if (Value[r][c] > MaxV)
                {
                    MaxV = Value[r][c];
                    Index = c;
                }
            }
            Ret.Value[r] = Index;
        }
    }
 
    return Ret;
}
 
Matrix2D Matrix2D::Slice(int Begin[2], int Size[2])
{
    Matrix2D Ret(Size[0], Size[1]);
 
    for (int r = 0; r < Size[0]; r++)
    {
        for (int c = 0; c < Size[1]; c++)
        {
            Ret.Value[r][c] = Value[Begin[0] + r][Begin[1] + c];
        }
    }
 
    return Ret;
}
 
 
///////////////////////////////////////////////////////////////////////////
// MatrixMath implementation
 
Matrix1D MatrixMath::Log(const Matrix1D &In)
{
    Matrix1D Ret(In.Column);
 
    for (int i = 0 ; i < In.Column; i++)
    {
        Ret.Value[i] = log(In.Value[i]);
    }
 
    return Ret;
}
 
Matrix2D MatrixMath::Log(const Matrix2D &In)
{
    Matrix2D Ret(In.Row, In.Column);
 
    for (int r = 0; r < In.Row; r++)
    {
        for (int c = 0; c < In.Column; c++)
        {
            Ret.Value[r][c] = log(In.Value[r][c]);
        }
    }
 
    return Ret;
}