\documentclass[12pt]{article}

\usepackage{times}
\usepackage{graphicx}
\usepackage{url}
\usepackage{amsmath}

\setlength{\textwidth}{6.5in}
\setlength{\textheight}{8.9in}
\setlength{\oddsidemargin}{0.0in}
\setlength{\topmargin}{0.05in}
\setlength{\headheight}{-0.05in}
\setlength{\headsep}{0.0in}

\begin{document}

\begin{center} 
{\bf CS 6300} \hfill {\large\bf HW06: Q Learning with
    Functional Approximation} \hfill {\bf Due March 13, 2023}
\end{center}

\noindent
Please use the \LaTeX\ template to produce your writeups. See the
Homework Assignments page on the class website for details.  Hand in
through gradescope.


\section{Functional Approximation}

We revisit the simplied version of blackjack from Homework 5. The deck is infinite and the
dealer always has a fixed count of 15.  The deck contains cards 2
through 10, J, Q, K, and A, each of which is equally likely to appear
when a card is drawn.  Each number card is worth the number of points
shown on it, the cards J, Q, and K are worth 10 points, and A is worth
11.  At each turn, you have two possible actions: either {\it hit} or {\it stay}.

Unhappy with your experience with basic Q-learning, you decide to
featurize your Q-values.  Consider the two feature functions:

$$
f_1(s,a) = \left\{ \begin{array}{ll}
                  0  & a=stay \\
                  +1 & a=hit, s \geq 15 \\
                 -1  & a=hit, s < 15
                 \end{array} \right.
\quad \hbox{and} \quad
f_2(s,a) = \left\{ \begin{array}{ll}
                  0  & a=stay \\
                  +1 & a=hit, s \geq 18 \\
                 -1  & a=hit, s < 18
                 \end{array} \right.
$$

\noindent
Which of the following partial policy tables may be represented by the
featurized Q-values unambiguously (without ties)?  Derive your answers
for each policy table.

\begin{center}
\begin{tabular}{ccccc}
\begin{tabular}{|c|c|} \hline
$s$ & $\pi(s)$ \\ \hline
14  & hit \\
15  & hit \\
16  & hit \\
17  & hit \\
18  & hit \\
19  & hit \\ \hline
\end{tabular} &
\begin{tabular}{|c|c|} \hline
$s$ & $\pi(s)$ \\ \hline
14  & stay \\
15  & hit \\
16  & hit \\
17  & hit \\
18  & stay \\
19  & stay \\ \hline
\end{tabular} &
\begin{tabular}{|c|c|} \hline
$s$ & $\pi(s)$ \\ \hline
14  & hit \\
15  & hit \\
16  & hit \\
17  & hit \\
18  & stay \\
19  & stay \\ \hline
\end{tabular} &
\begin{tabular}{|c|c|} \hline
$s$ & $\pi(s)$ \\ \hline
14  & hit \\
15  & hit \\
16  & hit \\
17  & hit \\
18  & hit \\
19  & stay \\ \hline
\end{tabular} &
\begin{tabular}{|c|c|} \hline
$s$ & $\pi(s)$ \\ \hline
14  & hit \\
15  & hit \\
16  & hit \\
17  & stay \\
18  & hit \\
19  & stay \\ \hline
\end{tabular} \\
(a) & (b) & (c) & (d) & (e)
\end{tabular}
\end{center}

\end{document}


