diff --git a/inputs/lecture_22.tex b/inputs/lecture_22.tex index ef04eb7..3cf5b1b 100644 --- a/inputs/lecture_22.tex +++ b/inputs/lecture_22.tex @@ -5,66 +5,65 @@ We want to start with the basics of the theory of Markov chains. \end{goal} -% \begin{example}[Markov chains with two states] -% Suppose there are two states of a phone line, -% $0$,``free'', or $1$, ``busy''. -% We assume that the state only changes at discrete units of time -% and model this as a sequence of random variables. -% Assume -% \begin{IEEEeqnarra*}{rCl} -% \bP[X_{n+1} = 0 | X_n = 0] &=& p\\ -% \bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\ -% \bP[X_{n+1} = 1 | X_n = 0] &=& q\\ -% \bP[X_{n+1} = 1 | X_n = 1] &=& (1-q) -% \end{IEEEeqnarra*} -% for some $p,q \in (0,1)$. -% We can write this as a matrix -% \begin{IEEEeqnarra*}{rCl} -% P &=& \begin{pmatrix} -% p & (1-p) \\ -% q & (1-q) -% \end{pmatrix} -% \end{IEEEeqnarra*} -% Note that the rows of this matrix sum up to $1$. -% -% Additionally, we make the following assmption: -% Given that at some time $n$, the phone is in state $i \in \{0,1\}$, -% the behavior of the phone after time $n$ does not depend -% on the way, the phone reached state $i$. -% -% \begin{question} -% Suppose $X_0 = 0$. -% What is the probability, that the phone will be free at times -% $1 \& 2$ and will become busy at time $3$, -% i.e.~what is $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$? -% \end{question} -% We have -% \begin{IEEEeqnarra*}{rCl} -% \bP[X_1 = 0, X_2 = 0, X_3 = 1] -% &=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\ -% &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\ -% &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 | X_1 = 0] \bP[X_1 = 0]\\ -% &=& P_{0,1} P_{0,0} P_{0,0} -% \end{IEEEeqnarra*} -% -% \begin{question} -% Assume $X_0 = 0$. What is $\bP[X_3 = 1]$? -% \end{question} -% For $\{X_3 = 1\}$ to happen, we need to look at the following -% disjoint events: -% % \begin{IEEEeqnarra*}{rCl} -% % \bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\ -% % \bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\ -% % \bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\ -% % \bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2. -% % \end{IEEEeqnarr*} -% -% More generally, consider a Matrix $P \in (0,1)^{n \times n}$ -% whose rows sum up to $1$. -% Then we get a Markov Chain with $n$ states -% by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$. -% -% \end{example} + \begin{example}[Markov chains with two states] + Suppose there are two states of a phone line, + $0$,``free'', or $1$, ``busy''. + We assume that the state only changes at discrete units of time + and model this as a sequence of random variables. + Assume + \begin{IEEEeqnarray*}{rCl} + \bP[X_{n+1} = 0 | X_n = 0] &=& p\\ + \bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\ + \bP[X_{n+1} = 1 | X_n = 0] &=& q\\ + \bP[X_{n+1} = 1 | X_n = 1] &=& (1-q) + \end{IEEEeqnarray*} + for some $p,q \in (0,1)$. + We can write this as a matrix + \begin{IEEEeqnarray*}{rCl} + P &=& \begin{pmatrix} + p & (1-p) \\ + q & (1-q) + \end{pmatrix} + \end{IEEEeqnarray*} + Note that the rows of this matrix sum up to $1$. + + Additionally, we make the following assmption: + Given that at some time $n$, the phone is in state $i \in \{0,1\}$, + the behavior of the phone after time $n$ does not depend + on the way, the phone reached state $i$. + + \begin{question} + Suppose $X_0 = 0$. + What is the probability, that the phone will be free at times + $1 \& 2$ and will become busy at time $3$, + i.e.~what is $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$? + \end{question} + We have + \begin{IEEEeqnarray*}{rCl} + \bP[X_1 = 0, X_2 = 0, X_3 = 1] + &=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\ + &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\ + &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 | X_1 = 0] \bP[X_1 = 0]\\ + &=& P_{0,1} P_{0,0} P_{0,0} + \end{IEEEeqnarray*} + + \begin{question} + Assume $X_0 = 0$. What is $\bP[X_3 = 1]$? + \end{question} + For $\{X_3 = 1\}$ to happen, we need to look at the following + disjoint events: + \begin{IEEEeqnarray*}{rCl} + \bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\ + \bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\ + \bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\ + \bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2. + \end{IEEEeqnarray*} + + More generally, consider a Matrix $P \in (0,1)^{n \times n}$ + whose rows sum up to $1$. + Then we get a Markov Chain with $n$ states + by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$. + \end{example} \begin{definition} Let $E$ denote a \vocab{discrete state space}, @@ -117,15 +116,15 @@ Consider \begin{IEEEeqnarray*}{rCl} P &=& - \begin{pmatrix} + \begin{pmatrix} & \ddots & \ddots & \ddots & & & & & 0\\ \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\ & \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\ & & \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\ 0 & & & & & \ddots & \ddots & \ddots & \\ - \end{pmatrix} + \end{pmatrix} \end{IEEEeqnarray*} - + \end{example} % \begin{example} @@ -144,9 +143,9 @@ For every $x \in E$, let $\mathbf{P}(x, \cdot )$ be a probability measure on $E$.% \footnote{$\mathbf{P}(x,\cdot )$ corresponds to a row of our matrix in the discrete case} - + Given the triples $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$, - we say that a stochastic process $(X_n)_{n \ge 0}$ + we say that a stochastic process $(X_n)_{n \ge 0}$ is a \vocab[Markov chain]{Markov chain taking values on $E$ % with starting distribution $\alpha$ % and transition probability $\{\mathbf{P}(x, \cdot )\}_{x \in E}$} @@ -157,9 +156,9 @@ \[ \bE[f(X_{n+1}) | \cF_n] = \bE[f(X_{n+1}) | X_n] = \int_E f(y) \mathbf{P}(X_n, \dif y) \text{ a.s.} - \] + \] \end{enumerate} - + \end{definition} \begin{remark} This agrees with the definition in the discrete case, @@ -170,49 +169,44 @@ \begin{notation} If $\{\mathbf{P}(x, \cdot )\}_{x \in E}$ is a transition probability, then for all $f: E \to \R$ bounded and measurable, - define $\mathbf{P} : \cB_{\text{bdd}}(E) \to \cB_{\text{bdd}}$ + define $\mathbf{P} : \cB_{\text{bdd}}(E) \to \cB_{\text{bdd}}$ by \[ (\mathbf{P} f)(x) \coloneqq \int_E f(y) \mathbf{P}(x, \dif y). - \] + \] \end{notation} We get the following fundamental link between martingales and Markov chains: \begin{theorem} \label{martingalesandmarkovchains} - Suppose $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$ + Suppose $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$ is given. Then a stochastic process $(X_n)_{n \ge 0}$ is a Markov chain iff for every $f: E \to \R$ bounded, measurable, \[ M_n(f) \coloneqq f(X_n) - f(X_0) - \sum_{j=1}^{n-1} (\mathbf{I} - \mathbf{P})f(X_j) - \] + \] is a martingale with respect to the canonical filtration of $(X_n)$. \end{theorem} \begin{proof} - $\implies$ + $\implies$ Fix some bounded, measurable $f : E \to \R$. Then, for all $n$, $M_n(f)$ is bounded and hence $M_n(f) \in L^1$. $M_n(f)$ is $\cF_n$-measurable for all $n \in \N$. - - \begin{claim} - $\bE[M_{n+1}(f) | \cF_n] = M_n(f)$. - \end{claim} - \begin{subproof} - It suffices to show - $\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s. - We have - \begin{IEEEeqnarray*}{rCl} - \bE[M_{n+1}(f) - M_n(f) | \cF_n] - &=& \bE[f(X_{n+1} | \cF_n] - (\mathbf{P}f)(X_n)\\ - &\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\ - &=& 0 - \end{IEEEeqnarray*} - \end{subproof} + In order to prove $\bE[M_{n+1}(f) | \cF_n] = M_n(f)$, + it suffices to show $\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s. - $\impliedby$ + We have + \begin{IEEEeqnarray*}{rCl} + \bE[M_{n+1}(f) - M_n(f) | \cF_n] + &=& \bE[f(X_{n+1} | \cF_n] - (\mathbf{P}f)(X_n)\\ + &\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\ + &=& 0 + \end{IEEEeqnarray*} + + $\impliedby$ Suppose $(M_n(f))_n$ is a martingale for all bounded, measurable $f$. By the martingale property, we have \begin{IEEEeqnarray*}{rCl}