lecture 22

This commit is contained in:
Josia Pietsch 2023-07-06 00:35:55 +02:00
parent 4cd05f37a0
commit 023e865eb4
Signed by untrusted user who does not match committer: josia
GPG key ID: E70B571D66986A2D

View file

@ -5,66 +5,65 @@
We want to start with the basics of the theory of Markov chains. We want to start with the basics of the theory of Markov chains.
\end{goal} \end{goal}
% \begin{example}[Markov chains with two states] \begin{example}[Markov chains with two states]
% Suppose there are two states of a phone line, Suppose there are two states of a phone line,
% $0$,``free'', or $1$, ``busy''. $0$,``free'', or $1$, ``busy''.
% We assume that the state only changes at discrete units of time We assume that the state only changes at discrete units of time
% and model this as a sequence of random variables. and model this as a sequence of random variables.
% Assume Assume
% \begin{IEEEeqnarra*}{rCl} \begin{IEEEeqnarray*}{rCl}
% \bP[X_{n+1} = 0 | X_n = 0] &=& p\\ \bP[X_{n+1} = 0 | X_n = 0] &=& p\\
% \bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\ \bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\
% \bP[X_{n+1} = 1 | X_n = 0] &=& q\\ \bP[X_{n+1} = 1 | X_n = 0] &=& q\\
% \bP[X_{n+1} = 1 | X_n = 1] &=& (1-q) \bP[X_{n+1} = 1 | X_n = 1] &=& (1-q)
% \end{IEEEeqnarra*} \end{IEEEeqnarray*}
% for some $p,q \in (0,1)$. for some $p,q \in (0,1)$.
% We can write this as a matrix We can write this as a matrix
% \begin{IEEEeqnarra*}{rCl} \begin{IEEEeqnarray*}{rCl}
% P &=& \begin{pmatrix} P &=& \begin{pmatrix}
% p & (1-p) \\ p & (1-p) \\
% q & (1-q) q & (1-q)
% \end{pmatrix} \end{pmatrix}
% \end{IEEEeqnarra*} \end{IEEEeqnarray*}
% Note that the rows of this matrix sum up to $1$. Note that the rows of this matrix sum up to $1$.
%
% Additionally, we make the following assmption: Additionally, we make the following assmption:
% Given that at some time $n$, the phone is in state $i \in \{0,1\}$, Given that at some time $n$, the phone is in state $i \in \{0,1\}$,
% the behavior of the phone after time $n$ does not depend the behavior of the phone after time $n$ does not depend
% on the way, the phone reached state $i$. on the way, the phone reached state $i$.
%
% \begin{question} \begin{question}
% Suppose $X_0 = 0$. Suppose $X_0 = 0$.
% What is the probability, that the phone will be free at times What is the probability, that the phone will be free at times
% $1 \& 2$ and will become busy at time $3$, $1 \& 2$ and will become busy at time $3$,
% i.e.~what is $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$? i.e.~what is $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$?
% \end{question} \end{question}
% We have We have
% \begin{IEEEeqnarra*}{rCl} \begin{IEEEeqnarray*}{rCl}
% \bP[X_1 = 0, X_2 = 0, X_3 = 1] \bP[X_1 = 0, X_2 = 0, X_3 = 1]
% &=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\ &=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\
% &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\ &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\
% &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 | X_1 = 0] \bP[X_1 = 0]\\ &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 | X_1 = 0] \bP[X_1 = 0]\\
% &=& P_{0,1} P_{0,0} P_{0,0} &=& P_{0,1} P_{0,0} P_{0,0}
% \end{IEEEeqnarra*} \end{IEEEeqnarray*}
%
% \begin{question} \begin{question}
% Assume $X_0 = 0$. What is $\bP[X_3 = 1]$? Assume $X_0 = 0$. What is $\bP[X_3 = 1]$?
% \end{question} \end{question}
% For $\{X_3 = 1\}$ to happen, we need to look at the following For $\{X_3 = 1\}$ to happen, we need to look at the following
% disjoint events: disjoint events:
% % \begin{IEEEeqnarra*}{rCl} \begin{IEEEeqnarray*}{rCl}
% % \bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\ \bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\
% % \bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\ \bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\
% % \bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\ \bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\
% % \bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2. \bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2.
% % \end{IEEEeqnarr*} \end{IEEEeqnarray*}
%
% More generally, consider a Matrix $P \in (0,1)^{n \times n}$ More generally, consider a Matrix $P \in (0,1)^{n \times n}$
% whose rows sum up to $1$. whose rows sum up to $1$.
% Then we get a Markov Chain with $n$ states Then we get a Markov Chain with $n$ states
% by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$. by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$.
% \end{example}
% \end{example}
\begin{definition} \begin{definition}
Let $E$ denote a \vocab{discrete state space}, Let $E$ denote a \vocab{discrete state space},
@ -117,15 +116,15 @@
Consider Consider
\begin{IEEEeqnarray*}{rCl} \begin{IEEEeqnarray*}{rCl}
P &=& P &=&
\begin{pmatrix} \begin{pmatrix}
& \ddots & \ddots & \ddots & & & & & 0\\ & \ddots & \ddots & \ddots & & & & & 0\\
\ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\ \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\
& \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\ & \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\
& & \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\ & & \ldots & 0 & \frac{1}{2} & 0 & \frac{1}{2} & 0 & \ldots \\
0 & & & & & \ddots & \ddots & \ddots & \\ 0 & & & & & \ddots & \ddots & \ddots & \\
\end{pmatrix} \end{pmatrix}
\end{IEEEeqnarray*} \end{IEEEeqnarray*}
\end{example} \end{example}
% \begin{example} % \begin{example}
@ -144,9 +143,9 @@
For every $x \in E$, For every $x \in E$,
let $\mathbf{P}(x, \cdot )$ be a probability measure on $E$.% let $\mathbf{P}(x, \cdot )$ be a probability measure on $E$.%
\footnote{$\mathbf{P}(x,\cdot )$ corresponds to a row of our matrix in the discrete case} \footnote{$\mathbf{P}(x,\cdot )$ corresponds to a row of our matrix in the discrete case}
Given the triples $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$, Given the triples $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$,
we say that a stochastic process $(X_n)_{n \ge 0}$ we say that a stochastic process $(X_n)_{n \ge 0}$
is a \vocab[Markov chain]{Markov chain taking values on $E$ % is a \vocab[Markov chain]{Markov chain taking values on $E$ %
with starting distribution $\alpha$ % with starting distribution $\alpha$ %
and transition probability $\{\mathbf{P}(x, \cdot )\}_{x \in E}$} and transition probability $\{\mathbf{P}(x, \cdot )\}_{x \in E}$}
@ -157,9 +156,9 @@
\[ \[
\bE[f(X_{n+1}) | \cF_n] = \bE[f(X_{n+1}) | X_n] \bE[f(X_{n+1}) | \cF_n] = \bE[f(X_{n+1}) | X_n]
= \int_E f(y) \mathbf{P}(X_n, \dif y) \text{ a.s.} = \int_E f(y) \mathbf{P}(X_n, \dif y) \text{ a.s.}
\] \]
\end{enumerate} \end{enumerate}
\end{definition} \end{definition}
\begin{remark} \begin{remark}
This agrees with the definition in the discrete case, This agrees with the definition in the discrete case,
@ -170,49 +169,44 @@
\begin{notation} \begin{notation}
If $\{\mathbf{P}(x, \cdot )\}_{x \in E}$ is a transition probability, If $\{\mathbf{P}(x, \cdot )\}_{x \in E}$ is a transition probability,
then for all $f: E \to \R$ bounded and measurable, then for all $f: E \to \R$ bounded and measurable,
define $\mathbf{P} : \cB_{\text{bdd}}(E) \to \cB_{\text{bdd}}$ define $\mathbf{P} : \cB_{\text{bdd}}(E) \to \cB_{\text{bdd}}$
by by
\[ \[
(\mathbf{P} f)(x) \coloneqq \int_E f(y) \mathbf{P}(x, \dif y). (\mathbf{P} f)(x) \coloneqq \int_E f(y) \mathbf{P}(x, \dif y).
\] \]
\end{notation} \end{notation}
We get the following fundamental link between martingales and Markov chains: We get the following fundamental link between martingales and Markov chains:
\begin{theorem} \begin{theorem}
\label{martingalesandmarkovchains} \label{martingalesandmarkovchains}
Suppose $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$ Suppose $(E, \alpha, \{\mathbf{P}(x, \cdot )\}_{x \in E})$
is given. is given.
Then a stochastic process $(X_n)_{n \ge 0}$ is a Markov chain Then a stochastic process $(X_n)_{n \ge 0}$ is a Markov chain
iff for every $f: E \to \R$ bounded, measurable, iff for every $f: E \to \R$ bounded, measurable,
\[ \[
M_n(f) \coloneqq f(X_n) - f(X_0) - \sum_{j=1}^{n-1} (\mathbf{I} - \mathbf{P})f(X_j) M_n(f) \coloneqq f(X_n) - f(X_0) - \sum_{j=1}^{n-1} (\mathbf{I} - \mathbf{P})f(X_j)
\] \]
is a martingale is a martingale
with respect to the canonical filtration of $(X_n)$. with respect to the canonical filtration of $(X_n)$.
\end{theorem} \end{theorem}
\begin{proof} \begin{proof}
$\implies$ $\implies$
Fix some bounded, measurable $f : E \to \R$. Fix some bounded, measurable $f : E \to \R$.
Then, for all $n$, $M_n(f)$ is bounded Then, for all $n$, $M_n(f)$ is bounded
and hence $M_n(f) \in L^1$. and hence $M_n(f) \in L^1$.
$M_n(f)$ is $\cF_n$-measurable for all $n \in \N$. $M_n(f)$ is $\cF_n$-measurable for all $n \in \N$.
\begin{claim}
$\bE[M_{n+1}(f) | \cF_n] = M_n(f)$.
\end{claim}
\begin{subproof}
It suffices to show
$\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s.
We have In order to prove $\bE[M_{n+1}(f) | \cF_n] = M_n(f)$,
\begin{IEEEeqnarray*}{rCl} it suffices to show $\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s.
\bE[M_{n+1}(f) - M_n(f) | \cF_n]
&=& \bE[f(X_{n+1} | \cF_n] - (\mathbf{P}f)(X_n)\\
&\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\
&=& 0
\end{IEEEeqnarray*}
\end{subproof}
$\impliedby$ We have
\begin{IEEEeqnarray*}{rCl}
\bE[M_{n+1}(f) - M_n(f) | \cF_n]
&=& \bE[f(X_{n+1} | \cF_n] - (\mathbf{P}f)(X_n)\\
&\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\
&=& 0
\end{IEEEeqnarray*}
$\impliedby$
Suppose $(M_n(f))_n$ is a martingale for all bounded, measurable $f$. Suppose $(M_n(f))_n$ is a martingale for all bounded, measurable $f$.
By the martingale property, we have By the martingale property, we have
\begin{IEEEeqnarray*}{rCl} \begin{IEEEeqnarray*}{rCl}