lecture 22
This commit is contained in:
parent
4cd05f37a0
commit
023e865eb4
1 changed files with 82 additions and 88 deletions
|
@ -5,66 +5,65 @@
|
||||||
We want to start with the basics of the theory of Markov chains.
|
We want to start with the basics of the theory of Markov chains.
|
||||||
\end{goal}
|
\end{goal}
|
||||||
|
|
||||||
% \begin{example}[Markov chains with two states]
|
\begin{example}[Markov chains with two states]
|
||||||
% Suppose there are two states of a phone line,
|
Suppose there are two states of a phone line,
|
||||||
% $0$,``free'', or $1$, ``busy''.
|
$0$,``free'', or $1$, ``busy''.
|
||||||
% We assume that the state only changes at discrete units of time
|
We assume that the state only changes at discrete units of time
|
||||||
% and model this as a sequence of random variables.
|
and model this as a sequence of random variables.
|
||||||
% Assume
|
Assume
|
||||||
% \begin{IEEEeqnarra*}{rCl}
|
\begin{IEEEeqnarray*}{rCl}
|
||||||
% \bP[X_{n+1} = 0 | X_n = 0] &=& p\\
|
\bP[X_{n+1} = 0 | X_n = 0] &=& p\\
|
||||||
% \bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\
|
\bP[X_{n+1} = 0 | X_n = 1] &=& (1-p)\\
|
||||||
% \bP[X_{n+1} = 1 | X_n = 0] &=& q\\
|
\bP[X_{n+1} = 1 | X_n = 0] &=& q\\
|
||||||
% \bP[X_{n+1} = 1 | X_n = 1] &=& (1-q)
|
\bP[X_{n+1} = 1 | X_n = 1] &=& (1-q)
|
||||||
% \end{IEEEeqnarra*}
|
\end{IEEEeqnarray*}
|
||||||
% for some $p,q \in (0,1)$.
|
for some $p,q \in (0,1)$.
|
||||||
% We can write this as a matrix
|
We can write this as a matrix
|
||||||
% \begin{IEEEeqnarra*}{rCl}
|
\begin{IEEEeqnarray*}{rCl}
|
||||||
% P &=& \begin{pmatrix}
|
P &=& \begin{pmatrix}
|
||||||
% p & (1-p) \\
|
p & (1-p) \\
|
||||||
% q & (1-q)
|
q & (1-q)
|
||||||
% \end{pmatrix}
|
\end{pmatrix}
|
||||||
% \end{IEEEeqnarra*}
|
\end{IEEEeqnarray*}
|
||||||
% Note that the rows of this matrix sum up to $1$.
|
Note that the rows of this matrix sum up to $1$.
|
||||||
%
|
|
||||||
% Additionally, we make the following assmption:
|
Additionally, we make the following assmption:
|
||||||
% Given that at some time $n$, the phone is in state $i \in \{0,1\}$,
|
Given that at some time $n$, the phone is in state $i \in \{0,1\}$,
|
||||||
% the behavior of the phone after time $n$ does not depend
|
the behavior of the phone after time $n$ does not depend
|
||||||
% on the way, the phone reached state $i$.
|
on the way, the phone reached state $i$.
|
||||||
%
|
|
||||||
% \begin{question}
|
\begin{question}
|
||||||
% Suppose $X_0 = 0$.
|
Suppose $X_0 = 0$.
|
||||||
% What is the probability, that the phone will be free at times
|
What is the probability, that the phone will be free at times
|
||||||
% $1 \& 2$ and will become busy at time $3$,
|
$1 \& 2$ and will become busy at time $3$,
|
||||||
% i.e.~what is $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$?
|
i.e.~what is $\bP[X_1 = 0, X_2 = 0, X_3 = 1]$?
|
||||||
% \end{question}
|
\end{question}
|
||||||
% We have
|
We have
|
||||||
% \begin{IEEEeqnarra*}{rCl}
|
\begin{IEEEeqnarray*}{rCl}
|
||||||
% \bP[X_1 = 0, X_2 = 0, X_3 = 1]
|
\bP[X_1 = 0, X_2 = 0, X_3 = 1]
|
||||||
% &=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\
|
&=& \bP[X_3 = 0 | X_2 = 0, X_1 = 0] \bP[X_2 = 0, X_1 = 0]\\
|
||||||
% &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\
|
&=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0, X_1 = 0]\\
|
||||||
% &=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 | X_1 = 0] \bP[X_1 = 0]\\
|
&=& \bP[X_3 = 0 | X_2 = 0] \bP[X_2 = 0 | X_1 = 0] \bP[X_1 = 0]\\
|
||||||
% &=& P_{0,1} P_{0,0} P_{0,0}
|
&=& P_{0,1} P_{0,0} P_{0,0}
|
||||||
% \end{IEEEeqnarra*}
|
\end{IEEEeqnarray*}
|
||||||
%
|
|
||||||
% \begin{question}
|
\begin{question}
|
||||||
% Assume $X_0 = 0$. What is $\bP[X_3 = 1]$?
|
Assume $X_0 = 0$. What is $\bP[X_3 = 1]$?
|
||||||
% \end{question}
|
\end{question}
|
||||||
% For $\{X_3 = 1\}$ to happen, we need to look at the following
|
For $\{X_3 = 1\}$ to happen, we need to look at the following
|
||||||
% disjoint events:
|
disjoint events:
|
||||||
% % \begin{IEEEeqnarra*}{rCl}
|
\begin{IEEEeqnarray*}{rCl}
|
||||||
% % \bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\
|
\bP(\{X_3 = 1, X_2 = 0, X_1 = 0\}) &=& P_{0,1} P_{0,0}^2,\\
|
||||||
% % \bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\
|
\bP(\{X_3 = 1, X_2 = 0, X_1 = 1\}) &=& P_{0,1}^2 P_{1,0},\\
|
||||||
% % \bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\
|
\bP(\{X_3 = 1, X_2 = 1, X_1 = 0\}) &=& P_{0,0} P_{0,1} P_{1,1},\\
|
||||||
% % \bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2.
|
\bP(\{X_3 = 1, X_2 = 1, X_1 = 1\}) &=& P_{0,1} P_{1,1}^2.
|
||||||
% % \end{IEEEeqnarr*}
|
\end{IEEEeqnarray*}
|
||||||
%
|
|
||||||
% More generally, consider a Matrix $P \in (0,1)^{n \times n}$
|
More generally, consider a Matrix $P \in (0,1)^{n \times n}$
|
||||||
% whose rows sum up to $1$.
|
whose rows sum up to $1$.
|
||||||
% Then we get a Markov Chain with $n$ states
|
Then we get a Markov Chain with $n$ states
|
||||||
% by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$.
|
by defining $\bP[X_{n+1} = i | X_{n} = j] = P_{i,j}$.
|
||||||
%
|
\end{example}
|
||||||
% \end{example}
|
|
||||||
|
|
||||||
\begin{definition}
|
\begin{definition}
|
||||||
Let $E$ denote a \vocab{discrete state space},
|
Let $E$ denote a \vocab{discrete state space},
|
||||||
|
@ -196,12 +195,8 @@ We get the following fundamental link between martingales and Markov chains:
|
||||||
and hence $M_n(f) \in L^1$.
|
and hence $M_n(f) \in L^1$.
|
||||||
$M_n(f)$ is $\cF_n$-measurable for all $n \in \N$.
|
$M_n(f)$ is $\cF_n$-measurable for all $n \in \N$.
|
||||||
|
|
||||||
\begin{claim}
|
In order to prove $\bE[M_{n+1}(f) | \cF_n] = M_n(f)$,
|
||||||
$\bE[M_{n+1}(f) | \cF_n] = M_n(f)$.
|
it suffices to show $\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s.
|
||||||
\end{claim}
|
|
||||||
\begin{subproof}
|
|
||||||
It suffices to show
|
|
||||||
$\bE[M_{n+1}(f) - M_n(f) | \cF_n] = 0$ a.s.
|
|
||||||
|
|
||||||
We have
|
We have
|
||||||
\begin{IEEEeqnarray*}{rCl}
|
\begin{IEEEeqnarray*}{rCl}
|
||||||
|
@ -210,7 +205,6 @@ We get the following fundamental link between martingales and Markov chains:
|
||||||
&\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\
|
&\overset{\text{Markov property}}{=}& (\mathbf{P}f)(X_n) - (\mathbf{P}f)(X_n)\\
|
||||||
&=& 0
|
&=& 0
|
||||||
\end{IEEEeqnarray*}
|
\end{IEEEeqnarray*}
|
||||||
\end{subproof}
|
|
||||||
|
|
||||||
$\impliedby$
|
$\impliedby$
|
||||||
Suppose $(M_n(f))_n$ is a martingale for all bounded, measurable $f$.
|
Suppose $(M_n(f))_n$ is a martingale for all bounded, measurable $f$.
|
||||||
|
|
Loading…
Reference in a new issue