306 lines
12 KiB
TeX
306 lines
12 KiB
TeX
\lecture{13}{2023-05}{}
|
|
%The difficult part is to show \autoref{levycontinuity}.
|
|
%This is the last lecture, where we will deal with independent random variables.
|
|
|
|
We have seen, that
|
|
if $X_1, X_2,\ldots$ are i.i.d.~with $ \mu = \bE[X_1]$,
|
|
$\sigma^2 = \Var(X_1)$,
|
|
then $\frac{\sum_{i=1}^{n} (X_i - \mu)}{\sigma \sqrt{n} } \xrightarrow{(d)} \cN(0,1)$.
|
|
|
|
\begin{question}
|
|
What happens if $X_1, X_2,\ldots$ are independent, but not identically distributed? Do we still have a CLT?
|
|
\end{question}
|
|
|
|
\begin{theorem}[Lindeberg CLT]
|
|
\label{lindebergclt}
|
|
Assume $X_1, X_2, \ldots,$ are independent (but not necessarily identically distributed) with $\mu_i = \bE[X_i] < \infty$ and $\sigma_i^2 = \Var(X_i) < \infty$.
|
|
Let $S_n = \sqrt{\sum_{i=1}^{n} \sigma_i^2}$
|
|
and assume that
|
|
\[\lim_{n \to \infty} \frac{1}{S_n^2} \sum_{i=1}^{n} \bE\left[
|
|
(X_i - \mu_i)^2 \One_{|X_i - \mu_i| > \epsilon S_n}
|
|
\right] = 0\]
|
|
for all $\epsilon > 0$
|
|
(\vocab{Lindeberg condition}\footnote{``The truncated variance is negligible compared to the variance.''}).
|
|
|
|
Then the CLT holds, i.e.~
|
|
\[
|
|
\frac{\sum_{i=1}^n (X_i - \mu_i)}{S_n} \xrightarrow{(d)} \cN(0,1).
|
|
\]
|
|
\end{theorem}
|
|
|
|
\begin{theorem}[Lyapunov condition]
|
|
\label{lyapunovclt}
|
|
Let $X_1, X_2,\ldots$ be independent, $\mu_i = \bE[X_i] < \infty$,
|
|
$\sigma_i^2 = \Var(X_i) < \infty$
|
|
and $S_n \coloneqq \sqrt{\sum_{i=1}^n \sigma_i^2}$.
|
|
Then, assume that, for some $\delta > 0$,
|
|
\[
|
|
\lim_{n \to \infty} \frac{1}{S_n^{2+\delta}}
|
|
\sum_{i=1}^{n} \bE[(X_i - \mu_i)^{2 + \delta}] = 0
|
|
\]
|
|
(\vocab{Lyapunov condition}).
|
|
Then the CLT holds.
|
|
\end{theorem}
|
|
\begin{remark}
|
|
The Lyapunov condition implies the Lindeberg condition.
|
|
(Exercise).
|
|
\end{remark}
|
|
We will not prove the \autoref{lindebergclt} or \autoref{lyapunovclt}
|
|
in this lecture. However, they are quite important.
|
|
|
|
We will now sketch the proof of \autoref{levycontinuity},
|
|
details can be found in the notes.\notes
|
|
\begin{definition}
|
|
Let $(X_n)_n$ be a sequence of random variables.
|
|
The distribution of $(X_n)_n$ is called
|
|
\vocab[Distribution!tight]{tight} (dt. ``straff''),
|
|
if
|
|
\[
|
|
\lim_{a \to \infty} \sup_{n \in \N} \bP[|X_n| > a] = 0.
|
|
\]
|
|
\end{definition}
|
|
\begin{example}+[Exercise 8.1]
|
|
\todo{Copy}
|
|
\end{example}
|
|
A generalized version of \autoref{levycontinuity} is the following:
|
|
\begin{theorem}[A generalized version of Levy's continuity \autoref{levycontinuity}]
|
|
\label{genlevycontinuity}
|
|
Suppose we have random variables $(X_n)_n$ such that
|
|
$\bE[e^{\i t X_n}] \xrightarrow{n \to \infty} \phi(t)$ for all $t \in \R$
|
|
for some function $\phi$ on $\R$.
|
|
Then the following are equivalent:
|
|
\begin{enumerate}[(a)]
|
|
\item The distribution of $X_n$ is tight.
|
|
\item $X_n \xrightarrow{(d)} X$ for some real-valued random variable $X$.
|
|
\item $\phi$ is the characteristic function of $X$.
|
|
\item $\phi$ is continuous on all of $\R$.
|
|
\item $\phi$ is continuous at $0$.
|
|
\end{enumerate}
|
|
\end{theorem}
|
|
\todo{Proof of \autoref{genlevycontinuity} (Exercise 8.2)}
|
|
\begin{example}
|
|
Let $Z \sim \cN(0,1)$ and $X_n \coloneqq n Z$.
|
|
We have $\phi_{X_n}(t) = \bE[[e^{\i t X_n}] = e^{-\frac{1}{2} t^2 n^2} \xrightarrow{n \to \infty} \One_{\{t = 0\} }$.
|
|
$\One_{\{t = 0\}}$ is not continuous at $0$.
|
|
By \autoref{genlevycontinuity}, $X_n$ can not converge to a real-valued
|
|
random variable.
|
|
|
|
Exercise: $X_n \xrightarrow{(d)} \overline{X}$,
|
|
where $\bP[\overline{X} = \infty] = \frac{1}{2} = \bP[\overline{X} = -\infty]$.
|
|
|
|
Similar examples are $\mu_n \coloneqq \delta_n$ and
|
|
$\mu_n \coloneqq \frac{1}{2} \delta_n + \frac{1}{2} \delta_{-n}$.
|
|
\end{example}
|
|
|
|
\begin{example}
|
|
Suppose that $X_1, X_2,\ldots$ are i.d.d.~with $\bE[X_1] = 0$.
|
|
Let $\sigma^2 \coloneqq \Var(X_i)$.
|
|
Then the distribution of $\frac{S_n}{\sigma \sqrt{n}}$ is tight:
|
|
|
|
\begin{IEEEeqnarray*}{rCl}
|
|
\bE\left[ \left( \frac{S_n}{\sqrt{n} }^2 \right)^2 \right] &=&
|
|
\frac{1}{n} \bE[ (X_1+ \ldots + X_n)^2]\\
|
|
&=& \sigma^2
|
|
\end{IEEEeqnarray*}
|
|
For $a > 0$, by Chebyshev's inequality, % TODO
|
|
we have
|
|
\[
|
|
\bP\left[ \left| \frac{S_n}{\sqrt{n}} \right| > a \right] \leq \frac{\sigma^2}{a^2} \xrightarrow{a \to \infty} 0.
|
|
\]
|
|
verifying \autoref{genlevycontinuity}.
|
|
\end{example}
|
|
|
|
\begin{example}
|
|
Suppose $C$ is a random variable which is \vocab[Cauchy distribution]{Cauchy distributed}, i.e.~$C$
|
|
has probability distribution $f_C(x) = \frac{1}{\pi} \frac{1}{1 + x^2}$.
|
|
|
|
\begin{figure}[H]
|
|
\centering
|
|
\begin{tikzpicture}
|
|
\begin{axis}[samples=100, smooth]
|
|
\addplot[] { (1/3.14159265358979323846) * (1 / ( 1 + x * x))};
|
|
\end{axis}
|
|
\end{tikzpicture}
|
|
\caption{Probability density function of $C$}
|
|
\end{figure}
|
|
|
|
We know that $\bE[|C|] = \infty$.
|
|
|
|
We have $\phi_C(t) = \bE[e^{\i t C}] = e^{-|t|}$.
|
|
Suppose $C_1, C_2, \ldots, C_n$ are i.i.d.~Cauchy distributed
|
|
and let $S_n \coloneqq C_1 + \ldots + C_n$.
|
|
|
|
Exercise: $\phi_{\frac{S_n}{n}}(t) = e^{-|t|} = \phi_{C_1}(t)$, thus $\frac{S_n}{n} \sim C$.
|
|
\end{example}
|
|
|
|
We will prove \autoref{levycontinuity} assuming
|
|
\autoref{lec10_thm1}.
|
|
\autoref{lec10_thm1} will be shown in the notes.\notes
|
|
We will need the following:
|
|
\begin{lemma}
|
|
\label{lec13_lem1}
|
|
Given a sequence $(F_n)_n$ of probability distribution functions,
|
|
there is a subsequence $(F_{n_k})_k$ of $F_n$
|
|
and a right continuous, non-decreasing function $F$,
|
|
such that $F_{n_k} \to F$ at all continuity points of $F$.
|
|
(We do not yet claim, that $F$ is a probability distribution function,
|
|
as we ignore $\lim_{x \to \infty} F(x)$ and $\lim_{x \to -\infty} F(x)$ for now).
|
|
\end{lemma}
|
|
\begin{lemma}
|
|
\label{s7e1}
|
|
Let $\mu \in M_1(\R)$, $A > 0$ and $\phi$ the characteristic function of $\mu$.
|
|
Then $\mu\left( (-A,A) \right) \ge \frac{A}{2} \left| \int_{-\frac{2}{A}}^{\frac{2}{A}} \phi(t) d t \right| - 1$.
|
|
\end{lemma}
|
|
\begin{refproof}{s7e1}
|
|
We have
|
|
\begin{IEEEeqnarray*}{rCl}
|
|
\int_{-\frac{2}{A}}^{\frac{2}{A}} \phi(t) \dif t
|
|
&=& \int_{-\frac{2}{A}}^{\frac{2}{A}} \int_{\R} e^{\i t x} \mu(\dif x) \dif t\\
|
|
&=& \int_{\R} \int_{-\frac{2}{A}}^{\frac{2}{A}} e^{\i t x} \dif t \mu(\dif x)\\
|
|
&=& \int_{\R} \int_{-\frac{2}{A}}^{\frac{2}{A}} \cos(t x) \dif t \mu(\dif x)\\
|
|
&=& \int_{\R} \frac{2 \sin\left( \frac{2x}{A}\right) }{x} \mu(\dif x).\\
|
|
\end{IEEEeqnarray*}
|
|
Hence
|
|
\begin{IEEEeqnarray*}{rCl}
|
|
\frac{A}{2}\left|\int_{-\frac{2}{A}}^{\frac{2}{A}} \phi(t) \dif t\right|
|
|
&=& \left| A \int_{\R} \frac{\sin\left( \frac{2x}{A} \right) }{x} \mu(\dif t)\right|\\
|
|
&=& 2\left| \int_{\R} \sinc\left( \frac{2x}{A} \right) \mu(\dif t)\right|\\
|
|
&\le& 2 \left[ \int_{|x| < A} \underbrace{\left|\sinc\left( \frac{2x}{A} \right) \right|}_{\le 1} \mu(\dif x)
|
|
+ \int_{|x| \ge A} \left|\sinc\left( \frac{2x}{A} \right)\right| \mu(\dif x) \right]\\
|
|
&\le& 2 \left[ \mu\left( (-A,A) \right)
|
|
+ \frac{A}{2} \int_{|x| \ge A} \frac{\sin(2x / A)|}{|x|} \mu(\dif x) \right]\\
|
|
&\le& 2 \left[ \mu\left( (-A,A) \right)
|
|
+ \frac{A}{2} \int_{|x| \ge A} \frac{1}{A} \mu(\dif x) \right]\\
|
|
&\le& 2 \mu((-A,A)) + \mu((-A,A)^c)\\
|
|
&=& 1 + \mu((-A,A)).
|
|
\end{IEEEeqnarray*}
|
|
\end{refproof}
|
|
|
|
|
|
\begin{refproof}{levycontinuity}
|
|
``$\implies$ '' If $\mu_n \implies \mu$,
|
|
then by definition
|
|
$\int f \dif \mu_n \to \int f \dif \mu$
|
|
for all $f \in C_b$.
|
|
Since $x \to e^{\i t x}$ is continuous and bounded,
|
|
it follows that $\phi_n(t) \to \phi(t)$
|
|
for all $t \in \R$.
|
|
|
|
|
|
``$ \impliedby$''
|
|
|
|
% Step 1:
|
|
\begin{claim}
|
|
\label{levyproofc1}
|
|
Given $\epsilon > 0$ there exists $A > 0$ such that
|
|
$\liminf_n \mu_n\left( (-A,A) \right) \ge 1 - 2 \epsilon$.
|
|
\end{claim}
|
|
\begin{refproof}{levyproofc1}
|
|
If $f$ is continuous, then
|
|
\[
|
|
\frac{1}{\eta} \int_{x - \eta}^{x + \eta} f(t) d t \xrightarrow{\eta \downarrow 0} f(x).
|
|
\]
|
|
Applying this to $\phi$ at $t = 0$, one obtains:
|
|
\begin{equation}
|
|
\left| \frac{A}{4} \int_{-\frac{2}{A}}^{\frac{2}{A}} \phi(t) dt - 1 \right| < \frac{\epsilon}{2}
|
|
\label{levyproofc1eqn1}
|
|
\end{equation}
|
|
|
|
\begin{claim}
|
|
For $n$ large enough, we have
|
|
\begin{equation}
|
|
\left| \frac{A}{4} \int_{-\frac{2}{A}}^{\frac{2}{A}} \phi_n(t) d t - 1\right| < \epsilon.
|
|
\label{levyproofc1eqn2}
|
|
\end{equation}
|
|
\end{claim}
|
|
\begin{subproof}
|
|
Apply dominated convergence.
|
|
\end{subproof}
|
|
So to prove $\mu_n\left( (-A,A) \right) \ge 1 - 2 \epsilon$,
|
|
apply \autoref{s7e1}.
|
|
It suffices to show that
|
|
\[
|
|
\frac{A}{2} \left| \int_{-\frac{2}{A}}^{\frac{2}{A}} \phi_n(t) dt\right| - 1 \ge 1 - 2\epsilon
|
|
\]
|
|
or
|
|
\[
|
|
1 - \frac{A}{4} \left|\int_{-\frac{2}{A}}^{\frac{2}{A}} \phi_n(t) dt \right| \le \epsilon,
|
|
\]
|
|
which follows from \autoref{levyproofc1eqn2}.
|
|
\end{refproof}
|
|
|
|
% Step 2
|
|
By \autoref{lec13_lem1}
|
|
there exists a right continuous, non-decreasing $F $
|
|
and a subsequence $(F_{n_k})_k$ of $(F_n)_n$ where $F_n$ is
|
|
the probability distribution function of $\mu_n$,
|
|
such that $F_{n_k}(x) \to F(x)$ for all $x$ where $F$ is continuous.
|
|
\begin{claim}
|
|
\[
|
|
\lim_{n \to -\infty} F(x) = 0
|
|
\]
|
|
and
|
|
\[
|
|
\lim_{n \to \infty} F(x) = 1,
|
|
\]
|
|
i.e.~$F$ is a probability distribution function.\footnote{This does not hold in general!}
|
|
\end{claim}
|
|
\begin{subproof}
|
|
We have
|
|
\[
|
|
\mu_{n_k}\left( (- \infty, x] \right) = F_{n_k}(x) \to F(x).
|
|
\]
|
|
Again, given $\epsilon > 0$, there exists $A > 0$, such that
|
|
$\mu_{n_k}\left( (-A,A) \right) > 1 - 2 \epsilon$ (\autoref{levyproofc1}).
|
|
|
|
Hence $F(x) \ge 1 - 2 \epsilon$ for $x > A $
|
|
and $F(x) \le 2\epsilon$ for $x < -A$.
|
|
This proves the claim.
|
|
\end{subproof}
|
|
|
|
Since $F$ is a probability distribution function, there exists
|
|
a probability measure $\nu$ on $\R$ such that $F$ is the distribution
|
|
function of $\nu$.
|
|
Since $F_{n_k}(x) \to F_n(x)$ at all continuity points $x$ of $F$,
|
|
by \autoref{lec10_thm1} we obtain that
|
|
$\mu_{n_k} \overset{k \to \infty}{\implies} \nu$.
|
|
Hence
|
|
$\phi_{\mu_{n_k}}(t) \to \phi_\nu(t)$, by the other direction of that theorem.
|
|
But by assumption,
|
|
$\phi_{\mu_{n_k}}(\cdot ) \to \phi_n(\cdot )$ so $\phi_{\mu}(\cdot) = \phi_{\nu}(\cdot )$.
|
|
By \autoref{charfuncuniqueness}, we get $\mu = \nu$.
|
|
|
|
We have shown, that $\mu_{n_k} \implies \mu$ along a subsequence.
|
|
We still need to show that $\mu_n \implies \mu$.
|
|
\begin{fact}
|
|
Suppose $a_n$ is a bounded sequence in $\R$,
|
|
such that any subsequence has a subsequence
|
|
that converges to $a \in \R$.
|
|
Then $a_n \to a$.
|
|
\end{fact}
|
|
\begin{subproof}
|
|
\notes
|
|
\end{subproof}
|
|
Assume that $\mu_n$ does not converge to $\mu$.
|
|
By \autoref{lec10_thm1}, pick a continuity point $x_0$ of $F$,
|
|
such that $F_n(x_0) \not\to F(x_0)$.
|
|
Pick $\delta > 0$ and a subsequence $F_{n_1}(x_0), F_{n_2}(x_0), \ldots$
|
|
which are all outside $(F(x_0) - \delta, F(x_0) + \delta)$.
|
|
Then $\phi_{n_1}, \phi_{n_2}, \ldots \to \phi$.
|
|
Now, there exists a further subsequence $G_1, G_2, \ldots$ of $F_{n_i}$,
|
|
which converges.
|
|
$G_1, G_2, \ldots$ is a subsequence of $F_1, F_2,\ldots$.
|
|
However $G_1, G_2, \ldots$ is not converging to $F$,
|
|
as this would fail at $x_0$. This is a contradiction.
|
|
\end{refproof}
|
|
|
|
% IID is over now
|
|
\subsection{Summary}
|
|
What did we learn:
|
|
|
|
\begin{itemize}
|
|
\item How to construct product measures
|
|
\item WLLN and SLLN
|
|
\item Kolmogorov's three series theorem
|
|
\item Fourier transform, weak convergence and CLT
|
|
\end{itemize}
|