
% BB Sep 11 2008.

\documentclass[20pt,landscape,footrule]{foils}
\usepackage[dvips]{graphicx}
\usepackage[pdftex]{geometry}
\geometry{headsep=3ex,hscale=0.9}
\usepackage[pdftex]{color} % for the colored block
\date{\color{blue} Singapore MCMC Meeting, March 2004}
\usepackage[ignore]{pause}
%\usepackage{pause}
\usepackage{background}
%\usepackage{pp4slide}
%\MyLogo{AV HEP,CSIT}
\LogoOff
\begin{document}
\def\r{\color{red}}
\def\b{\color{blue}}
\def\s{\color{black}}

\foilhead{\r Bayesian Statistics } \b

\centerline{\bf\r Kolmogorov Axioms and Conditional Probabilities}

We denote events by $\s A,\, B,\, C,\, \dots\ $, and use the following
notation:

\begin{enumerate}
\item $\s A\cap B = A\ {\rm and}\ B$, the event that $\s A$ and $\s B$ 
both occur.
\item $\s A^c = {\rm not}\ A$, the event that $\s A$ does not occur.
\item $\s E$, the event which always occurs.
\item $\s A\cup B = A\ {\rm or}\ B$, the event that $\s A$ or $\s B$, 
including both occur. Equivalently, at least one of the event $\s A$ 
or $\s B$ occurs.
\end{enumerate}

The probability of an event $\s A$ is denoted by $\s P(A)$. 
% One can interpret the operations $\s A\ {\rm and}\ B$, $\s A^c$, 
% and $\s A\ {\rm or}\ B$ mathematically as either a Boolean algebra 
% of events or as a field of sets. The two interpretations are 
% equivalent, since every Boolean algebra is isomorphic to a suitable 
% field of sets. 
Following Kolmogorov one can {\r interpret all events as sets of 
elementary events.} In this interpretation, the events are subsets 
of a set $\s E$, the set of all elementary events which are considered 
possible in a given situation, called {\r sample space}, $\s A\cap B$ 
is the {\r intersection}, $\s A^c$ the {\r complement} of $\s A$, and 
$\s A\cup B$ the {\r union}. Events $\s A$ and $\s B$ are said {\r 
mutually exclusive} if $\s A\cap B= \not{\!0}$, where $\s \not{\!0}$ 
is the empty set.

{\r Kolmogorov's axioms of probability theory} are:

\begin{enumerate}
\item If $\s A$ and $\s B$ are events, then $\s A^c$, $\s A\cap B$, and 
      $\s A\cup B$ are also events.
\item To each event there corresponds a real number $\s P(A)\ge 0$. 
      $\s E$ is an event with $\s P(E)=1.$
\item If $\s A$ and $\s B$ are mutually exclusive, then $\s P(A\cup B)
      = P(A)+P(B).$
\item If $\s A_1,\, A_2,\, \dots$ are events, which never occur all
      together, then
$$\s \lim_{n\to\infty} P(A_1\cap A_2\cap\,\dots\,\cap A_n)=0\ .$$
\end{enumerate}

% \noindent From axioms~2 and~3 it follows that
% $$\s P(A^c) = 1 - P(A)\,, $$
% and that the highest value of $\s P(A)$ is \s 1:
% $$\s 0\le P(A) \le 1\,. $$ \b
% In particular $\s P(\not{\!0})=0$ as $\s E^c=\not{\!0}$. If 
% $\s A_1,\, A_2,\, \dots\ $ are {\r mutually exclusive events}, 
% the {\r addition rule}
% $$\s P(A_1\cup A_2 \cup\, \dots) = P(A_1) + P(A_2) +\ \dots $$
% holds.

% \newpage

The {\r\bf conditional probability} $\s P(B|A)$ of $\s B$ under the 
assumption that $\s A$ has occured is defined by 
$$\s P(B\cap A) = P(B|A)\,P(A)\ .$$

\noindent Event $\s B$ is {\r independent} of $\s A$ if 
$\s P(B|A)=P(B)$, which implies 
$$\s P(B\cap A)=P(B) \,P(A)\ .$$
% In practice $\s P(B|A)$ is rarely calculated from the knowledge of 
% $\s P(B\cap A)$ and $\s P(A),$ but instead $\s P(B\cap A)$ from 
% $\s P(B|A)$ and $\s P(A).$ 
Also, as $\s B\cap A=A\cap B$, we have
\begin{equation} \label{cp_interchange} \s
  P(A|B)\,P(B) = P(B|A)\,P(A)
\end{equation}
\noindent for the conditional probability of $\s A$ under the 
assumption that $\s B$ has occured. \vfill

\newpage

\noindent {\r\bf Total probability:}

Events $\s A_1,A_2,\dots,A_n$ form a {\r partition of the sample 
space} $\s E$ if
\begin{enumerate}
\item They are mutually exclusive, {\it i.e.}, {\s $A_i\cap A_j=
\not{\!0}$ for $i\not= j$,} and
\item Their union is the sample space, $\s E=\cup_{i=1}^n A_i.$
\end{enumerate}

\noindent
Let the event of interest be $\s B$ and assume that the conditional
probabilites $\s P(B|A_i)$ are known. Then $\s P(B)$ can be calculated
by the total probability formula
\begin{equation} \label{total_prob} \s
  P(B) = \sum_{i=1}^n P(B|A_i)\,P(A_i)\ .
\end{equation}
% {\bf\r Example: Two-Headed Coin.} Assume that out of $\s N$ coins in a 
% box, one has heads on both sides. A coin is selected at random from 
% the box and (without inspecting it) flipped $\s k$ times. Denote the 
% event that the randomly selected coin lands heads up $\s k$ times by 
% $\s B_k$. What is the probability that the coin is two-headed?
% \newpage

% {\bf Answer:} Let us denote the event that the  coin is two-headed by 
% $\s A_1$ and that the coin is fair by $\s A_2$. Obviously, we have the 
% {\it a-priori} probabilities $\s P(A_1)=1/N$ and $\s P(A_2)=(N-1)/N$. 
% The conditional probabilities for $\s B_k$ are $\s P(B_k|A_1)=1$ and 
% $\s P(B_k|A_2)=2^{-k}.$ Hence, by the total probability formula
% \begin{equation} \label{Bk_two_headed} \s
%  P(B_k) = \frac{1}{N}+\frac{N-1}{2^kN} = \frac{2^k+N-1}{2^kN}\ .
% \end{equation}
% Therefore, the {\r probability that we picked the two-headed coin is}
% \begin{equation} \label{prob_two_headed} \s
%  P(A_1|B_k) = \frac{2^k}{2^k+N-1}
% \end{equation}
% as follows from 
% \begin{equation} \s
%  P(A_1|B_k)\,P(B_k) = P(B_k|A_1)\,P(A_1) = \frac{1}{N}
% \end{equation}
% and expression (\ref{Bk_two_headed}) for $\s P(B_k)$. \bigskip
% \newpage

\foilhead{\r Bayes Theorem} \b

The interchange identity (\ref{cp_interchange}) for conditional
probabilities together with the total probability formula 
(\ref{total_prob}) incorperate {\bf Bayes Theorem:}

Let the event $\s B$ happen under any of the possibilities \s $A_i$,
$i=1,\dots,n$, $A_1\cup A_2 \cup\, \dots\, \cup A_n =E$, $A_i\cap
A_j = \not{\!0}$ for $i\ne j$, \b with known conditional probabilities 
$\s P(B|A_i)$. Assume, in addition, that the probabilities $\s P(A_i)$ 
are known. Then the conditional probability of $\s A_i$, given that 
$\s B$ has happened, is
$$\s P(A_i|B)\ =\ \frac{P(B|A_i)\,P(A_i)}{P(B)}\ =\
  \frac{P(B|A_i)\,P(A_i)}{\sum_{i=1}^n P(B|A_i)\,P(A_i)}\ .$$
In the context of this equation the events $\s A_i$ are often called
{\bf hypotheses}, the $\s P(A_i)$ are the {\bf prior} and the $\s 
P(A_i|B)$ are the {\bf posteriori probabilities.} \bigskip

\foilhead{\r Example (was homework): A Medical Test.} \b \vskip -30pt

The following question was posed to students at the Harvard Medical 
School: A test for a disease whose prevalence is $\s 1/1\,000$ fails 
never when the person is infected, but has a false positive rate of 
\s 5\%. \b A person is picked randomly from the population at large 
and tests positive. \r What is the probability that this person 
actually has the disease? \b

Let $\s A$ be the event that the patient has the disease and $\s B$ the 
event that the test result is positive. The conditional probability is 
{\s \begin{eqnarray} \label{Bayes2} 
  P(A|B) & =&\s \frac{P(B|A)\,P(A)}{P(B|A)\,P(A)+P(B|A^c)\,P(A^c)}\\ 
  \nonumber & =&\s \frac{1\times 0.001}{1\times 0.001 + 
                     0.05\times 0.999} \approx 0.02\ .
\end{eqnarray} }
{\r Physicist's shortcut:} Let us test {\s 1001} people. Then, 
{\s 50} test positive by chance and one by infection. So, the 
likelihood to be infected is $\s 1/51\approx 0.02.$ 

Assume the false positive rates are purely due to chance ({\it i.e.},
no dependence on conditions of the persons tested). \r How often needs 
the test to be repeated, 
so that we are sure with at least \s 99.9\% \r probability that a 
person is infected? What is then the expected number of tests needed 
for a sample of $\s 1\,001$ persons? \b

First question: The failure rate decreases like $\s (0.05)^n$, where 
$\s n$ is the number of tests. Let $\s B_n$ be the event that a person 
tests $\s n$ times positive. We want
$$\s 0.999 < \frac{0.001}{0.001 + (0.05)^n\times 0.999}\ .$$
Solving for the smallest $\s n$ gives $\s n_{\min}=5.$

Second question (physicist's answer): After the intial $\s 1\,001$ 
tests we are (in average) down to {\s 51} persons. After the second 
series of tests down to $\s 51/20\approx 3$. So, altogether about 
$\s 1\,001+51+3 = 1\,055$ tests are needed. Add two more tests for 
each person still positive after three tests.

\foilhead{\r Example: Prosecutor's Fallacy.} \b  \vskip -30pt

Let $\s A$ be the event {\s ``innocent''} and $\s B$ the {\s evidence.} 
The prosecutor's fallacy is an exchange of $\s P(A|B)$ (the probability 
to be innocent, given the evidence) by $\s P(B|A)$ (the probability 
of the evidence, given to be innocent). Assume that the probability 
to match some evidence by chance is $\s P(B|A)= 10^{-6}$ and that a 
match is found in an {\s archive of $2\times 10^6$ candidates.} A 
zealous prosecutor may then argue that the probability of the accused 
being innocent is one in a million, while the correct probability is 
obtained by applying (\ref{Bayes2}). In lack of information we use 
{\s 50\%} for the probability that the guilty person is in the archive, 
so that \s $\s P(A) = 0.5/(2\times 10^6)$, $P(A)=1-P(A^c)\approx 1$, 
and $P(B|A^c)=1$. \b Therefore, 
$$\s  P(A|B) = \frac{P(B|A)\,P(A)}{P(B|A)\,P(A)+P(B|A^c)\,P(A^c)}
     = \frac{10^{-6}\times 1}{10^{-6}\times 1 + 
       1\times 0.25\times 10^{-6}}\ =\ \frac{4}{5}\,,              $$
which is quite distinct from $\s 10^{-6}$.  

\noindent {\r Shortcut:} In average one picks two innocent people plus 
half the real guy.

\newpage
As a {\r real-life example} the convinction of {\r Sally Clark} in the 
U.K. is sometimes cited. She was accused of having killed her first 
child at the age of 11~weeks and her second child at the age of 8~weeks. 
The prosecution had an expert witness testify that the probability of 
two children dying from sudden infant death syndrome is about 1 in 73 
million. Sally Clark was convicted in 1999 and finally aquitted four 
years later.

\foilhead{\r A Proof for lecture08short.tex } \b

\noindent {\bf Total probability:} \s \smallskip

$$  P(B) = P(B\cap E) = P[B\cap (A_1\cup A_2\ \dots\ \cup A_n)] $$

$$  = P[(B\cap A_1) \cup (B\cap A_2)\ \dots\ \cup (B\cap A_n)] $$ \b

\noindent The events $\s (B\cap A_i)$ are mutually exclusive, because 
the $\s A_i$ are mutually exclusive. \s
$$  = P(B\cap A_1) + P(B\cap A_2) + \dots + P(B\cap A_n) 
    = \sum_{i=1}^n P(B|A_i)\,P(A_i)\ . $$
\vfill

\end{document}
