7. Bayesian Simple Linear Regression

Problem

Given a sample of $(x_{1}, y_{1}), \dots, (x_{n}, y_{n})$
Give the bast line for the dataset
$Y$ is the Response variable
$X$ is the Predictor variable

Model: $y_{i} = α + β (x_{i} - \bar{x}) + ε_{i}$ where $ε_{i} \overset{i i d}{\sim} N (0, σ^{2})$
Parameters: $α, β,$ and $σ^{2}$
Assumption: $σ^{2}$ is known

Goal

min_{α, β} \sum_{i = 1}^{n} [y_{i} - α - β (x_{i} - \bar{x})]^{2}

We get least squares estimator $\hat{α} = \bar{y}$ and $\hat{β} = \frac{\sum_{}^{} (x_{i} - \bar{x}) y_{i}}{\sum_{}^{} (x_{i} - \bar{x})^{2}}$
see Centred Model LSE

Likelihood

\begin{aligned} p ({ε_{i}} | α, β, σ^{2}) & = {(\frac{1}{\sqrt{2 π σ^{2}}})}^{n} e^{- \frac{1}{2 σ^{2}} \sum_{}^{} (y_{i} - α - β (x_{i} - \bar{x}))^{2}} = p ({y_{i}} | α, β, σ^{2}) \\ \sum_{}^{} (y_{i} - α - β (x_{i} - \bar{x}))^{2} & = \sum_{}^{} [(y_{i} - \bar{y}) - (\bar{y} - α) - β (x_{i} - \bar{x}))^{2}] \\ = \sum_{}^{} [(y_{i} - \bar{y})^{2} + 2 (y_{i} - \bar{y}) [(\bar{y} - α) - β (x_{i} - \bar{x}) \\ + [(\bar{y} - α) - β (x_{i} - \bar{x})]^{2} \\ = \sum_{}^{} (y_{i} - \bar{y})^{2} + 2 (\bar{y} - α) \sum_{}^{} {(y_{i} - \bar{y})}^{0} \\ - 2 β \sum_{}^{} (x_{i} - \bar{x}) (y_{i} - \bar{y}) \\ + \sum_{}^{} [(\bar{y} - α)^{2} - 2 β (x_{i} - \bar{x}) (y_{i} - \bar{y}) + β^{2} (x_{i} - \bar{x})^{2}] \\ = S_{y y} - 2 β S_{x y} + n (\bar{y} - α)^{2} \\ - 2 β (\bar{y} - α) {\sum_{}^{} (x_{i} - \bar{x})}^{0} \\ + β^{2} \sum_{}^{} (x_{i} - \bar{x})^{2} \\ = S_{y y} - 2 β S_{x y} + n (\bar{y} - α)^{2} + β^{2} \sum_{}^{} (x_{i} - \bar{x})^{2} \\ ⟹ p ({y_{i}} | α, β, σ^{2}) & = {(\frac{1}{\sqrt{2 π σ^{2}}})}^{n} \times e^{- \frac{1}{2 σ^{2}} S_{y y}} \times e^{- \frac{1}{2 σ^{2}} [β^{2} S_{x x} - 2 β S_{x y}]} \times e^{- \frac{1}{2 σ^{2}} (n (\bar{y} - α)^{2})} \\ ⟹ p ({y_{i}} | α, β, σ^{2}) & \propto e^{- \frac{1}{2 σ^{2}} [β^{2} S_{x x} - 2 β S_{x y}]} \times e^{- \frac{1}{2 σ^{2}} (n (\bar{y} - α)^{2})} \\ \propto \exp {- \frac{S_{x x}}{2 σ^{2}} [β^{2} - 2 β \frac{S_{x y}}{S_{x x}} + [\frac{S_{x y}}{S_{x x}}]^{2} - [\frac{S_{x y}}{S_{x x}}]^{2}]} \\ \times \exp {- \frac{1}{2 \frac{σ^{2}}{n}} ((\bar{y} - α)^{2})} \\ \propto \exp {- \frac{1}{\frac{2 σ^{2}}{S_{x x}}} [β^{2} - 2 β \frac{S_{x y}}{S_{x x}} + [\frac{S_{x y}}{S_{x x}}]^{2} - [\frac{S_{x y}}{S_{x x}}]^{2}]} \\ \times \exp {- \frac{1}{2 \frac{σ^{2}}{n}} ((\bar{y} - α)^{2})} \\ \propto \exp {- \frac{1}{\frac{2 σ^{2}}{S_{x x}}} [β - \frac{S_{x y}}{S_{x x}}]^{2}} \times \exp {- \frac{1}{\frac{2 σ^{2}}{S_{x x}}} [\frac{S_{x y}}{S_{x x}}]^{2}} \\ \times \exp {- \frac{1}{2 \frac{σ^{2}}{n}} ((\bar{y} - α)^{2})} \\ p ({y_{i}} | α, β, σ^{2}) & \propto N (\bar{y}, \frac{σ^{2}}{n}) \times N (\frac{S_{x y}}{S_{x x}}, \frac{σ^{2}}{S_{x x}}) \end{aligned}

\hat{α} = \bar{y}, v a r (\hat{α}) = \frac{σ^{2}}{n} {\hat{β}}_{1} = \frac{S_{x y}}{S_{x x}}, v a r ({\hat{β}}_{1}) = \frac{σ^{2}}{S_{x x}}

Posterior

\begin{aligned} Want P (α, β | {y_{i}}) & given P (α, β) = P (α) P (β) \\ P (α) & \sim N (m_{α} = \bar{y}, s_{α}^{2} = \frac{σ^{2}}{n}) \\ P (β) & \sim N (m_{β} = \frac{S_{x y}}{S_{x x}}, s_{β}^{2} = \frac{σ^{2}}{S_{x x}}) \\ P (α, β | {y_{i}}) & \propto p ({y_{i}} | α, β) p (α, β) \\ \propto p ({y_{i}} | β) p ({y_{i}} | α) p (α) p (β) \\ \propto \underset{posterior α}{\underset{⏟}{p ({y_{i}} | α) p (α)}} \underset{posterior β}{\underset{⏟}{p ({y_{i}} | β) p (β)}} \end{aligned}

\begin{aligned} For α : \\ \frac{1}{s_{α *}^{2}} & = \frac{1}{s_{α}^{2}} + \frac{n}{σ^{2}} \\ m_{α *} & = [\frac{(\frac{1}{s_{α}^{2}})}{\frac{1}{s_{α *}^{2}}}] m_{α} + [\frac{\frac{n}{σ^{2}}}{\frac{1}{s_{α *}^{2}}}] \bar{y} \\ For β : \\ \frac{1}{s_{β *}^{2}} & = \frac{1}{s_{β}^{2}} + \frac{n}{σ^{2}} \\ m_{α *} & = [\frac{\frac{1}{s_{β}^{2}}}{\frac{1}{s_{β *}^{2}}}] m_{β} + [\frac{\frac{S_{x x}}{σ^{2}}}{\frac{1}{s_{β *}^{2}}}] \frac{S_{x y}}{S_{x x}} \end{aligned}

by normal updating rules