From 777d228ff5e6e71b778a971abe46e3138d6d6b02 Mon Sep 17 00:00:00 2001 From: paul-loedige Date: Sat, 13 Nov 2021 12:24:53 +0100 Subject: [PATCH] aktueller Stand 12:24 2021-11-13 --- exercise.ipynb | 99 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/exercise.ipynb b/exercise.ipynb index 8cd2a45..60b0111 100644 --- a/exercise.ipynb +++ b/exercise.ipynb @@ -661,7 +661,104 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Implementation**\n", + "#### Solution\n", + "\\begin{align}\n", + " p(\\boldsymbol{x_i}) &= \\mathcal{N}\\left(\\boldsymbol{x_i} | \\boldsymbol{\\mu}, \\boldsymbol{\\Sigma} \\right)\\\\\n", + " &= \\dfrac{1}{\\sqrt{\\det \\left(2 \\pi \\boldsymbol{\\Sigma}\\right)}} \\exp\\left( - \\dfrac{(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})}{2}\\right)\n", + "\\end{align}\n", + "\n", + "##### calculate the likelihood\n", + "\\begin{align}\n", + " \\text{lik}(\\boldsymbol{\\theta};D) &= \\prod_i p(x_i)\\\\\n", + " &= \\prod_i \\dfrac{1}{\\sqrt{\\det \\left(2 \\pi \\boldsymbol{\\Sigma}\\right)}} \\exp\\left( - \\dfrac{(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})}{2}\\right)\\\\\n", + " \\log\\text{lik}(\\boldsymbol{\\theta};D) &= \\sum_i \\log p(x_i)\\\\\n", + " &= \\sum_i \\log(1) - \\log(\\sqrt{\\det(2\\pi\\Sigma)})- \\dfrac{(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})}{2}\\\\\n", + " &= \\sum_i^N - \\frac{1}{2}\\log(\\det(2\\pi\\Sigma))- \\dfrac{(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})}{2}\\\\\n", + " &= - \\frac{N}{2}\\log(\\det(2\\pi\\Sigma))- \\dfrac{1}{2}\\sum_i^N (\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\\\\n", + "\\end{align}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### derive $\\mu$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\\begin{align}\n", + " \\frac{\\partial \\log\\text{lik}(\\boldsymbol{\\theta};D)}{\\partial\\mu}\n", + " &= \\frac{\\partial}{\\partial\\mu}\\left(- \\frac{N}{2}\\log(\\det(2\\pi\\Sigma))- \\dfrac{1}{2}\\sum_i^N(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\right)\\\\\n", + " &= \\frac{\\partial}{\\partial\\mu}-\\dfrac{1}{2}\\sum_i^N (\\boldsymbol{x_i}^T-\\boldsymbol{\\mu}^T) \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\\\\n", + " &= \\frac{\\partial}{\\partial\\mu}-\\dfrac{1}{2}\\sum_i^N \\boldsymbol{x_i}^T\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{x_i} - \\boldsymbol{x_i}^T\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{\\mu} - \\boldsymbol{\\mu}^T\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{x_i} + \\boldsymbol{\\mu}^T\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{\\mu}\\\\\n", + " &= \\frac{\\partial}{\\partial\\mu}-\\dfrac{1}{2}\\sum_i^N \\boldsymbol{x_i}^T\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{x_i} -2\\boldsymbol{x_i}^T\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{\\mu}+ \\boldsymbol{\\mu}^T\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{\\mu}\\\\\n", + " &= -\\dfrac{1}{2}\\sum_i^N -2\\boldsymbol{x_i}^T\\boldsymbol{\\Sigma}^{-1} + 2\\boldsymbol{\\Sigma}^{-1}\\boldsymbol{\\mu}\\\\\n", + " &= -\\sum_i^N -\\boldsymbol{x_i}^T\\boldsymbol{\\Sigma}^{-1} + \\boldsymbol{\\Sigma}^{-1}\\boldsymbol{\\mu}\\\\\n", + " &= -\\boldsymbol{\\Sigma}^{-1}\\sum_i^N -\\boldsymbol{x_i}^T + \\boldsymbol{\\mu}\\\\\n", + "\\end{align}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "for Optimum: $-\\boldsymbol{\\Sigma}^{-1}\\sum_i^N -\\boldsymbol{x_i}^T + \\boldsymbol{\\mu} = 0$\n", + "\\begin{align} \n", + " 0 &= -\\boldsymbol{\\Sigma}^{-1}\\sum_i^N -\\boldsymbol{x_i}^T + \\boldsymbol{\\mu}\\\\\n", + " 0 &= \\sum_i^N -\\boldsymbol{x_i}^T + \\boldsymbol{\\mu}\\\\\n", + " 0 &= -N\\boldsymbol{\\mu}\\sum_i^N \\boldsymbol{x_i}^T \\\\\n", + " \\boldsymbol{\\mu} &= \\frac{1}{N}\\sum_i^N \\boldsymbol{x_i}^T \\\\\n", + "\\end{align}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### derive $\\Sigma$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\\begin{align}\n", + " \\frac{\\partial \\log\\text{lik}(\\boldsymbol{\\theta};D)}{\\partial \\boldsymbol{\\Sigma}} &=\n", + " \\frac{\\partial}{\\partial \\boldsymbol{\\Sigma}} \\left(-\\frac{N}{2}\\log(\\det(2\\pi\\Sigma))- \\dfrac{1}{2}\\sum_i^N (\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\right)\\\\\n", + " &= -\\frac{\\partial}{\\partial \\boldsymbol{\\Sigma}}\\frac{N}{2}\\log(\\det(2\\pi\\Sigma)) - \\frac{\\partial}{\\partial \\boldsymbol{\\Sigma}}\\dfrac{1}{2}\\sum_i^N (\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\\\\n", + " &= -\\frac{N}{2}(2\\pi\\Sigma)^{-1} - \\frac{\\partial}{\\partial \\boldsymbol{\\Sigma}}\\dfrac{1}{2}\\sum_i^N (\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\\\\n", + " &= -\\frac{N}{2}(2\\pi\\Sigma)^{-1} - \\dfrac{1}{2}\\sum_i^N \\frac{\\partial}{\\partial \\boldsymbol{\\Sigma}}(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\\\\n", + "\\end{align}\n", + "Because $(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})$ is scalar, we can take its trace and obtain the following form \n", + "\\begin{align}\n", + " &= -\\frac{N}{2}(2\\pi\\Sigma)^{-1} - \\dfrac{1}{2}\\sum_i^N \\frac{\\partial}{\\partial \\boldsymbol{\\Sigma}} \\text{Tr}\\left((\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1} (\\boldsymbol{x_i}-\\boldsymbol{\\mu})\\right)\\\\\n", + "\\end{align}\n", + "The rule on page 10 (formula 63) of the matrix cookbook allows us to derive the trace\n", + "\\begin{align}\n", + " &= -\\frac{N}{2}(2\\pi\\Sigma)^{-1} - \\dfrac{1}{2}\\sum_i^N -\\left(\\boldsymbol{\\Sigma}^{-1}(\\boldsymbol{x_i}-\\boldsymbol{\\mu})(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1}\\right)^T\\\\\n", + "\\end{align}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$-\\frac{N}{2}(2\\pi\\Sigma)^{-1} - \\dfrac{1}{2}\\sum_i^N -\\left(\\boldsymbol{\\Sigma}^{-1}(\\boldsymbol{x_i}-\\boldsymbol{\\mu})(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1}\\right)^T = 0$ to optain the optimum:\n", + "\n", + "\\begin{align}\n", + " 0 &= -\\frac{N}{2}(2\\pi\\Sigma)^{-1} - \\dfrac{1}{2}\\sum_i^N -\\left(\\boldsymbol{\\Sigma}^{-1}(\\boldsymbol{x_i}-\\boldsymbol{\\mu})(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1}\\right)^T\\\\\n", + " &= N(2\\pi\\Sigma)^{-1} - \\sum_i^N \\left(\\boldsymbol{\\Sigma}^{-1}(\\boldsymbol{x_i}-\\boldsymbol{\\mu})(\\boldsymbol{x_i}-\\boldsymbol{\\mu})^T \\boldsymbol{\\Sigma}^{-1}\\right)^T\\\\\n", + "\\end{align}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### **Implementation**\n", "\n", "Lets reuse one of the implementations from the zeroth-exercise for that " ]