4pillars.tex~

%grep -r --include "*.tex" 'stochastic process' .

%\tiny, \scriptsize, \footnotesize, \small, \normalsize, \large, \Large, \LARGE, \huge, \Huge

%https://github.com/sisl/GitHub-ForceLargeFiles

% numinleft puts section numbers in left gutter
% itchap makes chapters in italics
%\documentclass[asymtwoside,fleqn,itchap,numinleft,MITBook,openright]{fbook}
\documentclass[fleqn,MITBook,openright]{fbook}
%\documentclass[fleqn,itchap,numinleft,MITBook,openright]{fbook}
%\setmonofont{DejaVu Sans Mono}[Scale=MatchLowercase]


\newcommand{\textdir}{..}
%\newcommand{\textdir}{../Text}
\newcommand{\figdir}{../figures}
%\newcommand{\figdir}{/Users/kpmurphy/figuresCMYK}


\input{\textdir/packages}

% control how fig references appear
\newcommand{\bookname}{book1}
\providetoggle{draft}
\settoggle{draft}{true}
%\settoggle{draft}{false}

\input{\textdir/macros}
%\input{\textdir/line-numbering-macros}

% for biblatex
%\addbibresource{\textdir/abbrShort,\textdir/bib}

\setevenfoottext{Draft of  ``Probabilistic Machine Learning: An Introduction''. \today}
\setoddfoottext{Author: Kevin P. Murphy. (C) MIT Press. CC-BY-NC-ND license}

% https://tex.stackexchange.com/questions/123652/change-index-font-size-with-imakeidx
\makeindex[columns=3]

%\usepackage[font=scriptsize]{idxlayout}
\indexsetup{othercode=\scriptsize}

% https://tex.stackexchange.com/questions/24561/setting-the-column-gap-in-a-twocolumn-or-multicol-document
\setlength\columnsep{40pt} % for bib

%\newcommand{\gdm}[1]{\textcolor{cyan}{[#1]}}
\newcommand{\fcontext}{f_{\text{context}}}
\newcommand{\fdynamics}{f_{\text{dynamics}}}
\newcommand{\fobservations}{f_{\text{observation}}}
\renewcommand{\freward}{f_{\text{reward}}}
\newcommand{\fpolicy}{g_{\text{sample-policy}}}
\newcommand{\fupdate}{g_{\text{update-policy}}}

\newcommand{\pdyn}{p_{\text{dyn}}}
\newcommand{\pobs}{p_{\text{obs}}}


\newcommand{\rec}{\mathrm{rec}}

\newcommand{\ytrue}{\stackrel{*}{y}}
\newcommand{\ypred}{\hat{y}}
%\newcommand{\MMD}{\text{MMD}}

\begin{document}

\begin{align*}
  \data_{t+1|t} &= \data_t \\
  \vtheta_{t+1|t} &= \vtheta_t + \gauss(\vzero, q \vI) \\
  \vb_{t+1|t}(\vz_{t+1}) &= \sum_{\vz_t} \pdyn(\vz_{t+1} | \vz_t, \vtheta_{t+1|t}^m)
  \vb_t(\vz_t) \\
  [\va_{t+1}, \vh_{t+1|t}] &= \text{policy}(\vh_t, \vb_{t+1|t}, \vtheta_{t+1|t}^a) 
\end{align*}

\begin{align*}
  \vb_{t+1} &=
  \text{bel-update}(\vb_{t+1|t}, \vo_t, \vtheta_{t+1|t}^m) 
\propto \vb_{t+1|t}(\vz_{t+1}) \pobs(\vo_{t+1}|\vz_{t+1},\vtheta_{t+1|t}^m) \\
  \hat{\vz}_{t+1} &= \argmax_{\vz} \vb_{t+1}(\vz) \\
  \data_{t+1} &= \text{append}(\data_{t+1|t}, \vo_{t+1}, \hat{\vz}_{t+1}) \\
  \vtheta^m_{t+1} &= \text{model-update}(\vtheta^m_{t+1|t}, \data_{t+1})\\
  \tilde{\data}^m_{t+1} &= \text{gen-data}(\vtheta_{t+1}^m) \\
  \vtheta^a_{t+1} &= \text{policy-update}(\vtheta^a_{t+1|t}, \data_{t+1},
  \tilde{\data}^m_{t+1})
\end{align*}


\end{document}

\begin{align*}
p(\vw_t|\vx_{1:t},\vy_{1:t})
\end{align*}

\begin{align*}
  (\pi_1,\ldots,\pi_O, \phi)
 = \argmin \sum_{i=1}^{N_{\text{pre}}} \sum_t ||a_i^t - \pi_i(\phi(\vx_i^t))||_2^2
\end{align*}


\begin{align*}
  \calX &=\{\vx_n \sim P\}_{n=1}^N, \calX' = \{\vx'_m \sim Q \}_{m=1}^M \\
\MMD^2(\calX, \calX')
 &\defeq ||\frac{1}{N} \sum_{n=1}^N \vphi(\vx_n) - \frac{1}{M}
\sum_{m=1}^M \vphi(\vx'_m) ||^2 \\
&=
\frac{1}{N^2} \sum_{n=1}^N \sum_{n'=1}^N \kernelfn(\vx_n,\vx_{n'})
  -
  \frac{2}{NM} \sum_{n=1}^N \sum_{m=1}^M \kernelfn(\vx_n, \vx'_m)
    +
    \frac{1}{M^2} \sum_{m=1}^M \sum_{m'=1}^M \kernelfn(\vx'_m,\vx'_{m'})
    \\
%  \calX &=\{\vx^* \sim P\}_{n=1}^N, \tilde{\calX} = \{\tilde{\vx}_m \sim Q \}_{m=1}^M \\
\MMD^2(\vx^*, \tilde{\calX}) &=
\kernelfn(\vx^*,\vx^*)
  -
  \frac{2}{M} \sum_{m=1}^M \kernelfn(\vx^*, \tilde{\vx}_m)
    +
    \frac{1}{M^2} \sum_{m=1}^M \sum_{m'=1}^M \kernelfn(\tilde{\vx}_m,\tilde{\vx}_{m'}) \\
    L_t(Q; \vy^*_{1:T}) &= \MMD^2( \vy_{t+\Delta}^*, \{ \tilde{\vy}_{t+\Delta} \sim Q(\cdot | \vy_{1:t}) \}) \\
  L(Q; \vy^*_{1:T}) &= \frac{1}{T} \sum_{t=1}^T L_t(Q_t; \vy^*_{1:T})
\end{align*}

Train MLP (5 hidden layers)
to predict $(\dot{x}[0], \dot{x}[1]) = b(s, x[0], x[1], x_0[0], x_0[1])$
for a given lag $\tau=0.5$ using set of $N=10^5$ pairs $(\vx_t, \vx_{t+0.5})$.

\begin{align*}
  \alpha_s=1-s, \beta_s=s^2, \sigma_s=\epsilon(1-s)
  \end{align*}

\begin{itemize}
\item $x_0$ is a noisy image, $x_1$ is clean version.
  
\item Given time series $\{\cdots, y_{-\tau}, y_0, t_{\tau}, \cdots\}= \{ y_{k \tau} \}$,
where $\tau$ is the lag (sampling frequency).\\
Let $x_0 = y_{k \tau}$, $x_1 = y_{(k+1) \tau}$.
If $\tau=1$, we predict $p(x_1 | x_0) = p(y_{k+1}|y_k)$ .
\end{itemize}

Here $\rho_0(x) = \delta(x-x_0)$, so the source distribution is a point mass.


Define $t=e^{-\tau}$ and evauate $y_{\tau}$ at $\tau=-\log t$ to get

If $x_1 \sim \rho_1$ then $y_{\tau} \ra \gauss(0,I)$ as $\tau \ra \infty$.
In practice we truncate to a finite time interval $[0,T]$ and assume
$y_T \sim \gauss(0,I)$.

To generate, we sample from $\rho_0$ and then sample
from the reverse reverse the SDE and sample from $

Consider a velocity field $b$ with probability flow
\begin{align*}
  \frac{d}{dt} X_t(x) =  b(t, X_t(x)), \;\; X_{t=0}(x) = x
\end{align*}
Let $X_{t=1}(z)$ be the solution to the ODE at $t=1$
(predicted clean image).
Use this to define the rectified flow interpolant
\begin{align*}
x_t^{\rec} = M_t(z) = \alpha(t) z + \beta(t) X_{t=1}(z)
\end{align*}
Intuitively this has ``precognition'' and predicts
where $z$ will decode to in $p_1$.
This interpolant has velocity field
\begin{align*}
  b^{\rec}(t,x) = E[\dot{x}_t^{\rec}| x_t^{\rec}=x]
  =
  \dot{\alpha}(t)E[z|x_t^{\rec}=x]
  +
    \dot{\beta}(t)E[X_{t=1}(z)|x_t^{\rec}=x]
\end{align*}
and probability flow
\begin{align*}
  \frac{d}{dt} X_t^{\rec}(x) =  b^{\rec}(t, X_t^{\rec}(x)),
  X_{t=0}^{\rec}(x) = x
\end{align*}

Thm 5.5: $X_{t=1}^{\rec}(z) \sim \rho_1$ if $z \sim \gauss(0,I)$.
Further, if mapping $M_t(z)= \alpha(t) z + \beta(t) X_{t=1}(z)$
is invertible (i.e., there is $N_t=M_t^{-1}$),
then
\begin{align*}
  b^{\rec}(t,x) 
  =
  \dot{\alpha}(t)N_t(x)
  +
    \dot{\beta}(t) X_{t=1}(N_t(x))
\end{align*}
Since $M_{t=0}(x)=N_{t=0}(x)=x$, we have at $t=0$
\begin{align*}
  b^{\rec}(t=0,x) 
  =
  \dot{\alpha}(0)x
  +
    \dot{\beta}(0) X_{t=1}(x)
\end{align*}
If $\dot{\alpha}(0)=0$ and $\dot{\beta}(0)=1$, then 
  $b^{\rec}(t=0,x)  = X_{t=1}(x)$, so we jump to the solution in one step.


\begin{align*}
  p(t,x) &\propto \exp(-E(t,x)) \\
  s(t,x) &= \nabla_x \log p(t,x) = -\nabla_x E(t,x)
\end{align*}


\begin{align*}
  \loss_{\eta}[\hat{\eta}_z]
  &= \half \int_0^1 E\left[
    ||\hat{\eta}_z(t,x_t)  -    z_t)||^2 \right] dt \\
  &= \int_0^1 E\left[
    \half ||\hat{\eta}(t,x_t)||^2
    -  \hat{\eta}_z(t,x_t) \cdot z_t)
\right] dt 
\end{align*}



\begin{align*}
  \loss_b[\hat{b}]
  &= \half \int_0^1 E\left[
    ||\hat{b}(t,x_t)  -
    (\partial_t I(t,x_0,x_1) + \dot{\gamma}(t) z_t)||^2 \right] dt \\
  &= \int_0^1 E\left[
    \half ||\hat{b}(t,x_t)||^2
    -   (\partial_t I(t,x_0,x_1) + \dot{\gamma}(t) z_t)
\cdot \hat{b}(t,x_t)\right] dt 
\end{align*}


\begin{align*}
  \loss_s[\hat{s}]
  &= \half \int_0^1 E\left[
    ||\hat{s}(t,x_t)  + \frac{z_t}{\gamma(t)}||^2 \right] dt \\
  &= \int_0^1 E\left[
    \half ||\hat{s}(t,x_t)||^2
    + \gamma^{-1}(t) z_t \cdot \hat{s}(t,x_t)\right] dt \\
\end{align*}


\begin{align*}
  \alpha(t) &=1-t, \beta(t)=t, \gamma(t)=0\\
b(t,x) &= -\eta_0(t,x) + \eta_1(t,x) = \eta_1 - \eta_0
\end{align*}

\begin{align*}
  \gamma(t) & =0 \\
  \gamma(t) & =\sqrt{2 t (1-t)}\\
    \gamma(t) & =\sin^2(\pi t)
\end{align*}


\begin{align*}
  \eta_1^{os}(t,x) &= \hat{x}_0,
  \eta_z^{os}(t,x) = \epsilon_{\theta}(t,x) \\
  \alpha(t) &= t,   \beta(t) = 1
  \Rightarrow
  \hat{x}_0 = x - \sigma \epsilon_{\theta}(\sigma,x)
\end{align*}

\begin{align*}
E[x_t|x_t=x]=x
\end{align*}

\begin{align*}
  \loss_b[\hat{b}]
   &= 
  \int_0^1
 \half E_{z \sim \gauss(0,I), (x_0, x_1) \sim \nu()}
  \left[ ||\hat{b}(t,x_t) - R_t||^2
    \right] dt \\
   &= 
  \int_0^1
 E
  \left[ \half ||\hat{b}(t,x_t)||^2  - R_t  \hat{b}(t,x_t)
    \right] dt  \\
    R_t &= \dot{x}_t = \partial_t I(t,x_0,x_1) + \dot{\gamma}(t) z 
\end{align*}

\begin{align*}
\nabla \cdot u  &= \sum_{i=1}^N \frac{\partial}{\partial x_i} u(x_{1:N}) 
\end{align*}

\begin{align*}
\Delta R(\vZ, \hat{\vZ}) 
&= \sum_{j=1}^k \Delta R(\vZ_j, \hat{\vZ}_j) \\
\Delta R(\vZ_j, \hat{\vZ}_j)
&= R(\vZ_j \union \hat{\vZ}_j) - \half\left(
R(\vZ_j) + R(\hat{\vZ}_j) \right)
  \end{align*}


\begin{align*}
  (\veta^*, \vtheta^*) &= \arg \min_{\veta} \max_{\vtheta} J(\vtheta,\veta)  \\
  J(\vtheta,\veta)
  &= \Delta R(f(\vX,\vtheta)) + \Delta R(h(\vX,\vtheta,\veta))
  + \Delta R(f(\vX,\vtheta), h(\vX,\vtheta,\veta)) \\
  &= \Delta R(\vZ) + \Delta R(\hat{\vZ})
  + \Delta R(\vZ, \hat{\vZ}) \\
  h(\vx,\vtheta, \veta) &= f(g(f(\vx,\vtheta), \veta), \vtheta) 
  \end{align*}

\begin{align*}
  J^b(\vtheta,\veta)
  &= \Delta R(\vZ, \hat{\vZ})
  =R(\vZ \union \hat{\vZ}) - \half\left(
R(\vZ) + R(\hat{\vZ}) \right)
  \end{align*}

\begin{align*}
  \hat{y}
  &= \argmin_{\hat{y} \in \calY}
  E_y\left[ \lossfn_{01}(y, \hat{y}) | \vx \right] \\
  &= \argmin_{\hat{y} \in \calY}
  \sum_{y \in \calY} p(y|\vx) \lossfn_{01}(y, \hat{y}) \\
  &= \argmin_{\hat{y} \in \calY} \left[
    1 \times \sum_{y \neq \hat{y}} p(y|\vx)
  +  0 \times \sum_{y = \hat{y}} p(y|\vx) \right] \\
  &= \argmax_{\hat{y} \in \calY}   p(\hat{y} | \vx)
  \end{align*}

\begin{tabular}{lllll}
  {\bf Prediction} (supervised ML) \\
  $\data=\{(\vx_n,\vy_n)\}$ & $\vh=(\vx,\vy)$ & $\vo=(\vx;\data)$
  & $\hat{\vy} = f(\vx;\data)$ & $\ell(\vy,\hat{\vy})$ \\ \\
  {\bf Analysis} (unsupervised ML) \\
  $\data=\{\vx_n\}$ & $\vh=(\vx,\vz)$ & $\vo=(\vx;\data)$
  & $\hat{\vz} = f(\vx;\data)$ & $\ell(\vz,\hat{\vz})$ \\ \\
  {\bf Synthesis} (generative ML) \\
  $\data=\{(\vc_n, \vx_n)\}$ & $\vh=(\vc, \vx)$ & $\vo=(\vc;\data)$
  & $\hat{\vx} = f(\vc;\data)$ & $\ell((\vc,\vx),\hat{\vx})$ \\ \\
  {\bf Planning} (RL)\\
  $\data=\{\}$ & $\vh_t$ & $\vo_t$
  & $\va_t = \pi(\vo_{1:t})$ & $\ell(\vh_t,\va_t)$ \\ 
\end{tabular}

\begin{align}
\va_t &= \pi(\vo_{1:t}) \\
\ell(\vh_t, \va_t)
\end{align}

\begin{align}
\hat{\vx} &= f(\vc;\data) \\
\ell((\vc,\vx), \hat{\vx})
\end{align}

\begin{align}
\hat{\vy} &= f(\vx;\data) \\
\ell(\vy, \hat{\vy})
\end{align}

\begin{align}
\hat{\vz} &= f(\vx;\data) \\
\ell(\vz, \hat{\vz})
\end{align}


\be
\hat{\vz}_n = f(\vx_n; \data) =  \argmin_{\hat{\vz}_n}
\int p(\vz_n|\vx_n, \data) \|\vz_n - \hat{\vz}_n\|_2^2 d\vz_n
 = E[\vz_n|\vx_n, \data]
\ee

\be
\hat{z}_n = f(\vx_n; \data) =  \argmin_{\hat{z}_n}
\sum_{z_n} p(z_n|\vx_n, \data) \ind{z \neq \hat{z}_n} 
 = \max_{z_n} p(z_n|\vx_n, \data)
\ee


\be
\hat{\vz}_n = f(\vx_n; \data) =  \argmin_{\hat{\vz}_n}
\int d\vz_n p(\vz_n|\vx_n, \data) \ell_2(\vz_n, \hat{\vz}_n)
 = E[\vz_n|\vx_n, \data]
\ee

\be
\hat{z}_n = f(\vx_n; \data) =  \argmin_{\hat{z}_n}
\sum_{z_n} p(z_n|\vx_n, \data) \ell_{01}(z, \hat{z}_n)
 = \max_{z_n} p(z_n|\vx_n, \data)
\ee


\begin{align}
  \hat{\vtheta} &=
  \argmax_{\vtheta} \sum_{n=1}^N \log p(\vx_n|\vtheta) \\
  &=  \argmax_{\vtheta} \sum_{n=1}^N \log \left[    \sum_{\vz_n} p(\vx_n|\vz_n,\vtheta) p(\vz_n|\vtheta) \right]
\end{align}


\begin{align}
  \hat{\vtheta} &=   \argmax_{\vtheta} \log p(\data|\vtheta) 
=  \argmax_{\vtheta} \sum_{n=1}^N \log p(y_n|\vx_n,\vtheta)
\end{align}

\begin{algorithm}
  \dontprintsemicolon
    \mbox{def PS-MBPO(env, E, T, T'):} \;
  $p(\vpi, \vtheta) =p(\vtheta) p(\vpi|\vtheta) \approx
  \frac{1}{NM} \sum_{n=1}^N \sum_{m=1}^M \delta(\vtheta-\vtheta_n) \delta(\vpi-\vpi_{nm})$ \;
  \For{$e=1:E$}{
  $\tilde{\vpi}_e \sim p(\vpi)$  // Sample a policy \;
  $\vs_0 \sim \mbox{env.init}()$ \;
    $\data_e = \mbox{collect-real-data}(\text{env}, \vs_0, \tilde{\vpi}_e, T)$ \;
   // Update model beliefs  using real data \;
  \For{$n=1:N$}{  
    $\vtheta_n = \text{SGD}(\data_{1:e},\vtheta_n)$
    // MLE for MLP reward and dynamics model\;
  }
  // Update policy beliefs using real and simulated data \\
    $\tilde{\vs}_0 \sim \data_{1:e}$ \;
  \For{$n=1:N$, $m=1:M$}{ 
    $\tilde{\data}_{nm} = \mbox{collect-synth-data}(\vtheta_{n}, \tilde{\vs}_0, \vpi_{nm}, T')$\;
    $\vpi_{nm} = \mbox{soft-actor-critic}(\lambda \data_{1:e} + (1-\lambda) \tilde{\data}_{nm},
    \vpi_{nm})$ 
  }
  }
\end{algorithm}



\begin{align}
  \mbox{def PS-MBPO(env, E, T, T'):} \\
  p(\vpi, \vtheta) &=p(\vtheta) p(\vpi|\vtheta) \approx
  \frac{1}{NM} \sum_{n=1}^N \sum_{m=1}^M \delta(\vtheta-\vtheta_n) \delta(\vpi-\vpi_{nm}) \\
  \text{for $e=1:E$} \\
  \tilde{\vpi}_e & \sim p(\vpi) \mbox{ // Sample a policy} \\
  \vs_0 &\sim \mbox{env.init}() \\
  \data_e &= \mbox{collect-real-data}(\text{env}, \vs_0, \tilde{\vpi}_e, T)\\
  p(\vtheta|\tilde{\vpi}_e) &= \mbox{update-bel-model}(p(\vtheta|\tilde{\vpi}_e), \data_{1:e})\\
  \tilde{\vtheta}_e &\sim p(\vtheta) \mbox{ // Sample a model} \\
  \tilde{\vs}_0 &\sim \data_{1:e} \\ 
  \tilde{\data}_e &= \mbox{collect-synth-data}(\tilde{\vtheta}_e, \tilde{\vs}_0, \tilde{\vpi}_e, T')\\
  p(\vpi) &= \mbox{update-bel-policy}(p(\vpi), \lambda \data_{1:e} \union (1-\lambda) \tilde{\data}_e) 
\end{align}



\begin{align}
\mbox{def collect-real-data}(\text{env}, \vs_0, \tilde{\vpi}_e, T): \\
  \data_e = \{ \} \\
  \text{for $t=1:T$} \\
  \va_t &\sim \tilde{\vpi}_e(\vs_{t-1}) \\
 (\vs_t, r_t) &= \mbox{env.step}(\vs_{t-1}, \va_t) \\
  \data_e &= \mbox{append}(\data_e, (\vs_t, \va_t, r_t)) \\
\end{align}



\begin{align}
  \mbox{def collect-synthetic-data}(\tilde{\vtheta}, \vs_0, \tilde{\vpi}_e, T'): \\
    \data_e = \{ \} \\
  \text{for $t=1:T'$} \\
  \va_t &\sim \tilde{\vpi}_e(\vs_{t-1}) \\
  \vs_t &= \fdynamics(\vs_{t-1}, \va_t, \tilde{\vtheta}) \\
  r_t &= \freward(\vs_{t}, \va_t, \tilde{\vtheta})\\
  \data_e &= \mbox{append}(\data_e, (\vs_t, \va_t, r_t)) \\
\end{align}

\begin{align}
  \mbox{def update-bel-model}(p(\vtheta)=\{\vtheta_{1:N}\}, \data): \\
  \text{for $n=1:N$}\\
  \vtheta_n &= \text{SGD}(\data, \text{seed}_n)
\end{align}

\begin{align}
  \mbox{def update-bel-policy}(p(\vpi)=\{\vpi_{1:M}\} , \data): \\
  \text{for $m=1:M$}\\
  \vpi_m &= \text{SAC}(\data, \text{seed}_m)
\end{align}


\begin{align}
  \tilde{\vpi}_e \sim p(\vpi | \data_{1:e-1}),  \data_e = \{ \},    \vs_0 &\sim \mbox{env-init}()
\end{align}



\begin{align}
  \tilde{\vtheta}^a &\sim \gauss(\vtheta^a | \vmu_{t-1}^a, \vSigma_{t-1}^a)
   \mbox{ // Explore (sample)} \\
  a_t &= \argmax_a R(\vs_t, a; \tilde{\vtheta}^a)
  \mbox{ //  Exploit (maximize)} \\
  r_t &= R^*(\vs_t, a_t)
  \mbox{ // Environment step} \\
  (\vmu_t^{a}, \vSigma_t^{a}) &=
  \begin{cases}
    \text{LOFI}(\vmu_{t-1}^a, \vSigma_{t-1}^a, r_t) &\mbox{if $a=a_t$ // Update belief state} \\
    (\vmu_{t-1}^a, \vSigma_{t-1}^a) &\mbox{if $a \neq a_t$ // No info. about unused arms}
    \end{cases}
\end{align}


\begin{align}
  \hat{a} = \pi(o; D)
  &= \argmin_{a \in \calA}
E_{h|o,D}\left[ \lossfn(h, a) \right] \\
  &= \argmax_{a \in \calA}
E_{h|o,D}\left[ U(h, a) \right]
\end{align}


\begin{align}
  L_t &= -\textcolor{blue}{\freward}(a_t) \\
  a_t &= \textcolor{black}\fpolicy(\vpi_t) \\
  \vpi_{t+1} &= \textcolor{Apricot}\fupdate(\vpi_t, a_t,  L_t) 
\end{align}

\begin{align}
  \vs_t &= \textcolor{red}{\fcontext}(t) \\
  L_t &= -\textcolor{blue}{\freward}(\vs_{t}, \va_t) \\
  \va_t &= \textcolor{black}\fpolicy(\vpi_t, \vs_t) \\
  \vpi_{t+1} &= \textcolor{Apricot}\fupdate(\vpi_t, \vs_t, \va_t,  L_t) 
\end{align}

\begin{align}
  \vs_t &= \textcolor{red}{\fdynamics}(\vs_{t-1}, \va_{t-1}) \\
  L_t &= -\textcolor{blue}{\freward}(\vs_{t}, \va_t) \\
  \va_t &= \textcolor{black}\fpolicy(\vpi_t, \vs_t) \\
  \vpi_{t+1} &= \textcolor{Apricot}\fupdate(\vpi_t, \vs_t, \va_t,  L_t) 
\end{align}

\begin{align}
  \vh_t &= \textcolor{red}{\fdynamics}(\vh_{t-1}, \va_{t-1}) \\
  \vo_t &= \textcolor{green}{\fobservations}(\vh_{t}) \\
  L_t &= -\textcolor{blue}{\freward}(\vh_{t}, \va_t) \\
  \va_t &= \textcolor{black}\fpolicy(\vpi_t, \vo_t) \\
  \vpi_{t+1} &= \textcolor{Apricot}\fupdate(\vpi_t, \vo_t, \va_t,  L_t) 
\end{align}


\begin{align}
\hat{\vz} &= \argmax_{\vz} p(\vz|\vx,\hat{\vtheta})
\end{align}





\begin{align}
  \ell_{01}(z,\hat{z}) &= \ind{z=\hat{z}} \\
  \ell_2(\vz,\hat{\vz}) &= ||\vz-\hat{\vz}||_2^2 \\
\end{align}


\begin{align}
  \pi = \mbox{train}(D)
\end{align}



\begin{align}
  p(\vx_t^i|\vx_{t-1})
  &= \psi_d(\vx_t^i | \vx_{t-1}^i) \prod_j \psi_r(\vx_t^i | \vx_{t-1}^j) \\
  &\approx \gauss^{-1}(\vx_t^i | \veta_{dti}, \vLambda_{dti}')
  \prod_j \gauss^{-1}(\vx_t^i | \veta_{rtij}, \vLambda_{rtij}')\\
  &=  \gauss^{-1}(\vx_t^i | \veta_{ti}, \vLambda_{ti}') \\
  \veta_{ti} &= \veta_{dti} + \sum_j \veta_{rtij} \\
  \Lambda'_{ti} &= \vLambda'_{dti} + \sum_j \vLambda'_{rtij} \\  
\end{align}


\begin{align}
  \psi_{r}(\vx_t^i | \vx_{t-1}^j) &\defeq  \psi_r(\vx_t^i, f_d(\vx_{t-1}^j))
  \approx \gauss^{-1}(\vx_t^i|\veta_{rtj}, \vLambda_{rtj}') \\
  (\veta_{rtj},\vLambda_{rtj}') &= \text{linearize}(h_r(\cdot, f_d(\vx_{t-1}^j)), f_d(\vx_{t-1}^i), \vLambda_r)
\end{align}


\begin{algorithm}
\dontprintsemicolon
\caption{Sampling trajectories of length $T$ for $N$ objects}
\label{algo:step}
$\vx_{0,1:N} = \text{init-state}(\vx_{<0, 1:N})$ \;
\For{$t=1:T$}
    {
      \For{$i=1:N$}
          {
            Compute posterior predictive $(\veta_{ti}, \vLambda_{ti}')
            = g(\vx_{t-1, 1:N}, i, f_d(), h_r(), \vLambda_d, \vLambda_r)$ \;
            Compute $\vSigma_{ti} = (\vLambda_{ti}')^{-1}$ \\
            Sample $\vx_{ti} \sim \gauss(\cdot | \vSigma_{ti} \veta_{ti}, \vSigma_{ti})$
          }
    }
\end{algorithm}

\begin{align}
  (\veta_s,\vLambda_s') &= \text{linearize}(h_s,\vx_0, \vLambda_s)
\end{align}


\begin{align}
  \psi_s(\vx_s) &= \exp\left(-\half (\vz_s - \vh_s(\vx_s))^\trans \vLambda_s  (\vz_s - \vh_s(\vx_s)) \right)  
\end{align}



\begin{align}
  \psi_d(\vx_t^i | \vx_{t-1}^i) &= \gauss^{-1}(\vx_t^i | \veta_{dti}, \vLambda_{dti}') \\
  \vLambda'_{dti} &= \vLambda_d \\
  \veta_{dti} &= \vLambda_d f_d(\vx_{t-1}^i)
\end{align}


\begin{align}
  f_d(\vx_{t-1}^i) &= \vPhi \vx_{t-1}^i \\
  \vPhi &= \begin{pmatrix} \vI & \Delta \vI \\
    \vzero & \vI \end{pmatrix} \\
  \vLambda_d^{-1} &= \begin{pmatrix}
    \frac{1}{3} \Delta^3 \vQ_d & \half \Delta^2 \vQ_d \\
    \half \Delta^2 \vQ_d & \Delta \vQ_d \end{pmatrix} \\
  \vQ_d &= \sigma_d^2 \vI_2
\end{align}


\begin{align}
  \psi_d(\vx_{t-1}^i, \vx_{t}^i) &=
  \exp(-\half \vh_d(\vx_{t-1}^i, \vx_{t}^i)^\trans \vLambda_d  \vh_d(\vx_{t-1}^i, \vx_{t}^i))  \\
  \vh_d(\vx_{t-1}^i, \vx_{t}^i) &= f_d(\vx_{t-1}^i) - \vx_{t}^i \\
p_d(\vx_{t}^i | \vx_{t-1}^i) &= \gauss(\vx_{t}^i | f_d(\vx_{t-1}^i), \vLambda_d^{-1})
  \end{align}





\begin{align}
  \psi_r(\vx_t^i, \vx_t^j)
  &= \exp(-\half \vh_r(\vx_t^i, \vx_t^j) ^\trans \vLambda_r \vh_r(\vx_t^i, \vx_t^j))  \\
  \vh_r(\vx_t^i, \vx_t^j) &=
  \begin{cases}
    1- \frac{d_r(\vx_t^i, \vx_t^j)}{r^*} &\mbox{if $d \leq r^*$} \\
    0 &\mbox{otherwise}
  \end{cases} \\
  d_r(\vx_t^i, \vx_t^j) &= ||\vx_t^i- \vx_t^j||_2 \\
  \vLambda_r &= \sigma_r^{-2} \vI_1
\end{align}



\begin{align}
  \veta_s &= \vLambda_s \vz_s \\
  \vLambda_s' &= \vLambda_s \\
 \gauss(\vx_s|\vz_s,\vLambda_s^{-1}) &\propto \gauss^{-1}(\vx_s | \veta_s, \vLambda_s)
\end{align}


\begin{align}
  \veta_s &=  \vH_s^\trans \vLambda_s[\vH_s \vm_s + \vz_s - \vH_s \vm_s -\vm_s)]
  =\vH_s^\trans \vLambda_s (\vz_s- \vm_s)\\
  \vLambda_s' &= \vH_s^\trans \vLambda_s \vH_s 
\end{align}

\begin{align}
  \psi_s(\vx_s) &\approx \gauss^{-1}(\vx_s |\veta_s, \vLambda_s') \\
  \gauss^{-1}(\vx_s | \veta_s, \vLambda_s')
  &\propto \exp(-\half \vx_s^\trans \vLambda_s' \vx_s + \veta_s^\trans \vx_s) \\
  \veta_s &= \vJ_s^\trans \vLambda_s[\vJ_s \vx_0 + \vz_s - \vh_s(\vx_0)] \\
  \vLambda_s' &= \vJ_s^\trans \vLambda_s \vJ_s 
\end{align}



\begin{align}
  \psi_f(\vx_s) &= \exp\left(-\half (\vz_s - \vh_f(\vx_s))^\trans \vLambda_f^{-1}  (\vz_s - \vh_f(\vx_s)) \right)  \\
  &  \propto \gauss(\vz_s | \vh_f(\vx_s),  \vLambda_f^{-1}) 
    \propto \gauss^{-1}(\vx_s | \veta_f(\vx_s), \vLambda_f) \\
  \veta_f(\vx_s) &= \vLambda_f(\vz_s - \vh_f(\vx_s)) \\
  \gauss^{-1}(\vx|\veta,\vLambda) &\propto \exp(\vx^\trans \veta - \half \vx^\trans \vLambda \vx)
\end{align}


\begin{align}
  p(\vx_t^i | \vx_{t-1})
  &\propto \gauss(\vx_t^i | \vmu_{t|t-1}^i, \vLambda_d^{-1})
  \prod_j \gauss(\vzero | \vh_r^j(\vx_t^i), \vLambda_r^{-1}) \\
  \vmu_{t|t-1}^i &= \vPhi \vx_{t-1}^i \\
  \vh_r^j(\vx_t^i) &=   \vh_r(\vx_t^i, \vmu_{t|t-1}^j) 
\end{align}


\begin{align}
  \vh_r^j(\vx_t^i) &\approx
  \vh_r^j(\vmu_{t|t-1}^i)  + \vH_r^j(\vx_{t}^i - \vmu_{t|t-1}^i)
   = \vH_r^j \vx_{t}^i + \tilde{\vmu}_{t|t-1}^i \\
   \vH_r^j &= \jac(\vh_r^j)(\vmu_{t|t-1}^i) \\
   \tilde{\vmu}_{t|t-1}^i
   &= \vh_r^j(\vmu_{t|t-1}^i)  - \vH_r^j \vmu_{t|t-1}^i
\end{align}




\begin{align}
  p(\vx_t^i | \vx_{t-1})
  &\propto \gauss(\vx_t^i | \vPhi \vx_{t-1}^i, \vLambda_d^{-1})
  \prod_j \gauss(\vx_t^i | \vh_r^j(\vx_t^i), \vLambda_r^{-1}) \\
\end{align}


\begin{align}
  p(\vx_t^i | \vx_{t-1})
  &\propto \gauss(\vx_t^i | \vPhi \vx_{t-1}^i, \vLambda_d^{-1}
  \prod_j \gauss(\vx_t^i | \vh_r(\vx_t^i, f_d(\vx_{t-1}^j), \vLambda_r^{-1}) \\
  \vh_r(\vx_t^i; \overline) &\approx \vh_r(\vm) + \vH(\vu-\vm)
  \vm &= E[\vu] \\
  \vH &= \jac(\vh)(\vm) \\
  \gauss(\vx_t^i | \vh_r(\vx_t^i, f_d(\vx_{t-1}^j), \vLambda_r^{-1})
  &\approx \gauss(\vx_t^i | \vH\vh_r(f_d(\vx_{t-1}^i), f_d(\vx_{t-1}^j), \vLambda_r^{-1}) \\
\end{align}


\begin{align}
  \vh_r(\vu) &\approx \vh_r(\vm) + \vH(\vu-\vm)
  \vm &= E[\vu] \\
  \vH &= \jac(\vh)(\vm) \\
  \vu &=(\vx_t^i, f_d(\vx_{t-1}^i)) \\
  \vm &= xx
\end{align}






\begin{align}
  p(\vx_t | \vx_{t-1}) &= \prod_i p(\vx_t^i | \vx_{t-1}) \\
  p(\vx_t^i | \vx_{t-1})
  &\propto \psi_d(\vx_{t-1}^i, \vx_{t}^i)
  \prod_{j \in  N(t,i)} \psi_r(\vx_t^i, f_d(\vx_{t-1}^j)) \\
  \end{align}



\begin{align}
  \vx_t^i &= (x_t^{i}, y_t^{i}, \theta_t^i, \dot{x}_t^{i}, \dot{y}_t^{i}, \dot{\theta}_t) \\
  \vo_t^i &= (x_t^{i}, y_t^{i}, \theta_t^i) \\
  \vx_t^i &= (x_t^{i}, y_t^{i},  \dot{x}_t^{i}, \dot{y}_t^{i})\\
  \end{align}


\begin{align}
  p_i(o_t^i | \vo_{<t}, g^i, \vtheta) &=
  \sum_{\vo_t^{-i}} \sum_{\vo_{t+1:T}^{1:N}} \sum_{\vg^{-i}}
  p(\vo_{t:T}^{1:N} | \vg^{1:N}, \vo_{<t}, \vtheta_d)
  p(\vg^{1:N} | \vo_{<t}, g^i, \vtheta_g)
  \\
  p(\vo_{t:T}^{1:N} | \vg^{1:N}, \vo_{<t}, \vtheta_d)
  &=
  \left[ \prod_{\tau=t+1}^T   \prod_{i=1}^N
    p(o_t^i | \vo_{t-1}^{1:N}, \vtheta_d) \right]
  \left[ \prod_{\tau=t}^T \prod_{i \sim j} \psi(o_{\tau}^i, o_{\tau}^j) \right]
  \left[ \prod_{i=1}^N \psi(o_T^i, g^i) \right]
  \\
    p(\vg^{1:N} | \vo_{<t}, g^i, \vtheta_g)
  &=    \prod_{j\neq i} p(g^j | \vo_{<t}, \vtheta_g)
  \end{align}


\begin{align}
  Q(y|x) &= \sum_z Q(y,z|x) \text{ // Marginalize out nuisance variable} \\
  Q(y,z|x) &= \frac{Q(x|y,z) Q(y,z)}{Q(x)} \text{ // Bayes rule} \\
  &=  \frac{P(x|y,z) Q(y,z)}{Q(x)} \text{ // Causal stability assumption}  \\
  P(x|y,z) &= \frac{P(y,z|x) p(x)}{p(y,z)} \text{ // Discriminative likelihood trick} \\
  Q(y,z|x) &= \frac{P(y,z|x) \frac{Q(y,z)}{P(y,z)}}{\frac{Q(x)}{P(x)}}
  \text{ // Putting it altogether}
\end{align}


\begin{align}
  p(y_t|\vw_t,\vx_t) &= \Ber(y_t|h(\vw_t,\vx_t)) \\
  h(\vw_t,\vx_t) &= \sigmoid(f(\vw_t, \vx_t))
\end{align}


\begin{align}
  p(\vy_t|\vx_t,\vtheta_t) &= \frac{1}{Z(\veta_t)} \exp(\veta_t^\trans \vz(\vy_t) + b(\vy_t))
  = \exp(\veta_t^\trans \vz(\vy_t) + b(\vy_t) - a(\veta_t)) \\
  \vz(\vy_t) &= \text{sufficient statistics}(\vy_t) \\
  \veta_t &= f(\vx_t, \vtheta_t)     \text{//  natural parameters } \\
   \vlambda_t &= \expect{\vz(\vy) | \veta_t} = h(\vx_t,\vtheta_t)
    = \nabla_{\veta} a(\veta_t)   \text{//  dual (moment) parameters } \\
  \vR_t &= \cov{\vz(\vy) | \veta_t} = \nabla_{\veta} \vlambda(\veta_t)
  = \nabla^2_{\veta} a(\veta_t)   \text{//  observation noise}
\end{align}


\begin{align}
  \vz(y_t) &= [\ind{y_t=1}, \ldots, \ind{y_t=C}] \\
  \veta_t &= \text{ // logits } \\
  \vlambda_t &= \expect{\vz(y_t)|\veta_t} =
         [p(y_t=1|\veta_t), \ldots,
           p(y_t=C|\veta_{t}] = \softmax(\veta_t) \text{ // probabilities}\\
 \vR_t &= \diag(\vlambda_t) - \vlambda_t \vlambda_t^\trans
\end{align}



\begin{align}
  \hat{\vx} = g(\vc,\data)
  = \argmin_{\hat{\vx}}   \int p(\vx|\vc,\data) \lossfn(\vx, \hat{\vx} |\vc) d\vx
\end{align}


\begin{align}
  p(\vz_n|\vx_n,\hat{\vtheta}(\data)) &\propto p(\vx_n|\vz_n,\hat{\vtheta}) p(\vz_n|\hat{\vtheta})
\end{align}

\begin{align}
  p(\vtheta|\data) &= \frac{p(\data|\vtheta) p(\vtheta)}
    {\int p(\data|\vtheta') p(\vtheta') d\vtheta'}
\end{align}

\begin{align}
  CI &= \delta(\data)  = [P^{-1}_{0.025}(\theta|\data), P^{-1}_{0.0975}(\theta|\data)] 
\end{align}


\begin{align}
\hat{y} = \pi(\vx,\data) = \argmin_{\hat{y}}
 \sum_{y \in \calY} \int p(y|\vx, \vtheta) b(\vtheta)  \lossfn(y, \hat{y}) 
\end{align}


\begin{align}
\loss(\vtheta)   &= -[\log p(\data|\vtheta) + \log p(\vtheta)]
\end{align}


\begin{align}
  E_y\left[ \lossfn(y, \hat{y}) | \vx,\data \right]
  &= \sum_{y \in \calY} p(y|\vx, \data) \lossfn(y, \hat{y}) \\
  p(y|\vx, \data)
  &= \int p(y|\vx, \vtheta) p(\vtheta|\data) d\vtheta \\
  &\approx p(y|\vx, \hat{\vtheta}(\data))
\end{align}


\begin{align}
  E_h\left[ \lossfn(h, a) | o \right]
  &= \int p(h|o) \lossfn(h,a) dh
\end{align}

\begin{align}
\hat{\theta} = \delta(D)
\end{align}

\begin{align}
  \vx_{\alpha_{t+1}} &= (1-\alpha_{t+1}) \overline{\vx}_0 + \alpha_{t+1} \overline{\vx}_1 \\
  &= \overline{\vx}_0 + \alpha_{t+1} (\overline{\vx}_0 - \overline{\vx}_1) \\
  &= \vx_{\alpha_t} - \alpha_t (\overline{\vx}_0 - \overline{\vx}_1) 
  + \alpha_{t+1} (\overline{\vx}_0 - \overline{\vx}_1) \\
  &=\vx_{\alpha_t} + (\alpha_{t+1} - \alpha_t) (\overline{\vx}_0 - \overline{\vx}_1) 
  \end{align}


\begin{align}
  \nabla_{\vx_k} \log q_k(\vx_k|\vx_0)
  &=   \nabla_{\vx_k} \frac{-||\vx_k - \vx_0||^2}{2\sigma_k^2} 
  =    \frac{-(\vx_k - \vx_0)}{\sigma_k^2}
  \end{align}


\begin{align}
  \E_{\vx_k}\left[
    (s_{\vtheta}(k, \vx_k) - \nabla \vx_k \log q_k(\vx_k))^2 \right]
    &=
      \E_{\vx_0} \E_{\vx_k|\vx_0}\left[
    (s_{\vtheta}(k, \vx_k) - \nabla \vx_k \log q_k(\vx_k|\vx_0))^2 \right]
  \end{align}


\be
L(\vx_k) = \log \frac{q_k(\vx_k)}{q_{k+1}(\vx_{k+1})}
\ee

\be
\lossfn(h,a) = \lossfn(\vx, \hat{\vx}|c)  = ?
\ee

\be
\hat{\vx} = g(\vc; \data) =  \argmin_{\hat{\vx}}
 E_{\vx}[\lossfn(\vx, \hat{\vx}) |\vc, \data]
\ee


\be
\hat{\vtheta} = \delta(\data) =  \argmin_{\hat{\vtheta}}
\int p(\vtheta|\data) \|\vtheta - \hat{\vtheta}\|_2^2 d\vtheta
 = E[\vtheta|\data]
\ee

\be
\hat{\vtheta} = \delta(\data) = \argmin_{\hat{\vtheta}}
E_{\vtheta} \left[
   \|\vtheta - \hat{\vtheta}\|_2^2 \| \data \right]
\ee

\be
\hat{y} = f(\vx; \data) = \argmin_{\hat{y} \in \calY}
E_y\left[ \lossfn(y, \hat{y}) | \vx,\data \right]
\ee


\begin{align}
  \hat{a} = \pi(o)
  &= \argmin_{a \in \calA}
E_h\left[ \lossfn(h, a) | o \right] \\
  &= \argmax_{a \in \calA}
E_h\left[ U(h, a) | o \right]
\end{align}



\be
p(y_t|\vw_t, \vx_t) = \Cat(y_t|\softmax(h(\vx_t,\vw_t)))
\ee


\be
\hat{y} =  \argmin_{\hat{y}=f(\vx;\data)}
E_y\left[ \lossfn(y, \hat{y}) | \vx,\data \right]
\ee

\be
\hat{\vx} \sim g(\vc,\data)
\ee


\begin{eqnarray}
  a_{1:T} = \argmin_{a_t = \policy(o_{1:t}, a_{1:t-1})}
  \sum_{t=1}^T
  \expectQ{\lossfn(s_t, a_t) | o_{1:t}, a_{1:t-1}}{s_t}
    \end{eqnarray}

\begin{eqnarray}
  a_{1:T} = \argmin_{a_{1:T}} \sum_{t=1}^T
  \expectQ{\lossfn(s_t, a_t) | o_{1:t}, a_{1:t-1}}{s_t}
    \end{eqnarray}

\be
\hat{y} =  \argmin_{\hat{y} \in \calY}
E_y\left[ \lossfn(y, \hat{y}) | \vx,\data \right]
\ee

\be
\hat{y} =  \argmin_{\hat{y}=f(\vx;\data) \in \calY}
E_y\left[ \lossfn(y, \hat{y}) | \vx,\data \right]
\ee


\begin{align}
  p(\vw_t | \vw_{t-1}, \vx_t) &= \gauss(\vw_t | f(\vw_{t-1}, \vx_t), q \vI) \\
  p(y_t | \vw_t, \vx_t) &= \gauss(y_t | h(\vw_t, \vx_t), \sigma^2) \\
  \end{align}

\begin{align}
  p(y_t | \vx_t, \data_{1:t-1}) &=
  \int p(y_t | \vw_t, \vx_t)
  p(\vw_t|\vx_t, \data_{1:t-1}) d\vw_t
\text{ // Predict }
  \\
   p(\vw_t | \vx_t, y_t, \data_{1:t-1}) &\propto
   p(y_t | \vw_t, \vx_t)    p(\vw_t|\vx_t, \data_{1:t-1}) 
   \text{ // Update }
  \end{align}

\be
\loss(\pi) = \expectQ{\sum_{t=1}^T r_t}{\pi}
\ee


\begin{align}
  \text{CATE}(a|\vx) &=      \expect{y|a=1,\vx} - \expect{y|a=0,\vx} \\
  p(y|a,\vx) &= N(y | \vw_a^\trans \vx, \sigma^2) \\
  \text{CATE}(a|\vx) &= \expect{\vw_1 - \vw_0|\data}
\end{align}


\begin{align}
  \mu_t(a) &= \expect{R(s_t, a) | \data_{1:t-1}} \\
  \sigma_t(a) &= \sqrt{\var{R(s_t, a) | \data_{1:t-1}}} \\
 \pi_t(a^*) &= \ind{a^* = \argmax_{a'} \mu_t(a') + c \sigma_t(a')}
    \end{align}


\begin{align}
  \pi_t(a^*) &= p(a^* = \argmax_{a'} E[R(s_t, a')] | \data_{1:t-1}) \\
  &= \int \ind{a^* = \argmax_{a'} f(s_t, a'; \vtheta)} p(\vtheta|\data_{1:t-1}) d\vtheta \\
  &\approx \ind{a^* = \argmax_{a'} f(s_t, a'; \tilde{\vtheta}_t)} \\
  & \text{ where } \tilde{\vtheta}_t \sim p(\vtheta|\data_{1:t-1})
    \end{align}


\begin{align}
  \pi_t(a^*) &= p(a^* = \argmax_{a'} R(a', x_t) | \data_{1:t-1}) \\
  &= \int \ind{a^* = \argmax_{a'} R(a', x_t; \vtheta)} p(\vtheta|\data_{1:t-1}) d\vtheta \\
  &\approx \ind{a^* = \argmax_{a'} R(a', x_t; \tilde{\vtheta}_t)} \\
  & \text{ where } \tilde{\vtheta}_t \sim p(\vtheta|\data_{1:t-1})
    \end{align}


\begin{align}
  q_t(y,z) &\defeq q_t(m) = \pi_m \\
  q_t^j(m|\vx_n) &\propto q_t(\vx_n|m) \pi_m^j \propto
  \frac{p_s(m|\vx_n)}{p_s(m)} \pi_m^j \text{ // E step } \\
  \pi_m^{j+1} &= \frac{1}{N} \sum_{n=1}^N q_t^j(m|\vx_n)
  \text{ // M step}
    \end{align}



\begin{align}
  p_s(\vx,y,z | \vrho, \vtheta) &= p_s(y,z|\vrho) p_s(\vx|y,z, \vtheta) \\
  q_t(\vx,y,z | \vpi, \vtheta) &= q_t(y,z|\vpi) p_s(\vx|y,z, \vtheta) \\
  \vpi &= \argmax_{\vpi}   \sum_{n=1}^N \log q_t(\vx_n|\vpi, \vtheta) \\
  q_t(\vx_n|\vpi) &= \sum_{y,z} p_s(\vx_n|y,z, \vtheta) q_t(y,z|\vpi)
  = \sum_{y,z} \frac{p_s(y,z|\vx_n,\vtheta)}{p_s(y,z|\vrho)} q_t(y,z|\vpi) + \text{const} \\
    \end{align}

\begin{eqnarray}
  \lefteqn{
    \text{Let us make a point estimate approximation to parameter posterior}
  }\\
  \lefteqn{
    p(\vtheta|\data) \approx \delta(\vtheta-\hat{\vtheta})
  }  \\
  \lefteqn{
    \text{where $\hat{\vtheta}$ is the MAP estimate}
    }\\
  \lefteqn{
    \hat{\vtheta} = \argmax_{\vtheta} p(\vtheta|\data)
    = \argmax_{\vtheta} \log p(\data|\vtheta) + \log p(\vtheta)
    }
    \\
    \lefteqn{   \text{or some other (regularized) empirical risk minimizing (ERM) estimate }
    }  \\
    \lefteqn{
    \hat{\vtheta} 
    = \argmin_{\vtheta} \loss(\vtheta,\data)  + \lambda R(\vtheta)
    }\\
    \lefteqn{
      \text{This gives a plugin approximation to the posterior predictive distribution}
  } \\
  \lefteqn{
    p(\vy|\vx,\data) \approx \int p(\vy|\vx,\vtheta) \delta(\vtheta-\hat{\vtheta})) d\vtheta
    = p(\vy|\vx,\hat{\vtheta})
  }
    \end{eqnarray}


\begin{align*}
  \hat{y} &= \argmax_y p(y|x,\data) = \text{mode}(p(y|x,\data)  \text{// Classification} \\
  \hat{y} &= \int y p(y|x,\data) dy = \text{mean}(p(y|x,\data)  \text{// Regression}
  \end{align*}

\begin{align*}
  \hat{y}
  &= \argmin_{\hat{y} \in \calY}
  E_y\left[ \lossfn_{2}(y, \hat{y}) | x, \data \right] \\
  &= \argmin_{\hat{y} \in \calY}
  \int p(y|x, \data) (y- \hat{y})^2 dy \\
  &= E[y | x, \data]
  \end{align*}



\be
\lossfn_{\text{perceptual}}(\vx, \hat{\vx}) = ??
\ee


\be
\lossfn_{2}(\vtheta, \hat{\vtheta}) = \|\vtheta - \hat{\vtheta}|_2^2
\ee


\be
\lossfn_{2}(z, \hat{z}) = \|z - \hat{z}|_2^2
\ee


\be
\lossfn_{2}(y, \hat{y}) = (y - \hat{y})^2
\ee

\be
\lossfn_{01}(y, \hat{y}) =
\left\{ \begin{array}{cc}
  1 & \mbox{if $y \neq \hat{y}$}\\
  0 & \mbox{otherwise}
\end{array}
\right.
\ee


\begin{eqnarray}
  \hat{\vtheta} = \argmax_{\vtheta} \sum_n \log p(\vx_n|\vtheta) \\
  p(\vx_n|\vtheta) = \int p(\vx_n|\vz_n, \vtheta) p(\vz_n|\vtheta) d\vz_n
\end{eqnarray}

\begin{eqnarray}
\lefteqn{
  \text{Observations: } \vx^* \union \data,
  \data = \{ (\vx_n, \vy_n): \vx_n \in \calX, \vy_n \in \calY \}
} \\
\lefteqn{
  \text{Action: }  \hat{\vy}^{1:S} \in \calY
}\\
\lefteqn{
  \text{Hidden state: } \vy^* \in \calY
}\\
\lefteqn{
  \text{Loss: }  \lossfn(\vy^*, \hat{\vy}^{1:S}) = ? \text { learn from human preferences? }
}\\
\end{eqnarray}



\begin{eqnarray}
\lefteqn{
  \text{Observations: } \data = \{ \vx_n: \vx_n \in \calX \}
} \\
\lefteqn{
  \text{Action: }  \hat{\vz}_n \in \real^H
}\\
\lefteqn{
  \text{Hidden state: } \vz_n \in \real^H
}\\
\lefteqn{
  \text{L2 loss: }  \lossfn_{2}(\vz_n^*, \hat{\vz}_n) = ||\vz_n^* - \hat{\vz}_n||_2^2
}\\
\lefteqn{
  \text{Optimal estimator: }
  \pi^*(\vx_n,\data) =  \expect{\vz_n|\vx_n,\data} 
}\\
\lefteqn{
  \text{State posterior }
  p(\vz_n|\vx_n,\data) \approx  p(\vz_n|\vx_n,\hat{\vtheta}) 
}\\
\lefteqn{
  \text{Maximize log marginal likelihood }
}
\end{eqnarray}






\begin{eqnarray}
\lefteqn{
  \text{Observations: } \vx \in \calX = \real^D
} \\
\lefteqn{
  \text{Action: }  \hat{\vz} \in \calZ = \real^H
}\\
\lefteqn{
  \text{Hidden state: } \vz^* \in \calZ = \real^H
}\\
\lefteqn{
  \text{L2 loss: }  \lossfn_{2}(\vz^*, \hat{\vz}) = ||\vz^* - \hat{\vz}||_2^2
}\\
\lefteqn{
  \text{Optimal estimator: Posterior mean }
\pi^*(\vx, \data) =  \expect{\vz^*|\vx, \data}
}\\
\end{eqnarray}


\begin{eqnarray}
J  \defeq
 \sum_{t=1}^T \expectQ{r_t}{p(s_t|s_{t-1},a_t) p(r_t|s_t,a_t)   p(x_t|s_t)   
  \policy_t(a_t|b_t)   \delta(b_t - U(b_{t-1}, x_t, a_t)) }
    \end{eqnarray}


\begin{eqnarray}
  \lefteqn{
    \text{Optimal action: }  \pi^*(\vx,\data) = F(p(\vy|\vx,\data)), \text{ where $F$=mode, mean, median }
    }\\
  \lefteqn{
    \text{Learn the posterior predictive distribution with a parametric model}
  }\\
  \lefteqn{
    p(\vy|\vx,\data) = \int p(\vy|\vx,\vtheta) p(\vtheta|\data) d\vtheta
  }  \\
  \lefteqn{   \text{where $\data=\{(\vx_n,\vy_n):n=1:N\}$  is training data and $\vtheta$ are parameters.}
  }  \\
  \lefteqn{  \text{Plugin approximation to the integral}
  } \\
  \lefteqn{
    p(\vy|\vx,\data) \approx \int p(\vy|\vx,\vtheta) \delta(\vtheta-\hat{\vtheta})) d\vtheta
    = p(\vy|\vx,\hat{\vtheta})
  } \\
    \lefteqn{   \text{where $\hat{\vtheta}$ is the MAP estimate}
    }  \\
    \lefteqn{
    \hat{\vtheta} = \argmax_{\vtheta} p(\vtheta|\data)
    = \argmax_{\vtheta} \log p(\data|\vtheta) + \log p(\vtheta)
    }\\
    \lefteqn{   \text{or some other (regularized) empirical risk minimizing (ERM) estimate }
    }  \\
    \lefteqn{
    \hat{\vtheta} 
    = \argmin_{\vtheta} \loss(\vtheta,\data)  + \lambda R(\vtheta)
    }\\
    \end{eqnarray}




\begin{eqnarray}
\lefteqn{
  \text{Observations: } \vx \in \calX = \real^D
} \\
\lefteqn{
  \text{Action: }  \hat{\vy} \in \calY = \real^C
}\\
\lefteqn{
  \text{Hidden state: } \vy^* \in \calY = \real^C
}\\
\lefteqn{
  \text{L2 loss: }  \lossfn_{2}(\vy^*, \hat{\vy}) = ||\vy^* - \hat{\vy}||_2^2
}\\
\lefteqn{
  \text{Optimal estimator: Posterior mean }
\pi^*(\vx) =  \expect{\vy^*|\vx}
}\\
\lefteqn{
  \text{L1 loss: }  \lossfn_{1}(\vy^*, \hat{\vy}) = ||\vy^* - \hat{\vy}||_1
}\\
\lefteqn{
  \text{Optimal estimator: Posterior median } \pi^*(\vx) =  \text{median}[{\vy^*|\vx}]
}\\
\end{eqnarray}


\begin{eqnarray}
\lefteqn{
  \text{Observations: } \vx \in \calX = \real^D
} \\
\lefteqn{
  \text{Action: }  \hat{a} \in \calY \union R = \{R, 1, \ldots, C\}
}\\
\lefteqn{
  \text{Hidden state: } y^* \in \calY 
}\\
\end{eqnarray}

\begin{eqnarray}
\lefteqn{
  \text{Observations: } \vx \in \calX = \real^D
} \\
\lefteqn{
  \text{Action: }  \hat{y} \in \calY = \{1, \ldots, C\}
}\\
\lefteqn{
  \text{Hidden state: } y^* \in \calY 
}\\
\lefteqn{
  \text{0-1 loss: }  \lossfn_{01}(y^*, \hat{y}) = \ind{y^* \neq \hat{y}}
}\\
\lefteqn{
  \text{Optimal estimator: MAP label }
}\\
\lefteqn{
\pi^*(\vx) = \argmax_{y \in \calY} p(y|\vx)
}\\
\end{eqnarray}


\begin{eqnarray}
  \lefteqn{
    \text{Given observations }  \vx \in \calX, \text{ choose action } \va \in \calA
    }\\
  \lefteqn{
    \text{to minimize posterior expected loss}  
  }\\
  \\
  \lefteqn{
    \rho(\va|\vx) = \expectQ{\lossfn(\vh, \va)}{p(\vh|\vx)}
  =\int p(\vh|\vx) \lossfn(\vh,\va) d\vh
  }  \\
  \\
  \lefteqn{   \text{where } \vh \in \calH \text{ is the hidden state of nature}.
  }  \\
  \lefteqn{  \text{Thus the  optimal estimator (policy) is given by } 
  } \\
  \\
  \lefteqn{ \pi^*(\vx) = \argmin_{\va \in \calA} \expectQ{\lossfn(\vh, \va)}{p(\vh|\vx)}
  } \\
  \\
    \lefteqn{
      \text{Equivalently we can maximize expected utility}
    }\\
    \\
      \lefteqn{
        \pi^*(\vx) = \argmax_{\va \in \calA} \expectQ{U(\vh, \va)}{p(\vh|\vx)}
      }\\
      \\
        \lefteqn{
          \text{where } U(\vh,\va) = -\lossfn(\vh,\va)
          } \\
    \end{eqnarray}



\begin{align}
 \delta^*(\vx) = \argmax_{a \in A} \expectQ{U(h,a)}{p(h|\vx)}
    \end{align}


\begin{align}
  p(\vtheta|\data) \propto p(\vtheta) \prod_{n=1}^N p(\vy_n|\vx_n,\vtheta)
    \end{align}


\begin{align}
  p(\vz_n | \vy_n, \data) \propto  p(\vy_n|\vz_n,\data) p(\vz_n|\data)
  \approx p(\vy_n|\vz_n,\hat{\vtheta}) p(\vz_n|\hat{\vtheta})
    \end{align}


\begin{align}
f(\vx) = \sigma(\vW \vx) \rightarrow \sigma(\vW(\vx) \vx)
    \end{align}


\begin{align}
  p_t(y,z) &= p_t(m)= \pi_m  \\
  \vpi &= \argmax_{\vpi \in \calS_m }  \ell(\vpi|\data_t)  \\
  \ell(\vpi|\data_t) &= \sum_{n=1}^N \log p_t(\vx_n|\vpi) \\
  p_t(\vx_n|\vpi) &= \sum_{m=1}^M p_t(\vx_n|m) \pi_m 
  = \sum_{m=1}^M p_s(\vx_n|m) \pi_m 
  = \sum_{m=1}^M \frac{p_s(m|\vx_n)}{p_s(m)} \pi_m + \text{const} \\
    \end{align}



\begin{align}
  \vtheta_t &= \vA \vz_t + \vtheta_* \\
  p(y_t|s_t,a_t) &= \gauss(y_t | f(s_t, a_t; \vtheta_t), \sigma^2)
\end{align}


\begin{align}
  p_t(y,z|\vx) &\propto p_t(\vx|y,z) p_t(y,z) \\
  &= p_s(\vx|y,z) p_t(y,z) \\
  &= \frac{p_s(y,z|\vx)}{p_s(y,z)} p_t(y,z)
\end{align}

\begin{align}
  p_t(y,z) &= p_t(m)= \pi_m  \\
  \ell(\vpi|\data_t) &= \sum_{n=1}^N \log p_t(\vx_n|\vpi) \\
  p_t(\vx_n|\vpi) &= \sum_{m=1}^M \pi_m \frac{p_s(m|\vx)}{p_s(m)} + \const \\
  \vpi &= \argmax_{\vpi \in \calS_m }  \ell(\vpi|\data_t) 
    \end{align}

\begin{align}
  p(\vtheta_t|\data_{1:t}) &= \gauss(\vtheta_t|\vmu_t, \vSigma_t) \\
  \vSigma_t^{-1} &= \diag(\Upsilon_t) + \vW_t \vW_t^\trans
\end{align}


\begin{align}
  p_t(y,z) &= p_t(m)= \pi_m  \\
  \ell(\vpi|\data_t) &= \sum_{n=1}^N \log p_t(\vx_n|\vpi) \\
  p_t(\vx_n|\vpi) &= \sum_{m=1}^M p_t(\vx_n|m) \pi_m \\
  &= \sum_{m=1}^M p_s(\vx_n|m) \pi_m \\
  &= \sum_{m=1}^M \frac{p_s(m|\vx_n)}{p_s(m)} \pi_m + \text{const} \\
  \vpi &= \argmax_{\vpi \in \calS_m }  \ell(\vpi|\data_t) 
    \end{align}



\begin{align}
  p_t(y,z|\vx) &=  \frac{p_s(\vx|y,z) p_s(y,z)}{p_s(\vx)} \\
  & \implies \\
 p_s(\vx|y,z) &\propto \frac{p_s(y,z|\vx)}{p_s(y,z)}
\end{align}


\begin{align}
  p_t(y,z|\vx) &\propto p_t(\vx|y,z) p_t(y,z) \\
  &= p_s(\vx|y,z) p_t(y,z)
\end{align}

\begin{align}
  \text{standard test error} &= \expectQ{\lossfn(\vy, f_s(\vx;\vtheta_s}{p_t(\vx,\vy)},
  \vtheta_s = \calE(\data_s^{xy}) \\
  \text{our test error} &= \expectQ{\lossfn(\vy, f_t(\vx;\vtheta_t)}{p_t(\vx,\vy)} ,
  \vtheta_t = \calA(\vtheta_s,  \data_t^{x})
  \end{align}


\begin{align}
  \text{standard test error} &= \expectQ{\lossfn(\vy, f_s(\vx;\vtheta(\data_s^{xy})}{p_t(\vx,\vy)} \\
  \text{our test error} &= \expectQ{\lossfn(\vy, f_t(\vx;\vtheta(\data_s^{xyz}, \data_t^{x}))}{p_t(\vx,\vy)} 
  \end{align}

\begin{align}
  p_t(\vx,\vy|z)  =  p_s(\vx,\vy|z) 
\end{align}


\begin{align}
\min_{\vtheta} \max_{z}  \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{p_s(\vx,\vy|z)} 
\end{align}

\begin{align}
  p_t(\vx,\vy) = \sum_z p_t(\vx,\vy|z) p_t(z) = \sum_z p_s(\vx,\vy|z) p_t(z)
\end{align}


\begin{align}
p(y_t|\vw_t,\vx_t) &= \gauss(y_t | h(\vx_t;\vw_t), \sigma^2)
\end{align}


\begin{align}
  p(r|s,a) &= \gauss(r|f(s,a), \sigma^2) \\
  p(f|\data_{1:t}) &= \text{GP}(f | \data_{1:T}, \kernelfn) \\
\end{align}


\begin{align}
a_t^* = \argmax_{a \in \calA} R(s_t, a)
\end{align}


\begin{align}
a^* = \argmax_{a \in \real^D} R_S(a)
\end{align}
 

\begin{align}
s_t \sim \color{red}{p(\cdot)} & \text{  state (context) distribution}\\
r_t \sim p(\cdot | s_t, a_t) & \text{  reward function} \\
b_t = U(b_{t-1}, {\color{red}{s_t}}, a_{t-1}, r_{t-1}) & \text{  belief update function} \\
a_t \sim \pi(b_t) & \text{  policy} 
\end{align}


\begin{align}
\min_{\vtheta} \max_{p_t \in \calP}  \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{p_t(\vx,\vy)} 
\end{align}

\begin{align}
    \loss_i(\vtheta) &= \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{p_i(\vx,\vy)} 
\end{align}


\begin{align}
\min_{\vtheta} \max_{m \in \calZ \times \calY}  \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{p_m(\vx,\vy)} 
\end{align}


\begin{align}
\estimator(\vx) &=
\begin{cases}
\argmax_c p(y=c|\vx) & \text{ if } \max_c p(y=c|\vx) > 1-\frac{\lambda_r}{\lambda_e} \\
\text{reject} & \text{otherwise}
\end{cases}
\end{align}

\be
\lossfn(y^*, a) =
\left\{ \begin{array}{cc}
  \lambda_r & \mbox{if $a=$ reject}\\
  0 & \mbox{if $y^*=a$ and $a \not = $ reject}\\
  \lambda_e & \mbox{if $y^* \neq a$ and $a \not = $ reject}
\end{array}
\right.
\ee



\begin{align}
s_t \sim p(\cdot | s_{t-1}, a_{t-1}) & \text{  state transition function}\\
x_t \sim p(\cdot | x_{t-1}) & \text{  observation function} \\
r_t \sim p(\cdot | s_t, a_t) & \text{  reward function} \\
b_t = U(b_{t-1}, x_t, a_{t-1}, r_{t-1}) & \text{  belief update function} \\
a_t \sim \pi(b_t) & \text{  policy} 
\end{align}

\begin{align}
  p_t(y,z) &= p_t(m)= \pi_m  \\
  \ell(\vpi|\data_t) &= \sum_{n=1}^N \log p_t(\vx_n|\vpi) \\
  p_t(\vx_n|\vpi) &= \sum_{m=1}^M \pi_m \frac{p_s(m|\vx)}{p_s(m)} + \const \\
  \vpi &= \argmax_{\vpi \in \calS_m }  \ell(\vpi|\data_t) 
    \end{align}

\begin{align}
  \max_{\vpi}  \sum_n \log\left[ \sum_m \pi_m \frac{p_s(m|\vx)}{p_s(m)} \right]
  \text{ s.t. } \sum_m \pi_m = 1, \; \pi_m \geq 0
\end{align}


\begin{align}
  \loss(\vtheta) &= \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{\pemp(\vx,\vy)}\\
  &= -\frac{1}{N} \sum_{n=1}^N \log p(\vy_n|\vx_n;\vtheta)  \\
  &\approx -\frac{1}{B} \sum_{b \in \calB}  \log p(\vy_b|\vx_b;\vtheta) \\
\end{align}



\begin{align}
  \data &= \{ (\vx_n, \vy_n) \sim p_s \} \\
  \loss_{D}(\vtheta) &= \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{p_D(\vx,\vy)} \\
  \loss_{s}(\vtheta) &= \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{p_s(\vx,\vy)} \\
  \loss_{D}(\vtheta) &\rightarrow \loss_{s}(\vtheta) \\
  \loss_{t}(\vtheta) &= \expectQ{\lossfn(\vy, f(\vx;\vtheta)}{p_t(\vx,\vy)},
     p_t \neq p_s
  \\
  \end{align}

\be
\pemp(\vx,\vy) = \frac{1}{N} \sum_{n=1}^N \delta(\vx-\vx_n, \vy-\vy_n)
\ee



\begin{align}
  p_i(u,y,z,x) &=    p_i(u) p(y,z|u) p(x|y,z)  \\
  p_s(x) &= \sum_{y,z} p_s(y,z) p(x|y,z) \\
    p_t(x) &= \sum_{y,z} p_t(y,z) p(x|y,z) 
\end{align}

\begin{align}
  \vy &= f_2(f_1(\vx)) \\
  \vh^1 & = f_1(\vx) = \activation(\vW_1 \vx + \vb_1) \\
    \vh^2 &= f_2(\vh^1) = \activation_2(\vW_2 \vh^1 + \vb_2)
\end{align}

\begin{align}
  y &= f_2(f_1(\vx))  = \vw_2^\trans f_1(\vx) + b_1 \\
  \vh & = f_1(\vx) = \activation(\vW_1 \vx + \vb_1) \\
  y &= f_2(\vh) = \vW_2 \vh + b_2
\end{align}