Extrap: write-up of quasi-Newton connection

Research/Extrapolation methods/notes.tex

@@ -37,6 +37,8 @@
 \newtheorem{defn}{Definition}
 \newtheorem{example}{Example}
 
+\let\vec\mathbf
+
 \begin{document}
 
 \section{Meeting notes}
@@ -261,4 +263,98 @@ By Equation (\ref{eq:gamma 2}) we have that $\langle v, r(T_n^{(k)}) \rangle = 0
 Therefore, MPE is an orthogonal projection method.
 In fact, it is identical in methodology to the Arnoldi process.
 
+\section{MPE as a quasi-Newton method}
+
+\subsection{Quasi-Newton methods}
+
+\newcommand{\fxi}[1]{\vec{f}(\vec{x}_{#1})}
+\newcommand{\Jxn}{J(\vec{x}_n)}
+
+A quasi-Newton method is any method of the form
+\begin{equation} \label{eq:quasiNewton}
+\hat{\vec{x}}_{n+1} = \vec{x}_n - \vec{u}_n
+\end{equation}
+where $\vec{u}_n$ is an approximate solution to the equation
+\begin{equation} \label{eq:Newtondirection}
+J(\vec{x}_n)  \vec{u} = \vec{f}(\vec{x}_n)
+\end{equation}
+where $J(\vec{x})$ is the Jacobain of $\vec{f}(\vec{x})$ evaluated at $\vec{x}_n$.
+
+One can expand the function $\vec{f}(\vec{x})$ into a Taylor series about $\vec{x}_n$:
+\begin{equation*}
+\fxi{n+i} = \vec{f}(\vec{x}_n) + J(\vec{x}_n) ( \vec{x}_{n+i} - \vec{x}_n ) + \frac{1}{2} ( \vec{x}_{n+i} - \vec{x}_n ) H(\vec{x}_n) ( \vec{x}_{n+i} - \vec{x}_n ) + \dots
+\end{equation*}
+As a first order approximation we can take the first two terms of this series, resulting in the following approximate equation:
+\begin{equation*}
+\fxi{n+i} - \fxi{n} \approx \Jxn ( \vec{x}_{n+i} - \vec{x}_n ).
+\end{equation*}
+Such a system can be solved for $\Jxn$ though such a system would be underdetermined.
+However, if one had as many $\fxi{n+i}$ as there are dimensions in the space then one could solve
+\begin{equation}
+\begin{bmatrix} \fxi{n+1} & \dots & \fxi{n+d} \end{bmatrix} - \fxi{n} = \hat{J} \left ( \begin{bmatrix}
+\vec{x}_{n+1} & \dots & \vec{x}_{n+d} \end{bmatrix} - \vec{x}_n \right )
+\end{equation}
+which is nonsingular given sufficient conditions on the choice of $\vec{x}_{n+i}$.
+
+Let $X = \begin{bmatrix} \vec{x}_{n+1} & \dots & \vec{x}_{n+d} \end{bmatrix} - \vec{x}_n$ and $F = \begin{bmatrix} \fxi{n+1} & \dots & \fxi{n+d} \end{bmatrix} - \fxi{n}$, then
+$$\hat{J}^{-1} = X F^{-1}.$$
+Combining this with equations (\ref{eq:quasiNewton}) and (\ref{eq:Newtondirection}) gives the quasi-Newton method
+\begin{equation}
+\hat{\vec{x}}_{n+1} = \vec{x}_n - X F^{-1} \fxi{n}.
+\end{equation}
+
+The vector $F^{-1} \fxi{n}$ may be found elementwise by Cramer's rule:
+\begin{align*}
+\left (F^{-1} \fxi{n} \right)_i = & \frac{ \vmat{
+\fxi{n+1} - \fxi{n} & \dots & \fxi{n+i-1} - \fxi{n} & \fxi{n} & \fxi{n+i+1}-\fxi{n} & \dots & \fxi{n+d}-\fxi{n}
+}
+}{ \vmat{
+\fxi{n+1}-\fxi{n} & \dots & \fxi{n+d}-\fxi{n}
+}
+} \\
+= & (-1)^i \frac{ \vmat{
+\fxi{n} & \dots & \fxi{n+i-1} & \fxi{n+i+1} & \fxi{n+d}
+}
+}{ \vmat{
+1 & \dots & 1 \\ \fxi{n} & \dots & \fxi{n+d}
+}}.
+\end{align*}
+The quasi-Newton method defined above may then be expressed as
+\begin{align*}
+\hat{\vec{x}}_{n+1} = & \vec{x}_n - \frac{\vmat{
+0 & \vec{x}_{n+1} - \vec{x}_n & \dots & \vec{x}_{n+d} - \vec{x}_n \\
+\fxi{n} & \fxi{n+1} & \dots & \fxi{n+d} }
+}{ \vmat{
+1 & \dots & 1 \\ \fxi{n} & \dots & \fxi{n+d} }} \\
+= & \frac{ \vmat{
+\vec{x}_n & \dots & \vec{x}_{n+d} \\ \fxi{n} & \dots & \fxi{n+d} }
+}{ \vmat{
+1 & \dots & 1 \\ \fxi{n} & \dots & \fxi{n+d} }}
+\end{align*}
+where one must expand the determinant along the top row to maintain the correct dimensions.
+
+Suppose, for whatever reason, that we do not have enough values of $\fxi{n+i}$ to fully determine $\hat{J}$.
+That is, suppose $F$ and $X$ have $d$ rows but only $k$ columns, and denote these submatrices by $F_k$ and $X_k$.
+Rather than solve $\hat{J} \vec{u}_n = \fxi{n}$ we can make a further approximation by solving $A^\top \hat{J} \vec{u}_n = A^\top \fxi{n}$ for some matrix $A$ with the same dimension as $F_k$.
+This has as its solution $\vec{u}_n = X_k (A^\top F_k)^{-1} A^\top \fxi{n}$.
+It is clear that the quasi-Newton method that results from this may be written as
+\begin{equation} \label{eq:uqn}
+\hat{\vec{x}}_{n+1} = \frac{ \vmat{
+\vec{x}_n & \dots & \vec{x}_{n+k} \\
+\vec{v}_1^\top \fxi{n} & \dots & \vec{v}_1^\top \fxi{n+k} \\
+\vdots & & \vdots \\
+\vec{v}_k^\top \fxi{n} & \dots & \vec{v}_k^\top \fxi{n+k} }}{ \vmat{
+1 & \dots & 1 \\
+\vec{v}_1^\top \fxi{n} & \dots & \vec{v}_1^\top \fxi{n+k} \\
+\vdots & & \vdots \\
+\vec{v}_k^\top \fxi{n} & \dots & \vec{v}_k^\top \fxi{n+k} }}
+\end{equation}
+where $\vec{v}_i$ is the $i$--th column of $A$.
+
+\subsection{Connection to extrapolation methods}
+
+Let $\vec{r}(\vec{x}_i) = \vec{x}_{i+1} - \vec{x}_i$ be the residual function.
+Then MPE may be expressed as equation (\ref{eq:uqn}) with $\fxi{i} = \vec{r}(\vec{x}_i)$ and $\vec{v}_i = \vec{r}(\vec{x}_{n+i-1})$.
+RRE and MMPE may also be expressed as such, using $\vec{v}_i = \vec{r}(\vec{x}_{n+i}) - \vec{r}(\vec{x}_{n+i-1})$ and $\vec{v}_i$ some fixed vector independent of $\vec{r}(\vec{x})$, respectively.
+
 \end{document}
\ No newline at end of file