Commit 41c59b00 authored by Conor McCoid's avatar Conor McCoid
Browse files

Extrap: write-up of quasi-Newton connection

parent 015eb174
......@@ -37,6 +37,8 @@
\newtheorem{defn}{Definition}
\newtheorem{example}{Example}
\let\vec\mathbf
\begin{document}
\section{Meeting notes}
......@@ -261,4 +263,98 @@ By Equation (\ref{eq:gamma 2}) we have that $\langle v, r(T_n^{(k)}) \rangle = 0
Therefore, MPE is an orthogonal projection method.
In fact, it is identical in methodology to the Arnoldi process.
\section{MPE as a quasi-Newton method}
\subsection{Quasi-Newton methods}
\newcommand{\fxi}[1]{\vec{f}(\vec{x}_{#1})}
\newcommand{\Jxn}{J(\vec{x}_n)}
A quasi-Newton method is any method of the form
\begin{equation} \label{eq:quasiNewton}
\hat{\vec{x}}_{n+1} = \vec{x}_n - \vec{u}_n
\end{equation}
where $\vec{u}_n$ is an approximate solution to the equation
\begin{equation} \label{eq:Newtondirection}
J(\vec{x}_n) \vec{u} = \vec{f}(\vec{x}_n)
\end{equation}
where $J(\vec{x})$ is the Jacobain of $\vec{f}(\vec{x})$ evaluated at $\vec{x}_n$.
One can expand the function $\vec{f}(\vec{x})$ into a Taylor series about $\vec{x}_n$:
\begin{equation*}
\fxi{n+i} = \vec{f}(\vec{x}_n) + J(\vec{x}_n) ( \vec{x}_{n+i} - \vec{x}_n ) + \frac{1}{2} ( \vec{x}_{n+i} - \vec{x}_n ) H(\vec{x}_n) ( \vec{x}_{n+i} - \vec{x}_n ) + \dots
\end{equation*}
As a first order approximation we can take the first two terms of this series, resulting in the following approximate equation:
\begin{equation*}
\fxi{n+i} - \fxi{n} \approx \Jxn ( \vec{x}_{n+i} - \vec{x}_n ).
\end{equation*}
Such a system can be solved for $\Jxn$ though such a system would be underdetermined.
However, if one had as many $\fxi{n+i}$ as there are dimensions in the space then one could solve
\begin{equation}
\begin{bmatrix} \fxi{n+1} & \dots & \fxi{n+d} \end{bmatrix} - \fxi{n} = \hat{J} \left ( \begin{bmatrix}
\vec{x}_{n+1} & \dots & \vec{x}_{n+d} \end{bmatrix} - \vec{x}_n \right )
\end{equation}
which is nonsingular given sufficient conditions on the choice of $\vec{x}_{n+i}$.
Let $X = \begin{bmatrix} \vec{x}_{n+1} & \dots & \vec{x}_{n+d} \end{bmatrix} - \vec{x}_n$ and $F = \begin{bmatrix} \fxi{n+1} & \dots & \fxi{n+d} \end{bmatrix} - \fxi{n}$, then
$$\hat{J}^{-1} = X F^{-1}.$$
Combining this with equations (\ref{eq:quasiNewton}) and (\ref{eq:Newtondirection}) gives the quasi-Newton method
\begin{equation}
\hat{\vec{x}}_{n+1} = \vec{x}_n - X F^{-1} \fxi{n}.
\end{equation}
The vector $F^{-1} \fxi{n}$ may be found elementwise by Cramer's rule:
\begin{align*}
\left (F^{-1} \fxi{n} \right)_i = & \frac{ \vmat{
\fxi{n+1} - \fxi{n} & \dots & \fxi{n+i-1} - \fxi{n} & \fxi{n} & \fxi{n+i+1}-\fxi{n} & \dots & \fxi{n+d}-\fxi{n}
}
}{ \vmat{
\fxi{n+1}-\fxi{n} & \dots & \fxi{n+d}-\fxi{n}
}
} \\
= & (-1)^i \frac{ \vmat{
\fxi{n} & \dots & \fxi{n+i-1} & \fxi{n+i+1} & \fxi{n+d}
}
}{ \vmat{
1 & \dots & 1 \\ \fxi{n} & \dots & \fxi{n+d}
}}.
\end{align*}
The quasi-Newton method defined above may then be expressed as
\begin{align*}
\hat{\vec{x}}_{n+1} = & \vec{x}_n - \frac{\vmat{
0 & \vec{x}_{n+1} - \vec{x}_n & \dots & \vec{x}_{n+d} - \vec{x}_n \\
\fxi{n} & \fxi{n+1} & \dots & \fxi{n+d} }
}{ \vmat{
1 & \dots & 1 \\ \fxi{n} & \dots & \fxi{n+d} }} \\
= & \frac{ \vmat{
\vec{x}_n & \dots & \vec{x}_{n+d} \\ \fxi{n} & \dots & \fxi{n+d} }
}{ \vmat{
1 & \dots & 1 \\ \fxi{n} & \dots & \fxi{n+d} }}
\end{align*}
where one must expand the determinant along the top row to maintain the correct dimensions.
Suppose, for whatever reason, that we do not have enough values of $\fxi{n+i}$ to fully determine $\hat{J}$.
That is, suppose $F$ and $X$ have $d$ rows but only $k$ columns, and denote these submatrices by $F_k$ and $X_k$.
Rather than solve $\hat{J} \vec{u}_n = \fxi{n}$ we can make a further approximation by solving $A^\top \hat{J} \vec{u}_n = A^\top \fxi{n}$ for some matrix $A$ with the same dimension as $F_k$.
This has as its solution $\vec{u}_n = X_k (A^\top F_k)^{-1} A^\top \fxi{n}$.
It is clear that the quasi-Newton method that results from this may be written as
\begin{equation} \label{eq:uqn}
\hat{\vec{x}}_{n+1} = \frac{ \vmat{
\vec{x}_n & \dots & \vec{x}_{n+k} \\
\vec{v}_1^\top \fxi{n} & \dots & \vec{v}_1^\top \fxi{n+k} \\
\vdots & & \vdots \\
\vec{v}_k^\top \fxi{n} & \dots & \vec{v}_k^\top \fxi{n+k} }}{ \vmat{
1 & \dots & 1 \\
\vec{v}_1^\top \fxi{n} & \dots & \vec{v}_1^\top \fxi{n+k} \\
\vdots & & \vdots \\
\vec{v}_k^\top \fxi{n} & \dots & \vec{v}_k^\top \fxi{n+k} }}
\end{equation}
where $\vec{v}_i$ is the $i$--th column of $A$.
\subsection{Connection to extrapolation methods}
Let $\vec{r}(\vec{x}_i) = \vec{x}_{i+1} - \vec{x}_i$ be the residual function.
Then MPE may be expressed as equation (\ref{eq:uqn}) with $\fxi{i} = \vec{r}(\vec{x}_i)$ and $\vec{v}_i = \vec{r}(\vec{x}_{n+i-1})$.
RRE and MMPE may also be expressed as such, using $\vec{v}_i = \vec{r}(\vec{x}_{n+i}) - \vec{r}(\vec{x}_{n+i-1})$ and $\vec{v}_i$ some fixed vector independent of $\vec{r}(\vec{x})$, respectively.
\end{document}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment