$\begin{split}\Big{(}M^{+}(\mu)\Big{)}^{-1}=&\frac{S_{1}A_{-1,1}^{1}S_{1}}{\mu}+S_{0}A_{01}^{1}Q+QA_{02}^{1}S_{0}+\mu\big{(}QA_{11}^{1}Q+S_{0}A_{12}^{1}+A_{13}^{1}S_{0}\big{)}\\\
&+\mu^{2}(QA_{21}^{1}+A_{22}^{1}Q)+\Gamma_{3}^{1}(\mu).\end{split}$ (3.50)

(3) Second Kind of Resonance. In this case, we using $\eqref{eq-M-
expansions-1-3}$ with $|V(x)|\lesssim(1+|x|)^{-25-},$ see Lemma 3.1. Then by
the same process, we can rewrite $M_{2}^{+}(\mu)$ as follows:

$\begin{split}M_{2}^{+}(\mu)=&\mu
S_{0}B_{1}^{0}S_{0}+\sum_{k=2}^{8}\mu^{k}B_{k}^{0}+S_{0}\Gamma_{10}(\mu)S_{0}\\\
=&\frac{\mu}{a_{-1}^{+}}\Big{(}S_{0}T_{0}S_{0}+a_{-1}^{+}\sum_{k=2}^{5}\mu^{k-1}B_{k}^{0}+S_{0}\Gamma_{9}(\mu)S_{0}\Big{)}\\\
:=&\frac{\mu}{a_{-1}^{+}}\widetilde{M_{2}^{+}}(\mu).\end{split}$

Furthermore, we obtain a more detail expansion of $M_{3}^{+}(\mu):$

$\begin{split}M_{3}^{+}(\mu)=&a_{1}^{+}\mu
T_{1}+\sum_{k=2}^{8}\mu^{k}S_{1}B_{k}^{1}S_{1}+S_{1}\Gamma_{9}(\mu)S_{1}\\\
=&a_{1}^{+}\mu\Big{(}T_{1}+\frac{1}{a_{1}^{+}}(\mu
S_{1}B_{2}^{1}S_{1}+\sum_{k=2}^{8}\mu^{k-1}S_{1}B_{k}^{1}S_{1}+S_{1}\Gamma_{8}(\mu)S_{1}\Big{)}\\\
:=&a_{1}^{+}\mu\widetilde{M_{3}^{+}}(\mu),\end{split}$

where $B_{2}^{1},B_{3}^{1},B_{4}^{1}$ are given in (3.44) and $B_{k}^{1}(5\leq
k\leq 8)$ are also bounded operators in $L^{2}(\mathbf{R}).$

For the second kind of resonance, we aim to derive the expansions of
$\Big{(}\widetilde{M_{3}^{+}}(\mu)\Big{)}^{-1}.$ By the definition of the
second kind of resonance, we know that the operator $T_{1}$ is not invertible
on $S_{1}L^{2}(\mathbf{R}).$ Since $S_{2}$ is the Riesz projection onto the
kernel of $T_{1}$ on $S_{1}L^{2}(\mathbf{R}),$ thus $T_{1}+S_{2}$ is
invertible on $S_{1}L^{2}(\mathbf{R}).$ Using Neumann series, we have

$\begin{split}\Big{(}\widetilde{M_{3}^{+}}(\mu)+S_{2}\Big{)}^{-1}=&D_{2}-\sum_{k=1}^{7}\mu^{k}B_{k}^{2}+\Gamma_{8}(\mu),\end{split}$
(3.51)

where $B_{k}^{2}(1\leq k\leq 7)$ are bounded operators in
$S_{1}L^{2}(\mathbf{R}),$ and $B_{1}^{2},B_{2}^{2},B_{3}^{2}$ are given as
follows:

$\begin{split}B_{1}^{2}=&\frac{1}{a_{1}^{+}}D_{2}B_{2}^{1}D_{2},\ \ \ \ \ \
B_{2}^{2}=\frac{1}{a_{1}^{+}}D_{2}B_{3}^{1}D_{2}-\frac{1}{(a_{1}^{+})^{2}}D_{2}B_{2}^{1}D_{2}B_{2}^{1}D_{2},\\\
B_{3}^{2}=&\frac{1}{a_{1}^{+}}D_{2}B_{4}^{1}D_{2}+\frac{1}{(a_{1}^{+})^{3}}D_{2}B_{2}^{1}D_{2}B_{2}^{1}D_{2}-\frac{1}{(a_{1}^{+})^{2}}\big{(}D_{2}B_{2}^{1}D_{2}B_{3}^{1}D_{2}+D_{2}B_{3}^{1}D_{2}B_{2}^{1}D_{2}\big{)}.\end{split}$
(3.52)

By Lemma 3.3, we know that $\widetilde{M_{3}^{+}}(\mu)$ is invertible on
$S_{1}L^{2}(\mathbf{R})$ if and only if

$M_{4}^{+}(\mu):=S_{2}-S_{2}\big{(}\widetilde{M_{3}^{+}}(\mu)+S_{2}\big{)}^{-1}S_{2}$

is invertible on $S_{2}L^{2}(\mathbf{R})$. Moreover, using (3.51) and noting
that $S_{2}D_{2}=D_{2}S_{2}=S_{2}$, we obtain

$\begin{split}M_{4}^{+}(\mu)=&\sum_{k=1}^{7}\mu^{k}S_{2}B_{k}^{2}S_{2}+\Gamma_{8}(\mu).\end{split}$
(3.53)

Indeed, using the orthogonal properties of $S_{2}$ (see (3.17)-(3.20)), we
obtain that $S_{2}B_{1}^{2}S_{2}=0$ and

$\begin{split}S_{2}B_{2}^{2}S_{2}=\frac{a_{3}^{+}}{a_{1}^{+}}&\Big{(}S_{2}vG_{3}vS_{2}-\frac{1}{a_{2}^{+}\widetilde{a}^{+}}S_{2}T_{0}^{2}S_{2}-\frac{(a_{1}^{+})^{2}}{a_{-1}^{+}a_{3}^{+}}S_{2}vG_{1}vD_{0}vG_{1}vS_{2}\\\
&\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \
-\frac{(a_{-1}^{+})^{2}}{a_{1}^{+}a_{3}^{+}(\widetilde{a}^{+})^{2}}S_{2}T_{0}vG_{-1}vD_{2}vG_{-1}vT_{0}S_{2}\Big{)}\\\
=\frac{a_{3}^{+}}{a_{1}^{+}}&\Big{(}S_{2}vG_{3}vS_{2}-\frac{8\cdot(6!)}{\|V\|_{L^{1}}}S_{2}T_{0}^{2}S_{2}+\frac{5}{2}S_{2}vG_{1}vD_{0}vG_{1}vS_{2}\\\
&+\frac{48\cdot(6!)}{\|V\|_{L^{1}}^{2}}\Big{(}S_{2}T_{0}vG_{-1}v-\frac{\|V\|_{L^{1}}}{6}S_{2}vG_{1}vD_{0}T_{0}\Big{)}D_{2}\Big{(}vG_{-1}vT_{0}S_{2}-\frac{\|V\|_{L^{1}}}{6}T_{0}D_{0}vG_{1}vS_{2}\Big{)}\\\
:=\frac{a_{3}^{+}}{a_{1}^{+}}&T_{2}.\end{split}$ (3.54)

Furthermore, we can rewrite (3.53) as follows:

$\begin{split}M_{4}^{+}(\mu)=\
&\frac{a_{3}^{+}}{a_{1}}\mu^{2}T_{2}+\sum_{k=3}^{7}\mu^{k}S_{2}B_{k}^{2}S_{2}+S_{2}\Gamma_{8}(\mu)S_{2}\\\
=\
&\frac{a_{3}^{+}}{a_{1}^{+}}\mu^{2}\Big{(}T_{2}+\frac{a_{1}^{+}}{a_{2}^{+}}\sum_{k=3}^{7}\mu^{k-2}S_{2}B_{k}^{2}S_{2}+S_{2}\Gamma_{6}(\mu)S_{2}\Big{)}\\\
:=\ &\frac{a_{3}^{+}}{a_{1}^{+}}\mu^{2}\widetilde{M_{4}^{+}}(\mu).\end{split}$

According to Lemma 3.4 and Lemma 3.5, we know that $\ker T_{2}=\\{0\\},$ which
means that $T_{2}$ is invertible on $S_{2}L^{2}(\mathbf{R}),$ by Neumann
series and using $T_{2}^{-1}=S_{2}T_{2}^{-1}=T_{2}^{-1}S_{2}$, we have the
following asymptotic expansions of
$\Big{(}\widetilde{M_{4}^{+}}(\mu)\Big{)}^{-1}$ as $\mu\rightarrow 0:$

$\begin{split}\Big{(}\widetilde{M_{4}^{+}}(\mu)\Big{)}^{-1}=&S_{2}A_{01}^{2}S_{2}+\sum_{k=1}^{5}\mu^{k}S_{2}A_{k1}^{2}S_{2}+\Gamma_{6}(\mu).\end{split}$

Thus we have

$\Big{(}M_{4}^{+}(\mu)\Big{)}^{-1}=\frac{S_{2}A_{-2,1}^{2}S_{2}}{\mu^{2}}+\sum_{k=-1}^{3}\mu^{k}S_{2}A_{k1}^{2}S_{2}+\Gamma_{4}(\mu).$

Then by Lemma 3.3 and using the orthogonal relationships (3.17), we have

$\begin{split}\Big{(}\widetilde{M_{3}^{+}}(\mu)\Big{)}^{-1}=&\Big{(}\widetilde{M_{3}^{+}}(\mu)+S_{2}\Big{)}^{-1}+\Big{(}\widetilde{M_{3}^{+}}(\mu)+S_{2}\Big{)}^{-1}S_{2}\Big{(}M_{4}^{+}(\mu)\Big{)}^{-1}S_{2}\Big{(}\widetilde{M_{3}^{+}}(\mu)+S_{2}\Big{)}^{-1}\\\
=&\frac{S_{2}A_{-2,1}^{2}S_{2}}{\mu^{2}}+\frac{S_{2}{A}_{-1,1}^{2}S_{1}+S_{1}{A}_{-1,2}^{2}S_{2}}{\mu}+\sum_{k=0}^{3}\mu^{k}S_{1}A_{k1}^{2}S_{1}+\Gamma_{4}(\mu).\end{split}$

Next, we can obtain that

$\begin{split}\Big{(}M_{3}^{+}(\mu)\Big{)}^{-1}=&\frac{S_{2}A_{-3,1}^{2}S_{2}}{\mu^{3}}+\frac{S_{2}{A}_{-2,1}^{2}S_{1}+S_{1}{A}_{-2,2}^{2}S_{2}}{\mu^{2}}+\sum_{k=-1}^{2}\mu^{k}S_{1}A_{k1}^{2}S_{1}+\Gamma_{3}(\mu).\end{split}$

Using the orthogonal relationships (3.17) and Lemma 3.3, we have

$\begin{split}\Big{(}\widetilde{M_{2}^{+}}(\mu)\Big{)}^{-1}=&\Big{(}\widetilde{M_{2}^{+}}(\mu)+S_{1}\Big{)}^{-1}+\Big{(}\widetilde{M_{2}^{+}}(\mu)+S_{1}\Big{)}^{-1}S_{1}\Big{(}M_{3}^{+}(\mu)\Big{)}^{-1}S_{1}\Big{(}\widetilde{M_{2}^{+}}(\mu)+S_{1}\Big{)}^{-1}\\\
=&\frac{S_{2}A_{-3,1}^{2}S_{2}}{\mu^{3}}+\frac{S_{2}{A}_{-2,1}^{2}S_{0}+S_{0}{A}_{-2,2}^{2}S_{2}}{\mu^{2}}+\sum_{k=-1}^{2}\mu^{k}S_{0}A_{k1}^{2}S_{0}+\Gamma_{3}(\mu).\end{split}$

Furthermore, since $\displaystyle
M_{2}^{+}(\mu)=\frac{\mu}{a_{-1}^{+}}\widetilde{M_{2}^{+}}(\mu)$, then

$\begin{split}\Big{(}{M_{2}^{+}}(\mu)\Big{)}^{-1}=&\frac{S_{2}A_{-4,1}^{2}S_{2}}{\mu^{4}}+\frac{S_{2}{A}_{-3,1}^{2}S_{0}+S_{0}{A}_{-3,2}^{2}S_{2}}{\mu^{3}}+\frac{S_{0}A_{-2,1}^{2}S_{0}}{\mu^{2}}+\frac{S_{0}A_{-1,1}^{2}S_{0}}{\mu}\\\
&\ \ +\mu S_{0}A_{01}^{2}S_{0}+\Gamma_{2}(\mu).\end{split}$

Next, using Lemma 3.3 and the orthogonal properties of $S_{2}$ (see
(3.16)-(3.20)), we have

$\begin{split}\Big{(}\widetilde{M_{1}^{+}}(\mu)\Big{)}^{-1}=&\Big{(}\widetilde{M_{1}^{+}}(\mu)+S_{0}\Big{)}^{-1}+\Big{(}\widetilde{M_{1}^{+}}(\mu)+S_{0}\Big{)}^{-1}S_{0}\Big{(}M_{2}^{+}(\mu)\Big{)}^{-1}S_{0}\Big{(}\widetilde{M_{1}^{+}}(\mu)+S_{0}\Big{)}^{-1}\\\
=&\frac{S_{2}A_{-4,1}^{2}S_{2}}{\mu^{4}}+\frac{S_{2}{A}_{-3,1}^{2}S_{0}+S_{0}{A}_{-3,2}^{2}S_{2}}{\mu^{3}}+\frac{S_{0}A_{-2,1}^{2}S_{0}+QA_{-2,2}^{2}S_{2}+S_{2}A_{-2,3}^{2}Q}{\mu^{2}}\\\
&\ \ +\frac{QA_{-1,1}^{2}S_{0}+S_{0}A_{-1,2}^{2}Q}{\mu}+QA_{01}^{2}Q+\mu
QA_{11}^{2}Q+\Gamma_{2}(\mu).\end{split}$

Since $\displaystyle
M_{1}^{+}(\mu)=\frac{a_{-1}^{+}}{\widetilde{a}^{+}}\widetilde{M}_{1}^{+}(\mu),$
then

$\begin{split}\Big{(}{M_{1}^{+}}(\mu)\Big{)}^{-1}=&\frac{S_{2}A_{-6,1}^{2}S_{2}}{\mu^{6}}+\frac{S_{2}{A}_{-5,1}^{2}S_{0}+S_{0}{A}_{-5,2}^{2}S_{2}}{\mu^{5}}+\frac{S_{0}A_{-4,1}^{2}S_{0}+QA_{-4,2}^{2}S_{2}+S_{2}A_{-4,3}^{2}Q}{\mu^{4}}\\\
&\ \
+\frac{QA_{-3,1}^{2}S_{0}+S_{0}A_{-3,2}^{2}Q}{\mu^{3}}+\frac{QA_{-2,1}^{2}Q}{\mu^{2}}+\frac{QA_{11}^{2}Q}{\mu}+\Gamma_{1}(\mu).\end{split}$

Finally, using the orthogonality relationships (see (3.16)-(3.20)), we have

$\begin{split}\Big{(}\widetilde{M^{+}}(\mu)\Big{)}^{-1}=&\Big{(}\widetilde{M^{+}}(\mu)+Q\Big{)}^{-1}+\Big{(}\widetilde{M^{+}}(\mu)+Q\Big{)}^{-1}Q\Big{(}M_{1}^{+}(\mu)\Big{)}^{-1}Q\Big{(}\widetilde{M^{+}}(\mu)+Q\Big{)}^{-1}\\\
=&\frac{S_{2}A_{-6,1}^{2}S_{2}}{\mu^{6}}+\frac{S_{2}{A}_{-5,1}^{2}S_{0}+S_{0}{A}_{-5,2}^{2}S_{2}}{\mu^{5}}+\frac{S_{0}A_{-4,1}^{2}S_{0}+QA_{-4,2}^{2}S_{2}+S_{2}A_{-4,3}^{2}Q}{\mu^{4}}\\\
&+\frac{QA_{-3,1}^{2}S_{0}+S_{0}A_{-3,2}^{2}Q+S_{2}A_{-3,3}^{2}+A_{-3,4}^{2}S_{2}}{\mu^{3}}+\frac{QA_{-2,1}^{2}Q+S_{0}A_{-2,2}^{2}+A_{-2,3}^{2}S_{0}}{\mu^{2}}\\\
&+\frac{QA_{-1,1}^{2}+A_{-1,2}^{2}}{\mu}+\Gamma_{0}(\mu).\end{split}$

Furthermore, we obtain the following asymptotic expansions of
$\Big{(}M^{+}(\mu)\Big{)}^{-1}$ for zero is the second resonance of $H$,

$\begin{split}\Big{(}M^{+}(\mu)\Big{)}^{-1}=&\frac{S_{2}A_{-3,1}^{2}S_{2}}{\mu^{3}}+\frac{S_{2}A_{-2,1}^{2}S_{0}+S_{0}A_{-2,2}^{2}S_{2}}{\mu^{2}}+\frac{S_{0}A_{-1,1}^{2}S_{0}+QA_{-1,2}^{2}S_{2}+S_{2}A_{-1,3}^{2}Q}{\mu}\\\
&\ \
+\big{(}QA_{01}^{2}S_{0}+S_{0}A_{02}^{2}Q+S_{2}A_{03}^{2}+A_{04}^{2}S_{2}\big{)}+\mu\big{(}QA_{11}^{2}Q+S_{0}A_{12}^{2}+A_{13}S_{0}\big{)}\\\
&\ \ +\mu^{2}(QA_{21}^{2}+A_{21}^{2}Q)+\Gamma_{3}^{2}(\mu).\end{split}$ (3.55)

##  4\. Proof of Theorem 1.7

In this part, we aim to give the proof of Theorem 1.7. In fact, it suffices to
establish the following characterizations of resonance spaces.

######  Theorem 4.1.

Assume that $|V(x)|\lesssim(1+|x|)^{-\beta}$ with some $\beta>17$. Then the
following statements hold:

(i) $f(x)\in S_{1}L^{2}(\mathbf{R})$ if and only if $f(x)=Uv\phi$ with
$\phi\in W_{\frac{3}{2}}(\mathbf{R})$ such that $H\phi=0$ in the
distributional sense. Furthermore,

$\phi=-G_{0}vf+c_{1}x+c_{2},$

where

$c_{1}=\frac{\langle v_{1},T_{0}f\rangle}{\|v_{1}\|_{L^{2}(\mathbf{R})}}\ \ \
\hbox{and}\ \ \ c_{2}=\frac{\langle
v,T_{0}f\rangle}{\|V\|_{L^{1}(\mathbf{R})}}-\frac{\langle
xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}c_{1}$

with $\displaystyle v_{1}=xv-\frac{\langle
xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}v\in(Q-S_{0})L^{2}(\mathbf{R}).$

(ii) $f(x)\in S_{2}L^{2}(\mathbf{R})$ if and only if $f(x)=Uv\phi$ with
$\phi\in W_{\frac{1}{2}}(\mathbf{R})$ such that $H\phi=0$ in the
distributional sense. Furthermore,

$\phi=-G_{0}vf+\frac{\langle v,T_{0}f\rangle}{\|V\|_{L^{1}(\mathbf{R})}}.$

###### Proof.

(i) Assume that $f\in S_{1}L^{2}(\mathbf{R})$. One has

$0=S_{0}T_{0}S_{0}f=S_{0}(U+vG_{0}v)f.$

Note that

$0=S_{0}(U+vG_{0}v)f=(I-\widetilde{P})(U+vG_{0}v)f=Uf+vG_{0}vf-\widetilde{P}T_{0}f,$

where $\widetilde{P}$ is the projection onto the
span$\\{v,xv\\}=(S_{0}L^{2}(\mathbf{R}))^{\perp}$. In fact, we know that $v$
and $\displaystyle v_{1}=xv-\frac{\langle
xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}v$ are the orthogonal basis of
$(S_{0}L^{2}(\mathbf{R}))^{\perp}$ . Hence, we have

$\begin{split}\widetilde{P}T_{0}f=&\frac{\langle
v_{1},T_{0}f\rangle}{\|v_{1}\|_{L^{2}(\mathbf{R})}}v_{1}+\frac{\langle
v,T_{0}f\rangle}{\|V\|_{L^{1}(\mathbf{R})}}v\\\ =&xv\frac{\langle
v_{1},T_{0}f\rangle}{\|v_{1}\|_{L^{2}(\mathbf{R})}}++v\Big{(}\frac{\langle
v,T_{0}f\rangle}{\|V\|_{L^{1}(\mathbf{R})}}-\frac{\langle
xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}\cdot\frac{\langle
v_{1},T_{0}f\rangle}{\|v_{1}\|_{L^{2}(\mathbf{R})}}\Big{)}\\\
:=&c_{1}xv+c_{2}v.\end{split}$

Furthermore, we have

$Uf=-vG_{0}vf+\widetilde{P}T_{0}f=-vG_{0}vf+c_{1}xv+c_{2}v.$

Since $U^{2}=1,$ then

$f=U^{2}f=Uv(-G_{0}vf)+U(c_{1}xv+c_{2}v)=Uv(-G_{0}vf+c_{1}x+c_{2}):=Uv\phi,$

and $vf=vUv\phi=V\phi$. By $G_{0}=\big{(}\Delta^{2}\big{)}^{-1}$ and noting
that $\Delta^{2}(c_{1}x+c_{2})=0$, we have

$H\phi=\big{(}\Delta^{2}+V\big{)}\phi=\Delta^{2}\phi+V\phi=-vf+V\phi=0.$

Next, we show that $\phi\in W_{\frac{3}{2}}(\mathbf{R})$. It is obvious that
$c_{1}x+c_{2}\in W_{\frac{3}{2}}(\mathbf{R})$, so it suffices to show that

$G_{0}vf=\frac{1}{12}\int_{\mathbf{R}}|x-y|^{3}v(y)f(y)dy\in
W_{\frac{3}{2}}(\mathbf{R}).$

Since $S_{1}\leq S_{0},$ and we know that if $f\in S_{0}L^{2}(\mathbf{R})$,
then

$\int_{\mathbf{R}}v(y)f(y)dy=0,\ \ \hbox{and}\ \
\int_{\mathbf{R}}yv(y)f(y)dy=0.$

Hence

$G_{0}vf=\frac{1}{12}\int_{\mathbf{R}}\big{(}|x-y|^{3}-|x|^{3}+3|x|x\cdot
y\big{)}v(y)f(y)dy:=\frac{1}{12}\int_{\mathbf{R}}K_{1}(x,y)v(y)f(y)dy$

with $K_{1}(x,y)=|x-y|^{3}-|x|^{3}+3|x|x\cdot y$.

Indeed, we can divided $K_{1}(x,y)$ into three parts as follows:

$\begin{split}K_{1}(x,y)=&|x-y|^{3}-|x|^{3}+3|x|x\cdot y\\\
=&\Big{(}x^{2}(|x-y|-|x|)+|x|x\cdot y\Big{)}+y^{2}(|x-y|-|x|)\\\ &\ \
+\Big{(}y^{2}|x|-2x\cdot y(|x-y|-|x|)\Big{)}\\\
:=&K_{11}(x,y)+K_{12}(x,y)+K_{13}(x,y).\end{split}$

For the first part $K_{11}(x,y),$ we have

$\begin{split}K_{11}(x,y)&=\left|\frac{x^{2}(y^{2}-2xy)}{|x-y|+|x|}+\frac{x^{3}y+xy|x|\cdot|x-y|}{|x-y|+|x|}\right|\\\
&\leq\frac{x^{2}y^{2}}{|x-y|+|x|}+\left|\frac{|x|xy(|x-y|-|x|)}{|x-y|+|x|}\right|\\\
&\lesssim|x|y^{2}.\end{split}$

Thus, we have

$\Big{|}\int_{\mathbf{R}}K_{11}(x,y)v(y)f(y)dy\Big{|}\in
W_{\frac{3}{2}}(\mathbf{R}).$

Then, for the second part $K_{22}(x,y),$ we have

$|K_{12}(x,y)|=\Big{|}y^{2}(|x-y|-|x|)\Big{|}\lesssim|y|^{3}.$

It is obvious that

$\Big{|}\int_{\mathbf{R}}K_{12}(x,y)v(y)f(y)dy\Big{|}\lesssim\int_{\mathbf{R}}|y|^{3}v(y)f(y)dy\in
W_{\frac{3}{2}}(\mathbf{R}).$

Finally, for the last part $K_{13}(x,y),$ we have

$|K_{13}(x,y)|=\Big{|}~{}~{}y^{2}|x|-2xy(|x-y|-|x|)\Big{|}\leq|x|y^{2}+2|x|y^{2}\lesssim|x|y^{2}.$

It is easy to know that

$\Big{|}\int_{\mathbf{R}}K_{13}(x,y)v(y)f(y)dy\Big{|}\lesssim|x|\int_{\mathbf{R}}|y|^{2}v(y)f(y)dy\in
W_{\frac{3}{2}}(\mathbf{R}).$

Hence, we obtain that

$\begin{split}|G_{0}vf|=&\Big{|}\frac{1}{12}\int_{\mathbf{R}}K_{1}(x,y)v(y)f(y)dy\Big{|}\\\
\lesssim&\Big{|}\int_{\mathbf{R}}K_{11}(x,y)v(y)f(y)dy\Big{|}+\Big{|}\int_{\mathbf{R}}K_{12}(x,y)v(y)f(y)dy\Big{|}\\\
&+\Big{|}\int_{\mathbf{R}}K_{13}(x,y)v(y)f(y)dy\Big{|},\end{split}$ (4.1)

which implies that $G_{0}vf\in W_{\frac{3}{2}}(\mathbf{R}).$

On the other hand, let $\phi\in W_{\frac{3}{2}}(\mathbf{R})$ such that
$H\phi=0.$ We first show that $f=Uv\phi\in S_{0}L^{2}(\mathbf{R}),$ that is,

$\int_{\mathbf{R}}v(y)f(y)dy=0\ \ \hbox{and}\ \
\int_{\mathbf{R}}yv(y)f(y)dy=0.$

Noting that $vf=V\phi=-\Delta^{2}\phi$. Let $\eta(x)\in
C_{0}^{\infty}(\mathbf{R})$ such that $\eta(x)=1$ for $|x|\leq 1$ and
$\eta(x)=0$ for $|x|>2.$ Then for any $\delta>0,$ we have

$\begin{split}\Big{|}\int_{\mathbf{R}}v(y)f(y)\eta(\delta
y)dy\Big{|}=&\Big{|}\int_{\mathbf{R}}\big{(}\Delta^{2}\phi(y)\big{)}\eta(\delta
y)dy\Big{|}=\Big{|}\int_{\mathbf{R}}\phi(y)\delta^{4}\eta^{(4)}(\delta
y)dy\Big{|}\\\ \lesssim&\delta^{4}\big{\|}\langle
y\rangle^{-\frac{3}{2}-}\phi(y)\big{\|}_{L^{2}(\mathbf{R})}\cdot\|\langle
y\rangle^{\frac{3}{2}+}\eta^{(4)}(\delta y)\|_{L^{2}(\mathbf{R})}\\\
\lesssim&\delta^{\frac{5}{2}-}\big{\|}\langle
y\rangle^{-\frac{3}{2}-}\phi(y)\big{\|}_{L^{2}(\mathbf{R})}\cdot\|\eta^{(4)}(y)\|_{L^{2}(\mathbf{R})}.\end{split}$
(4.2)

Taking the limit $\delta\rightarrow 0$ and using the dominated convergence
theorem we obtain $\displaystyle\int_{\mathbf{R}}v(y)f(y)dy=0.$ Similarly, we
have $\displaystyle\int_{\mathbf{R}}v(y)f(y)dy=0.$ Thus, $f=Uv\phi\in
S_{0}L^{2}(\mathbf{R}).$

Now we turn to prove that $\phi=-G_{0}vf+c_{1}x+c_{2}.$ Let
$\widetilde{\phi}=\phi+G_{0}vf,$ then by assumption and (4.1),
$\widetilde{\phi}\in W_{\frac{3}{2}}(\mathbf{R})$ and
$\Delta^{2}\widetilde{\phi}=0,$ thus
$\widetilde{\phi}=\widetilde{c}_{1}x+\widetilde{c}_{2}.$ This implies that
$\phi=-G_{0}vf+\widetilde{c}_{1}x+\widetilde{c}_{2}.$

Since

$\displaystyle 0=H\phi=\big{[}\Delta^{2}+V\big{]}\phi=$
$\displaystyle-\big{[}\Delta^{2}+V\big{]}(G_{0}vf)+\big{[}\Delta^{2}+V\big{]}(\widetilde{c}_{1}x+\widetilde{c}_{2})$
$\displaystyle=$ $\displaystyle-
Uv\big{(}U+vG_{0}v\big{)}f+U\big{(}\widetilde{c}_{1}xv^{2}+\widetilde{c}_{2}v^{2}\big{)}$
$\displaystyle=$ $\displaystyle
U\big{(}-vT_{0}f+\widetilde{c}_{1}xv^{2}+\widetilde{c}_{2}v^{2}\big{)},$

thus we have

$\displaystyle\widetilde{c}_{1}xv^{2}+\widetilde{c}_{2}v^{2}=vT_{0}f\ \
\Longrightarrow$ $\displaystyle\ \ \widetilde{c}_{1}\langle
xv,v\rangle+\widetilde{c}_{2}\langle v,v\rangle=\langle v,T_{0}f\rangle,$
(4.3)
$\displaystyle\widetilde{c}_{1}x^{2}v^{2}+\widetilde{c}_{2}xv^{2}=xvT_{0}f\ \
\Longrightarrow$ $\displaystyle\ \ \widetilde{c}_{1}\langle
xv,xv\rangle+\widetilde{c}_{2}\langle xv,v\rangle=\langle xv,T_{0}f\rangle.$
(4.4)

By (4.3), we have

$\widetilde{c_{2}}=\frac{\langle
v,T_{0}f}{\|V\|_{L^{1}(\mathbf{R})}}-\widetilde{c}_{1}\frac{\langle
xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}.$ (4.5)

Then, substituting (4.5), $\displaystyle xv=v_{1}+\frac{\langle
xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}v$ and $\langle v_{1},v\rangle=0$ into
(4.4), we have

$\widetilde{c}_{1}=\frac{\langle
v_{1},T_{0}f\rangle}{\|v_{1}\|_{L^{2}(\mathbf{R})}}\ \ \ \hbox{and}\ \ \
\widetilde{c}_{2}=\frac{\langle
v,T_{0}f\rangle}{\|V\|_{L^{1}(\mathbf{R})}}-\frac{\langle
xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}\widetilde{c}_{1},$

which means that $\widetilde{c}_{1}=c_{1},\widetilde{c}_{2}=c_{2}.$
Furthermore, using $f=Uv\phi$ and $H\phi=0$, we have

$\begin{split}S_{0}T_{0}S_{0}f&=S_{0}T_{0}(Uv\phi)=S_{0}(U+vG_{0}v)(Uv\phi)=S_{0}(v\phi+vG_{0}vUv\phi)\\\
&=S_{0}vG_{0}\big{(}\Delta^{2}+V\big{)}\phi=S_{0}vG_{0}H\phi=0.\end{split}$
(4.6)

Hence, we obtain that $f\in S_{1}L^{2}(\mathbf{R})$. The proof of Theorem 4.1
(i) is completed.

(ii) Since $S_{2}\leq S_{1}$, by Theorem 4.1(i), we know that $f\in
S_{1}L^{2}(\mathbf{R})$ if and only if $f=Uv\phi$ with $\phi\in
W_{\frac{3}{2}}(\mathbf{R})$ such that $H\phi=0$ in the distributional sense
and

$\phi=-G_{0}vf+c_{1}x+c_{2}.$

Thus, we just need to prove that $c_{1}=0$ and $\phi\in
W_{\frac{1}{2}}(\mathbf{R})$. Indeed, by the proof of Theorem 4.1(i), we know
that

$v_{1}=xv-\frac{\langle xv,v\rangle}{\|V\|_{L^{1}(\mathbf{R})}}v\in
QL^{2}(\mathbf{R}).$

By Table 1, if $f\in S_{2}L^{2}(\mathbf{R})$, then $QT_{0}f=0,$ thus

$c_{1}=\langle v_{1},T_{0}f\rangle=0.$

Next we show that $\phi\in W_{\frac{1}{2}}(\mathbf{R})$. Since
$\phi=-G_{0}vf+\displaystyle\frac{\langle
v,T_{0}f\rangle}{\|V\|_{L^{1}(\mathbf{R})}},$ thus it suffices to show that

$G_{0}vf=\frac{1}{12}\int_{\mathbf{R}}|x-y|^{3}v(y)f(y)dy\in
W_{\frac{1}{2}}(\mathbf{R}).$

We obtain that if $f\in S_{2}L^{2}(\mathbf{R})$, then

$\int_{\mathbf{R}}v(y)f(y)dy=\int_{\mathbf{R}}yv(y)f(y)dy=\int_{\mathbf{R}}y^{2}v(y)f(y)dy=0.$

Thus,

$\displaystyle G_{0}vf=$
$\displaystyle\frac{1}{12}\int_{\mathbf{R}}\big{(}|x-y|^{3}-|x|^{3}+3|x|x\cdot
y-3|x|y^{2}\big{)}v(y)f(y)dy$ $\displaystyle:=$
$\displaystyle\frac{1}{12}\int_{\mathbf{R}}K_{2}(x,y)v(y)f(y)dy$

with $K_{2}(x,y)=|x-y|^{3}-|x|^{3}+3|x|x\cdot y-3|x|y^{2}$.

Indeed, we can divided $K_{2}(x,y)$ into four parts as follows

$\begin{split}K_{2}(x,y)=&y^{2}\big{(}|x-y|-|x|\big{)}-2\big{[}xy(|x-y|-|x|)+|x|y^{2}\big{]}+\Big{[}\frac{x^{2}y^{2}}{|x-y|+|x|}-\frac{1}{2}|x|y^{2}\Big{]}\\\
&+\Big{[}\frac{|x|x\cdot
y(|x-y|-|x|)}{|x-y|+|x|}+\frac{1}{2}|x|y^{2}\Big{]}\\\
:=&K_{21}(x,y)+K_{22}(x,y)+K_{23}(x,y)+K_{24}(x,y).\end{split}$

For the first part $K_{21}(x,y),$ we have

$|K_{21}(x,y)|=\big{|}y^{2}(|x-y|-|x|)\big{|}\lesssim|y|^{3}.$

Thus

$\Big{|}\int_{\mathbf{R}}K_{21}(x,y)v(y)f(y)dy\Big{|}\lesssim\int_{\mathbf{R}}|y|^{3}|v(y)f(y)|dy\in
W_{\frac{1}{2}}(\mathbf{R}).$

For the second part $K_{22}(x,y),$ we have

$\begin{split}|K_{22}(x,y)|=&2\Big{|}\frac{x\cdot y(y^{2}-2x\cdot
y)}{|x-y|+|x|}+\frac{|x|y^{2}(|x-y|+|x|)}{|x-y|+|x|}\Big{|}\\\
\lesssim&\frac{|x|\cdot|y|^{3}}{|x-y|+|x|}+\Big{|}\frac{|x|y^{2}(|x-y|-|x|)}{|x-y|+|x|}\Big{|}\\\
\lesssim&|y|^{3}.\end{split}$

Thus

$\Big{|}\int_{\mathbf{R}}K_{22}(x,y)v(y)f(y)dy\Big{|}\lesssim\int_{\mathbf{R}}|y|^{3}|v(y)f(y)|dy\in
W_{\frac{1}{2}}(\mathbf{R}).$

Then, for the third part $K_{23}(x,y)$, we have

$\begin{split}|K_{23}(x,y)|=&\Big{|}\frac{2x^{2}y^{2}}{2(|x-y|+|x|)}-\frac{|x|y^{2}(|x-y|+|x|)}{2(|x-y|+|x|)}\Big{|}\\\
=&\Big{|}\frac{|x|y^{2}\cdot|(|x-y|-|x|)|}{2(|x-y|+|x|)}\Big{|}\\\
\lesssim&|y|^{3}.\end{split}$

Thus

$\Big{|}\int_{\mathbf{R}}K_{23}(x,y)v(y)f(y)dy\Big{|}\lesssim\int_{\mathbf{R}}|y|^{3}|v(y)f(y)|dy\in
W_{\frac{1}{2}}(\mathbf{R}).$

Finally, for the fourth part $K_{24}(x,y)$, we have

$\begin{split}|K_{24}(x,y)|=&\Big{|}\frac{|x|x\cdot y(y^{2}-2x\cdot
y)}{(|x-y|+|x|)^{2}}+\frac{|x|y^{2}(|x-y|+|x|)^{2}}{2(|x-y|+|x|)^{2}}\Big{|}\\\
\leq&\frac{|x|y^{4}}{2(|x-y|+|x|)^{2}}+\Big{|}\frac{x^{2}y^{2}(|x-y|-|x|)}{(|x-y|+|x|)^{2}}\Big{|}\\\
\lesssim&|y|^{3}.\end{split}$

Thus

$\Big{|}\int_{\mathbf{R}}K_{24}(x,y)v(y)f(y)dy\Big{|}\lesssim\int_{\mathbf{R}}|y|^{3}\cdot|v(y)f(y)|dy\in
W_{\frac{1}{2}}(\mathbf{R}).$

Hence, we obtain that $G_{0}vf\in W_{\frac{1}{2}}(\mathbf{R})$. Furthermore,
we have $\phi\in W_{\frac{1}{2}}(\mathbf{R})$.

On the other hand, let $\phi(x)\in W_{\frac{1}{2}}(\mathbf{R})$ such that
$H\phi(x)=0.$ Denote $f=Uv\phi,$ with the similar method of (4.2) and (4.6),
we have

$\int_{\mathbf{R}}v(y)f(y)dy=0,\ \int_{\mathbf{R}}yv(y)f(y)dy=0,\
\int_{\mathbf{R}}y^{2}v(y)f(y)dy=0,S_{0}T_{0}S_{0}f=0,$

then we can obtain that $f\in S_{1}L^{2}(\mathbf{R})$ and $G_{0}vf\in
W_{\frac{1}{2}}(\mathbf{R}).$ Moreover, by Theorem 4.1(i), we know that
$\phi=-G_{0}vf+c_{1}x+c_{2}$ since $W_{\frac{1}{2}}(\mathbf{R})\subset
W_{\frac{3}{2}}(\mathbf{R}).$ Thus we need that $c_{1}x=\phi+G_{0}vf-c_{2}\in
W_{\frac{1}{2}}(\mathbf{R})$ this necessitates that $c_{1}=0,$ which implies
that $T_{0}f\in\text{span}\\{v\\}$ and
$\phi=-G_{0}vf+\displaystyle\frac{\langle
v,T_{0}f\rangle}{\|V\|_{L^{1}(\mathbf{R})}}$. Hence, we have $f\in
S_{2}L^{2}(\mathbf{R})$. The proof of Theorem 4.1(ii) is completed. ∎

Acknowledgements: A. Soffer is partially supported by the Simon’s Foundation
(No.395767). Z. Wu and X. H. Yao are partially supported by NSFC (No.11771165
and 12171182). The authors would like to express their sincere gratitude to
the reviewing referee for many constructive comments which helped us greatly
improve the previous version. We also would like to thank Professor P. Li for
his useful discussions, and thank Professor M. Goldberg for his helpful
comments and insights for the previous version.

## References

  * [Agm75] S. Agmon, _Spectral properties of Schrödinger operators and scattering theory_ , Ann. Scuola Norm. Sup. Pisa Cl. Sci. (4) 2 (1975), no. 2, 151–218. MR 0397194
  * [BKS] M. Ben-Artzi, H.Koch, and J.Saut, _Dispersion estimates for fourth order Schroödinger equations,_ C. R. Acad. Sci. Paris S$\acute{e}$r. I Math., 330(2000), No.1,87-92. MR 1745182
  * [BS91] M. Sh. Birman and M. Z. Solomyak, _Estimates for the number of negative eigenvalues of the Schrödinger operator and its generalizations_ , Estimates and asymptotics for discrete spectra of integral and differential equations (Leningrad, 1989–90), Adv. Soviet Math., vol. 7, Amer. Math. Soc., Providence, RI, 1991, pp. 1–55. MR 1306507
  * [Bou91] J. Bourgain, _Global solutions of nonlinear Schrödinger equations_. American Mathematical Society Colloquium Publications, 46. American Mathematical Society, Providence, RI, 1999.MR 1691575
  * [Caz03] Thierry Cazenave, _Semilinear Schrödinger equations_ , Courant Lecture Notes in Mathematics, vol. 10, New York University, Courant Institute of Mathematical Sciences, New York; American Mathematical Society, Providence, RI, 2003. MR 2002047
  * [CMY12] W. Chen, C. Miao and X. Yao, _Dispersive estimates with geometry of finite type_ , Comm. Partial Differential Equations. 37(2012) 479-510. MR 2889560
  * [CS01] O. Costin and A. Soffer, _Resonance theory for Schrödinger operators_ , Comm. Math. Phys. 224 (2001), no. 1, 133–152, Dedicated to Joel L. Lebowitz. MR 1868995
  * [Dav97] E. B. Davies, _Limits on $L^{p}$ regularity of self-adjoint elliptic operators_, J. Differential Equations 135 (1997), no. 1, 83–102. MR 1434916
  * [DDY14] Q. Deng, Y. Ding, and X. Yao, _Gaussian bounds for higher-order elliptic differential operators with Kato type potentials_ , J. Funct. Anal. 266 (2014), no. 8, 5377–5397. MR 3177340
  * [DT79] P. A. Deift and E. Trubowitz, _Inverse scattering on the line_ , Comm. Pure Appl. Math. 32 (1979), no. 2, 121–251. MR 0512420
  * [DF08] P. D’Ancona and L. Fanelli, _Strichartz and smoothing estimates of dispersive equations with magnetic potentials_ , Comm. Partial Differential Equations 33 (2008), no. 4-6, 1082–1112. MR 2424390
  * [DH98a] E. B. Davies and A. M. Hinz, _Explicit constants for Rellich inequalities in $L_{p}(\Omega)$_, Math. Z. 227 (1998), no. 3, 511–523. MR 1612685
  * [DH98b] E. B. Davies and A. M. Hinz, _Kato class potentials for higher order elliptic operators_ , J. London Math. Soc. (2) 58 (1998), no. 3, 669–678. MR 1678156
  * [EGT21] M. B. Erdoğan, W. R. Green, and E. Toprak, _On the fourth order schrödinger equation in three dimensions: dispersive estimates and zero energy resonance_ , J. Differential Equations 271 (2021), 152–185. MR 4151179
  * [EKMT16] I. E. Egorova, E. A. Kopylova, V. A. Marchenko, and G. Teschl, _On the sharpening of dispersion estimates for One-Dimensional Schrödinger and Klein-Gordon equations_ , Russian Math. Surveys 71 (2016), no. 3, 391–415. MR 3535363
  * [ES04] M. B. Erdoğan and W. Schlag, _Dispersive estimates for Schrödinger operators in the presence of a resonance and/or an eigenvalue at zero energy in dimension three. I_ , Dyn. Partial Differ. Equ. 1 (2004), no. 4, 359–379. MR 2127577
  * [ES06] M. B. Erdoğan and W. Schlag, _Dispersive estimates for Schrödinger operators in the presence of a resonance and/or an eigenvalue at zero energy in dimension three. II_ , J. Anal. Math. 99 (2006), 199–248. MR 2279551 MR 687957
  * [FSY18] H. Feng, A. Soffer, and X. Yao, _Decay estimates and Strichartz estimates of fourth-order Schrödinger operator_ , J. Funct. Anal. 274 (2018), no. 2, 605–658. MR 3724151
  * [FSWY20] H. Feng, A. Soffer, Z. Wu and X. Yao, _Decay estimates for higher-order elliptic operators_ , Trans. Amer. Math. Soc. 373 (2020), no. 4, 2805–2859. MR 3724151
  * [FWY18] H. Feng, Z. Wu, and X. Yao, _Time asymptotic expansions of solution for fourth-order schrödinger equation with zero resonance or eigenvalue_ , https://arxiv.org/abs/1812.00223 (2018).
  * [Gold21] M. Goldberg, In private communication.
  * [GG15] M. Goldberg and W. R. Green, _Dispersive estimates for higher dimensional Schrödinger operators with threshold eigenvalues I: The odd dimensional case_ , J. Funct. Anal. 269 (2015), no. 3, 633–682. MR 3350725
  * [GG17] M. Goldberg and W. R. Green, _Dispersive estimates for higher dimensional Schrödinger operators with threshold eigenvalues II. The even dimensional case_ , J. Spectr. Theory 7 (2017), no. 1, 33–86. MR 3629407
  * [GS04] M. Goldberg and W. Schlag, _Dispersive estimates for Schrödinger operators in dimensions one and three_ , Comm. Math. Phys. 251 (2004), no. 1, 157–178. MR 2096737
  * [GT19] W. R. Green and E. Toprak, _On the fourth order schrödinger equation in four dimensions: dispersive estimates and zero energy resonance_ , J. Differential Equations 267 (2019), no. 3, 1899–1954. MR 3945621
  * [GV06] M. Goldberg and M. Visan, _A counterexample to dispersive estimates for Schrödinger operators in higher dimensions_ , Comm. Math. Phys. 266 (2006), no. 1, 211–238. MR 2231971
  * [Hill20] T. Hill, _Dispersive estimates of Schrödinger and Schrödinger-like equations in one dimension_ , Thesis (Ph.D.)–University of Cincinnati. 2020. MR 4187680
  * [Hö05] L. Hörmander, _The analysis of linear partial differential operators. II_ , Classics in Mathematics, Springer-Verlag, Berlin, 2005, Differential operators with constant coefficients, Reprint of the 1983 original. MR 2108588
  * [HS15] I. Herbst and E. Skibsted, _Decay of eigenfunctions of elliptic PDE’s, I_ , Adv. Math. 270 (2015), 138–180. MR 3286533
  * [HS17] I. Herbst and E. Skibsted, _Decay of eigenfunctions of elliptic PDE’s, II_ , Adv. Math. 306 (2017), 177–199. MR 3581301
  * [JK79] A. Jensen and T. Kato, _Spectral properties of Schrödinger operators and time-decay of the wave functions_ , Duke Math. J. 46 (1979), no. 3, 583–611. MR 544248
  * [JN01] A. Jensen and G. Nenciu, _A unified approach to resolvent expansions at thresholds_ , Rev. Math. Phys. 13 (2001), no. 6, 717–754. MR 1841744
  * [JN04] A. Jensen and G. Nenciu, _Erratum: “A unified approach to resolvent expansions at thresholds” [Rev. Math. Phys. 13 (2001), no. 6, 717–754; mr1841744]_, Rev. Math. Phys. 16 (2004), no. 5, 675–677. MR 2079122
  * [JSS91] J.-L. Journé, A. Soffer, and C. D. Sogge, _Decay estimates for Schrödinger operators_ , Comm. Pure Appl. Math. 44 (1991), no. 5, 573–604. MR 1105875
  * [Kat59] T. Kato, _Growth properties of solutions of the reduced wave equation with a variable coefficient_ , Comm. Pure Appl. Math. 12 (1959), 403–425. MR 0108633
  * [KT98] M. Keel and T. Tao, _Endpoint Strichartz estimates_ , Amer. J. Math. 120 (1998), no. 5, 955–980. MR 1646048
  * [KT06] H. Koch and D. Tataru, _Carleman estimates and absence of embedded eigenvalues_ , Comm. Math. Phys. 267 (2006), no. 2, 419–449. MR 2252331
  * [Kur78] S. T. Kuroda, _An introduction to scattering theory_ , Lecture Notes Series, vol. 51, Aarhus Universitet, Matematisk Institut, Aarhus, 1978. MR 528757
  * [LSY21] P. Li, A. Soffer and X. Yao, _Decay Estimates for Fourth-Order Schrödinger Operator in Dimension Two_ , http://arxiv.org/abs/2110.07154.
  * [Mou81] E. Mourre, _Absence of singular continuous spectrum for certain selfadjoint operators_ , Comm. Math. Phys. 78 (1980/81), no. 3, 391–408. MR 603501
  * [Mur82] M. Murata, _Asymptotic expansions in time for solutions of Schrödinger-type equations_ , J. Funct. Anal. 49 (1982), no. 1, 10–56. MR 680855
  * [NaSc11] K. Nakanishi, W. Schlag, _Invariant manifolds and dispersive Hamiltonian evolution equations_. Zurich Lectures in Advanced Mathematics. European Mathematical Society (EMS), Z rich, 2011. MR 2847755
  * [RS04] I. Rodnianski and W. Schlag, _Time decay for solutions of Schrödinger equations with rough and time-dependent potentials_ , Invent. Math. 155 (2004), no. 3, 451–513. MR 2038194
  * [Sch71] M. Schechter, _Spectra of partial differential operators_ , North-Holland Publishing Co., Amsterdam-London; American Elsevier Publishing Co., Inc., New York, 1971, North-Holland Series in Applied Mathematics and Mechanics, Vol. 14. MR 0447834
  * [Sch05] W. Schlag, _Dispersive estimates for Schrödinger operators in dimension two_ , Comm. Math. Phys. 257 (2005), no. 1, 87–117. MR 2163570
  * [Sch07] W. Schlag, _Dispersive estimates for Schrödinger operators: a survey_ , Mathematical aspects of nonlinear dispersive equations, Ann. of Math. Stud., vol. 163, Princeton Univ. Press, Princeton, NJ, 2007, pp. 255–285. MR 2333215
  * [Sch20] W. Schlag,_On pointwise decay of waves_ , J. Math. Phys. 62 (2021), no. 6, Paper No. 061509, 27 pp. MR 4276283
  * [Sim18a] B. Simon, _Tosio Kato’s work on non-relativistic quantum mechanics: part 1_ , Bull. Math. Sci. 8 (2018), no. 1, 121–232. MR 3775269
  * [Sim18b] B. Simon, _Tosio Kato’s work on non-relativistic quantum mechanics: part 2_ , Bull. Math. Sci. (2018), In Press.
  * [Ste93] E. M. Stein, _Harmonic analysis: real-variable methods, orthogonality, and oscillatory integrals_ , Princeton Mathematical Series, vol. 43, Princeton University Press, Princeton, NJ, 1993, With the assistance of Timothy S. Murphy, Monographs in Harmonic Analysis, III. MR 1232192
  * [SYY18] A. Sikora, L. Yan, and X. Yao, _Spectral multipliers, Bochner-Riesz means and uniform Sobolev inequalities for elliptic operators_ , Int. Math. Res. Not. IMRN (2018), no. 10, 3070–3121. MR 3805196
  * [Tao06] T. Tao, _Nonlinear dispersive equations_ , CBMS Regional Conference Series in Mathematics, vol. 106, Published for the Conference Board of the Mathematical Sciences, Washington, DC; by the American Mathematical Society, Providence, RI, 2006, Local and global analysis. MR 2233925
  * [Wed99] R. A. Weder, _The $W^{k,p}-$continuity of the Schrödinger wave operatos on the line_, Comm. Math. Phys. 208 (1999), no. 2, 507–520. MR 1729096
  * [Wed00] R. A. Weder, _$L^{p}-L^{p^{\prime}}$ estimates for the Schrödinger equation on the line and inverse scattering for the non linear Schrödinger equation with a potential_, J. Funct. Analysis 170(2000), 37–68.
  * [Yaj95] K. Yajima, _The $W^{k,p}$-continuity of wave operators for Schrödinger operators_, J. Math. Soc. Japan 47 (1995), no. 3, 551–581. MR 1331331


# A quantum information theoretic analysis of
reinforcement learning-assisted quantum architecture search

Abhishek Sadhu<EMAIL_ADDRESS>Raman Research Institute, Bengaluru,
India  Centre for Quantum Science and Technology (CQST), International
Institute of Information Technology, Hyderabad, Telangana, India  Aritra
Sarkar<EMAIL_ADDRESS>QuTech, Delft University of Technology, Delft,
The Netherlands  Akash Kundu<EMAIL_ADDRESS>corresponding author Joint
Doctoral School, Silesian University of Technology, Gliwice, Poland  Institute
of Theoretical and Applied Informatics, Polish Academy of Sciences, Gliwice,
Poland  QTF Centre of Excellence, Department of Physics, University of
Helsinki, Finland

###### Abstract

In the field of quantum computing, variational quantum algorithms (VQAs)
represent a pivotal category of quantum solutions across a broad spectrum of
applications. These algorithms demonstrate significant potential for realising
quantum computational advantage. A fundamental aspect of VQAs involves
formulating expressive and efficient quantum circuits (namely ansatz) and
automating the search of such ansatz is known as quantum architecture search
(QAS). RL-QAS involves optimising QAS using reinforcement learning techniques.
This study investigates RL-QAS for crafting ansatzes tailored to the
variational quantum state diagonalisation problem. Our investigation includes
a comprehensive analysis of various dimensions, such as the entanglement
thresholds of the resultant states, the impact of initial conditions on the
performance of RL-agent, the phase change behaviour of correlation in
concurrence bounds, and the discrete contributions of qubits in deducing
eigenvalues through conditional entropy metrics. We leverage these insights to
devise entanglement-guided admissible ansatz in QAS to diagonalise random
quantum states using optimal resources. Furthermore, the methodologies
presented herein offer a generalised framework for constructing reward
functions within RL-QAS applicable to variational quantum algorithms.

##  I Introduction

Quantum computing represents a paradigm-shifting approach to computation,
leveraging the principles of quantum mechanics to process information in ways
fundamentally different from classical computers. At its core, this technology
employs quantum bits, or qubits, which can exist in superposition states,
enabling parallel computation paths. The information in qubits is transformed
via quantum circuits consisting of quantum logic gates that are capable of
acting on these computation paths in tandem. The quantum circuits represent a
corresponding quantum algorithm that can effectuate a computational advantage
for a target application Harrow and Montanaro (2017); Montanaro (2016); Sadhu
_et al._ (2023). These algorithms are either constructed manually, based on
conventional logical reasoning or automatically by composing the quantum gates
to replicate a required input-output behaviour via training. The latter
approach is termed quantum architecture search (QAS) Zhang  _et al._ (2022);
Du  _et al._ (2022); Kuo  _et al._ (2021); Zhang  _et al._ (2021).

QAS typically consists of two parts. First, a template of the circuits is
built, called the ansatz. The ansatz can have parametric quantum gates, e.g.,
rotation angles. Then, these parameters are determined via the variational
principle using a classical optimiser in a feedback loop. Algorithms
constructed via this technique are called variational quantum algorithms (VQA)
McClean  _et al._ (2016); Cerezo  _et al._ (2021). QAS can also be used to
determine non-parametric circuits as an approach for quantum program synthesis
Sarkar (2022). In this research, however, we will focus on QAS in the context
of VQA.

The ansatz design is critical, as it directly influences the expressivity and
efficiency of the quantum solution. Given the vast potential configuration
space of quantum circuits, the process of identifying a suitable ansatz is a
challenging problem. In Ostaszewski  _et al._ (2021); Kundu  _et al._ (2024);
Patel  _et al._ (2024); Du  _et al._ (2022), the search for the quantum
circuits that solve various VQA has been automated using reinforcement
learning (RL) techniques Sutton and Barto (2018). We particularly call
ansatzes capable of solving a VQA an admissible ansatz. The automated search
for such admissible ansatz using the RL-agent is termed reinforcement
learning-assisted quantum architecture search (RL-QAS). In this setting, after
a certain number of episodes, the RL agent returns possible configurations of
quantum ansatz that solve the problem. For $E_{tot}$ episodes, the agent
returns $E_{s}$ successful episodes, and in $E_{tot}-E_{s}$ episodes, the
agent fails to find a configuration within a specific predefined accuracy and
depth. It has been shown in Ostaszewski  _et al._ (2021); Wu  _et al._ (2023);
Kundu  _et al._ (2024); Patel  _et al._ (2024); Kundu (2024) that classical
machine learning-driven QAS algorithms outperform the state-of-art structures
of ansatz and effectively optimise the number of parameters, depth, and impact
of noise in VQAs.

The use-cases of such admissible ansatzes cover applications in quantum
information theory LaRose  _et al._ (2019); Cerezo  _et al._ (2021); Tan and
Volkoff (2021); Cerezo  _et al._ (2020); Kundu and Miszczak (2022), chemistry
Kandala  _et al._ (2017); Mustafa  _et al._ (2022); Delgado  _et al._ (2021)
and combinatorial optimization Khairy  _et al._ (2020); Liu  _et al._ (2022);
Glos  _et al._ (2022). Yet, a thorough investigation of such admissible
ansatzes is still missing in recent literature. It has been previously shown
Patel  _et al._ (2024) that in RL-QAS methods, it is crucial to feed proper
information about the problem to the agent to enhance the agent’s performance.
One of the ways of providing this information is by engineering a reward
function. In RL, the formulation of the reward function and its engineering
can be sparse or dense. The choice of optimal reward function depends on trial
and error by investigating the performance of the agent under various reward
configurations.

In this article, we investigate the admissible ansatzes proposed by the RL-
agent from the perspective of quantum information theory Wilde (2013). For the
investigation, we specifically focus on the reinforcement learning enhanced
variational quantum state diagonalisation (RL-VQSD) problem Kundu  _et al._
(2024). In the VQSD algorithm LaRose  _et al._ (2019), the problem-inspired
ansatz is based on density matrix exponentiation Lloyd  _et al._ (2014), but
its depth and number of gates increase exponentially with the number of
qubits. Hence, searching for an optimal ansatz (with a minimum number of gates
and depth) that diagonalises a quantum state is an open research problem. In
RL-VQSD, the search for the optimal configuration of ansatz is done by using a
double deep-Q network (DDQN). See Appendix B for further details on the VQSD
algorithm, the agent and environment specifications. The RL-agent proposes
$E_{s}<E_{tot}$ admissible ansatzes that solves the VQSD problem.

Our investigation shows that the concurrence of the $E_{s}$ ansatz lies
between an upper and lower bound, independent of the different initialisation
of the double deep-Q network (DDQN), where each initialisation of DDQN
corresponds to a different random initialisation of the weights. Moreover, we
show that the upper and lower bounds are anti-correlated with respect to the
increment in the initial state entanglement. The anti-correlation turns into a
mild, and later strong, correlation when the initial state entanglement goes
beyond $0.322$, indicating a phase change in correlation between the upper and
lower bounds of the concurrence. The phase transition provides us with an in-
depth insight into the relation between the input and the output of the VQSD
algorithm. Further, our investigation reveals that the RL-agent can generate
high concurrence admissible ansatz with fewer 2-qubit gates and circuit depth
compared to ansatzes with very low concurrence. This is used as a guiding
principle to design a two-part ansatz, where an entanglement enhancing initial
block adjusts the input state’s concurrence thereby enhancing the performance
of the RL-agent by a factor of $2$. Furthermore, we quantify the contribution
of individual qubits of the RL-ansatz using conditional quantum entropy. Using
such a measure, we observe a correlation between entanglement and quantum
entropy, revealing why RL-VQSD can efficiently find the largest eigenvalues
but fails to find the smallest ones.

The structure of the paper is as follows. We present an overview of our main
results in Sec. II. In Sec. III.1, we analyse the upper and lower bounds on
the concurrence of the admissible ansatzes that solve the VQSD problem.
Specifically, in Sec. III.1.1 we present the variation of the entanglement
bounds for different weight initialisation of the DDQN. We observe the
correlation properties of the entanglement bounds dependent on the concurrence
of the initial state in Sec. III.1.2. In Sec. III.1.3, we present the
dependence of the entanglement bounds on the number of gates and circuit depth
of the ansatz. The analysis of the contribution of individual qubits to the
admissible ansatz is provided in Sec. III.2. In particular, we present the
contribution of each qubit for change in the entanglement of the ansatz in
Sec. III.2.1. In Sec. III.2.2, we explain why RL-VQSD can efficiently find the
largest eigenvalues of the initial state but not the smallest eigenvalues. We
provide concluding remarks and discuss possible future directions in Sec. IV.

##  II Contributions

Through the following points, we summarise the contribution of the paper.

  1. 1.

Demonstrating that the entanglement of the states generated by the RL-agent-
proposed admissible ansatzes lies within a lower and upper concurrence bound.
The upper and lower bounds of concurrence remain consistent across various
weight initialisation of the DDQN. Furthermore, we show that these bounds are
anti-correlated with respect to increasing entanglement until concurrence
surpasses $0.322$; after this point, the anti-correlation turns into
correlation. Hence, we introduce the concurrence $0.322$ as the phase change
point.

  2. 2.

Investigation of ansatzes in upper and lower bounds reveals that the optimal
ansatz for VQSD generates strongly entangled states. The RL-agent requires
fewer gates and circuit depth to generate such states to produce strongly
entangled admissible ansatzes than weakly entangled ansatzes. We then utilise
this information to minimise RL-agent and environment (which is the
configuration of ansatz) interaction, thereby optimising the convergence time
of the RL-VQSD. Specifically, we propose a two-part ansatz, where an
entanglement enhancing initial block adjusts the input state’s concurrence
thereby enhancing the performance of the RL-agent by a factor of $2$.

  3. 3.

We evaluate individual qubits’ contribution in diagonalising 2-qubit quantum
states using conditional quantum entropy. Our investigation shows that the
qubits have equal contributions on average in diagonalising random quantum
states.

  4. 4.

We further investigate the correlation in conditional entropy between qubits
for different inferred eigenvalues. It requires mild anti-correlation for the
first two largest eigenvalues and strong anti-correlation for the smallest
eigenvalues. Most admissible ansatzes lie in a mild anti-correlation regime,
explaining VQSD’s limitation in finding the smallest eigenvalues.

##  III Main results

In this section, we investigate the upper and lower bounds of concurrence of
the ansatzes produced by the RL agent in diagonalising $2$-qubit random
quantum states sampled from IBM qiskit’s random_density_matrix module. For the
simulation, we consider multiple such quantum states (specified in the caption
of figures), and for each state, we use $10000$ episodes of RL-VQSD to
diagonalise the state. The configuration of the RL-VQSD is provided in detail
in Appendix B.

###  III.1 Entanglement

####  III.1.1 DDQN weight independent entanglement bound

To benchmark the performance of the RL-agent for a specific problem, it is
ideal to take multiple initialisations of the DDQN (see Appendix B for a brief
description).

state | max. concurrence | min. concurrence
---|---|---
no. | avg. | std. div | avg. | std. div.
1 | 0.874 | 0.002 | 0.183 | 0.004
2 | 0.849 | 0.002 | 0.108 | 0.028
3 | 0.975 | 0.001 | 0.084 | 0.006
4 | 0.936 | 0.002 | 0.118 | 0.009
5 | 0.880 | 0.001 | 0.188 | 0.013
6 | 0.911 | 0.003 | 0.156 | 0.024
7 | 0.933 | 0.001 | 0.235 | 0.013
8 | 0.928 | 0.005 | 0.208 | 0.013
9 | 0.831 | 0.011 | 0.302 | 0.010
Table 1: The average and standard deviation of the final state’s maximum (max.
concurrence) and minimum concurrence (min. concurrence) after evolution
through the circuit for five different initialisations of the DDQN weights for
different quantum seed states. To obtain the results, we run RL-VQSD to
diagonalize $2$-qubit random quantum states for $10000$ episodes. The agent
then proposes $E_{s}<10000$ admissible ansatzes, which generate $E_{s}$ states
after the random quantum state evolves them. The max. (min.) concurrence is
attributed to the ansatz that generates a state with the highest (lowest)
concurrence following the evolution of the random quantum state.

To efficiently investigate the upper and lower bounds of concurrence Hill and
Wootters (1997); Wootters (1998) of the admissible ansatz, we calculate the
concurrence of admissible ansatz, which is defined by the entanglement induced
by the admissible ansatz when initialised with $\ket{00}$. We first sample
different random quantum states, then diagonalise them using RL-VQSD Kundu
_et al._ (2024) for five different initialisations of DDQN weights.
Thereafter, the agent suggests different possible configurations for the RL-
ansatz, which diagonalises the sampled random quantum states. For a specific
state, we collect such valid quantum circuits and find the concurrence of the
state obtained after evolving through the RL-ansatz. It should be noted that
finding the concurrence of the evolved state indicates how much the RL-ansatz
induces entanglement to the input sampled state. See Appendix C for further
analysis of the sampled random quantum states, where we show that in qiskit,
random_density_matrix even for $100000$ seeds, the probability of sampling
states with concurrence greater than $0.6$ is very low.

The results are presented in Table 1. We observe that the optimal
configuration of the RL-ansatz always gives us a state with concurrence close
to one. Hence, the deviation in the maximum concurrence over different weights
of the DDQN is more significant than the deviation in the minima of the
concurrence. As seen in Table 1, the deviation in the max concurrence is in
the order of $10^{-3}$. The low standard deviation implies that there is a
negligible deviation in concurrence across different initialisation of the
DDQN for a specific state. This indicates that the bounds on concurrence are
invariant for different configurations of the RL-ansatz. Hence, proving that
the entanglement bounds of the RL-ansatz are independent of initialisation of
DDQNs. Due to this observation, it is sufficient to consider a single
initialisation of DDQN to benchmark the outcomes in the remaining simulations.

In the following subsection, we investigate how the upper and lower bounds on
the entanglement of the RL-ansatz depend on the entanglement of the input
quantum state.

####  III.1.2 Anti-correlation between entanglement bounds

Corresponding to a predefined threshold, the agent proposes different
architectures of quantum circuits that diagonalise the quantum state. To
observe the amount of entanglement generated by the RL-ansatz we calculate the
concurrence of the state after passing through the RL-ansatz, just before the
dephasing operation. By sorting out the quantum circuit with maximum and
minimum entanglement, we observe their variation with the entanglement of the
input state. It has been proved in the previous section that the upper and
lower bounds of entanglement are independent of different initialisation of
DDQNs; hence, for further benchmarking, we can constrain ourselves to just one
initialisation of the DDQN weights.

Figure 1: Reciprocal behaviour in the upper and lower bounds of concurrence
with respect to the increasing concurrence in the input random state. For the
first eleven points, the Pearson correlation coefficient (PCC) is $-0.982$,
denoting a strong anti-correlation between the upper and lower bounds of
concurrence. The strong anti-correlation turns into a mild correlation after
concurrence of the initial state surpasses $0.322$. The correlation after the
first eleven points changes from $-0.055$ to $0.049$. There is a phase
transition from weak anti-correlation to weak correlation.

In Figure 1, we illustrate the variation of the entanglement bounds with the
entanglement of the input quantum state. To investigate the dependency, we use
the Pearson correlation coefficient (PCC) Lee Rodgers and Nicewander (1988) as
a quantifier of correlation. See Appendix A for a brief description. We see
that the upper and lower bounds of entanglement are initially strongly anti-
correlation when the input state entanglement is below $0.322$ and then
strongly correlate afterwards, indicating a phase change in the upper and
lower bounds of entanglement.

Figure 2: We present the phase transition of correlation between the upper and
lower bounds of concurrence for the admissible RL-VQSD ansatz. To show this,
we split the total range of the concurrence of the input state in two
intervals $\mathbbm{u}_{1}\coloneqq[i,k)$ and $\mathbbm{u}_{2}\coloneqq[k,j]$,
where $k$ is a free parameter and plot the variation of the Pearson
correlation coefficient between the upper and lower bound on the concurrence
of the ansatz corresponding to these two intervals. We denote by
$\textrm{PCC}_{ik}$ and $\textrm{PCC}_{kj}$ the correlation coefficients in
the intervals $\mathbbm{u}_{1}$ and $\mathbbm{u}_{2}$ respectively. The phase
transition point is observed at $k=0.322$ where the concurrence bounds change
from mild anti-correlation to mild correlation for the interval
$\mathbbm{u}_{2}$ while remaining strongly anti-correlated in the interval
$\mathbbm{u}_{1}$.

Next, we obtain the variation of PCC between the upper and lower bound on the
concurrence of the initial state. Specifically, we introduce a parameter
$k\in(i,j)$ to split the range of initial state concurrence
$\mathbbm{u}\coloneqq[i,j]$ into two intervals:
$\mathbbm{u}_{1}\coloneqq[i,k)$ and $\mathbbm{u}_{2}\coloneqq[k,j]$. Also, let
us denote the set of lower and upper bounds on the concurrence of the ansatz
as $u$ and $l$, respectively. For both the ranges $\mathbbm{u}_{1}$ and
$\mathbbm{u}_{2}$, we introduce the following correlation function:

$\displaystyle\eta^{k}_{w_{ik}}=w_{ik}\textrm{PCC}_{ik}+(1-w_{ik})\textrm{PCC}_{kj},$
(1)

where $w_{ik}\in\\{0,1\\}$ and $\textrm{PCC}_{ik}$, $\textrm{PCC}_{kj}$ are
the value of PCC between the upper and lower bounds of concurrence of the
ansatz for the intervals $\mathbbm{u}_{1}$ and $\mathbbm{u}_{2}$ respectively.
For the sake of simplicity we denote $\textrm{PCC}_{ik}$ as
$\textrm{PCC}_{ik}^{ul}$ and also $\textrm{PCC}_{kj}$ as
$\textrm{PCC}_{kj}^{ul}$.

######  Note 1.

We observe that depending on the values of $w_{ik}$, $\eta^{k}_{w_{ik}}$ can
take values either $\textrm{PCC}_{ik}$ or $\textrm{PCC}_{kj}$. For
$\eta^{k}_{w_{ik}}=\textrm{PCC}_{ik}$, we obtain the amount of
(anti-)correlation between the lower and upper bounds of concurrence of the
ansatz for the interval $\mathbbm{u}_{1}$. While for
$\eta^{k}_{w_{ik}}=\textrm{PCC}_{kj}$, we obtain the amount of
(anti-)correlation for the interval $\mathbbm{u}_{2}$. We leave proving
$\eta^{k}_{w_{ik}}$ is a valid measure of correlation for arbitrary values of
$w_{ik}\in(0,1)$ as an open problem.

We present in Figure 2, the variation of $\eta_{0}^{k}$ (in blue) and
$\eta_{1}^{k}$ (in red) for different values of $k$. We observe that
$\eta_{1}^{k}\approx-1~{}\forall k\in(i,j)$. This indicates strong cumulative
anti-correlation between the upper and lower bounds of concurrence. For
$\eta_{0}^{k}$, we observe a gradual change of $\text{PCC}_{kj}$ from -0.582
to 1. We note that $\eta_{0}^{k}$ changes from $-0.0451$ to $0.279$ for
$k\in[0.313,0.379]$ with $\eta_{0}^{k}=0$ at $k=k_{\ast}=0.322$. We call this
point the phase transition point of correlation between upper and lower bounds
of concurrence for the VQSD ansatz. At values of $k<k_{\ast}$, there is anti-
correlation between the upper and lower bounds of concurrence of the ansatz
for both the intervals $\mathbbm{u}_{1}$ and $\mathbbm{u}_{2}$. While for
values of $k>k_{\ast}$, there is (anti-)correlation between the upper and
lower bounds of concurrence of the ansatz for both the intervals
$(\mathbbm{u}_{2})~{}\mathbbm{u}_{1}$.

####  III.1.3 Enhancing the performance of agent based on initial state
entanglement

Here, we provide a detailed investigation of the admissible ansatzes produced
by the RL-agent in terms of ansatz depth, the total number of 2-qubit gates
and the total number of gates in the upper and lower bounds of concurrence.

Figure 3: The RL-agent can generate an ansatz with strong entanglement with
fewer 2-qubit gates and circuit depth compared to the ansatzes with weak
entanglement. As the main aim of RL-VQSD is to diagonalise unitary with a
small number of gates and depth, our analysis shows that if we particularly
focus on admissible ansatzes near the upper bound of concurrence, we can not
only reduce the computational cost of RL-VQSD severely but find optimal
admissible ansatzes. The numbers in the bars correspond to the amount of
entanglement generated by the RL-ansatz when initialised in the vacuum state.

Our investigation reveals that the admissible ansatzes corresponding to the
upper bound of concurrence has, on average, a shorter depth of $13.23$ and a
smaller number of gates of $18.5$ compared to the admissible ansatzes
corresponding to the lower bound of concurrence where the depth is $13.9$ and
number of gates is $19$ over $25$ random quantum states. Moreover, Figure 3
makes it more prominent that the minimum number of gates and the depth of the
admissible ansatzes are smaller for the upper bound of concurrence than the
lower. This observation is crucial in reducing the subspace of all possible
admissible ansatzes. The main motivation of RL-QAS in VQAs is to find ansatzes
that solve a VQA with a minimum number of gates and circuit depth; in the case
of RL-VQSD, this observation can help us to narrow down the search for an
optimal admissible ansatz in the two following ways.

#### a. Entanglement dependent initialisation of RL-state

In RL-VQSD Kundu  _et al._ (2024), the RL-state that encodes the ansatz
initiates from an empty circuit (i.e., in circuit 2 $U_{\text{RL-
agent}}=\mathds{1}$). We showed that the optimal admissible ansatz, on
average, lies in the upper bound of concurrence, motivating us to initialise
the quantum state (that is diagonalised using RL-VQSD) closest to the upper
bound of entanglement. To do this, the ansatz is divided into two parts

$\small\quantikz\lstick[2]{\rho}&\gate{H}\gategroup[2,steps=3,style={dashed,roundedcorners,innerxsep=2pt},background,labelstyle={labelposition=below,anchor=north,yshift=-0.2cm}]{{\sc\small
EntanglementEnhancingBlock}}\ctrl{1}\gate{H}\gate[2]{U_{\text{RL-ansatz}}}\\\
\gate{CRX(\theta)},$ (2)

where the first, Entanglement Enhancing (EE) block, helps the diagonalising
state to achieve the highest entanglement before it is fed to the second part,
$U_{\text{RL-agent}}$, which is decided by the RL-agent. For example, for seed
$27$ (see Appendix C for details), we get a random quantum state with
concurrence $0.198$, and using the EE, we can increase the input state
concurrence to $0.215$ at $\theta=0.5$ as shown in Fig. 4.

Figure 4: The concurrence of input state with $\theta$ of CRX($\theta$) gate
in Entanglement Enhancing increases the concurrence of the input random state
from $0.198$ to $0.215$ when $\theta=0.5$.

Our investigation reveals that turning on the EE block with $\theta=0.5$
enhances the performance of the RL-agent by $2$-times compared to the default
setting at $\theta=0$. The results are summarised in the Tab. 2 where we see
that the number of successful episodes and the reward acquired (where the
reward function is in the form defined in Eq. 12) by the agent in the RL-VQSD
process increase by $2$-fold while the EE block is activated for $\theta=0.5$
compared to default settings at $\theta=0$.

Table 2: Turning on the Entanglement Enhancing (EE) mode (denoted as Enhanced ($\theta=0.5$)) enhances the learning of the performance of the RL-agent in the absence of the EE mode (denoted as Default ($\theta=0$)). In the first $3500$, the Enhanced ($\theta=0.5$) collects $2.08$ times more reward (tot. rwd. coll.) and $2.22$ times more successful episodes (Tot. succ. ep.) than Default ($\theta=0$). Moreover, the average number of 2-qubit (2q gate) and depth over the first $100$ successful episodes reduced in the case of Enhanced ($\theta=0.5$) settings compared to Default ($\theta=0$), showing enhanced learning of the RL-agent for entanglement enhanced input state. EE mode | Tot. succ. | Tot. rwd. | 1st 100 succ. ep. (avg.)
---|---|---|---
($\theta$) | ep. | coll. | 1q gate | 2q gate | depth
Enhanced ($\theta=0.5$) | $858$ | $4.45\times 10^{5}$ | $15.41$ | $4.02$ | $14.6$
Default ($\theta=0$) | $387$ | $2.14\times 10^{5}$ | $14.71$ | $5.29$ | $15.59$

Meanwhile, in the Tab. 3, we show that the EE block also helps to optimise the
cost function and the minimum number of 2-qubit gates in the optimal
admissible ansatz compared to the default settings.

Table 3: Turning on the Entanglement Enhancing (EE) mode (denoted as Enhanced ($\theta=0.5$)) optimises the number of minimum number of 2-qubit gates (Min. 2q gate) in RL-ansatz better than in the absence of the EE mode (denoted as Default ($\theta=0$)). Moreover, the Enhanced ($\theta=0.5$) mode minimises the cost function better than the Default ($\theta=0$) mode. EE mode ($\theta$) |  Cost function |  Min. 1q gate |  Min. 2q gate
---|---|---|---
Enhanced ($\theta=0.5$) | $5.12\times 10^{-7}$ | $14$ | $2$
Default ($\theta=0$) | $6.08\times 10^{-7}$ | $11$ | $5$

The aforementioned observations lead us to the conclusion that maximising the
entanglement of the input state through the initialisation strategy of Fig. 2
enhances the learning process of the RL-agent and facilitates the further
optimisation of the admissible ansatzes compared to the start-of-art Kundu
(2024).

###  III.2 Conditional quantum entropy

####  III.2.1 Contribution of individual qubits for change in entanglement

We quantify the contribution of each qubit using the conditional quantum
entropy $S_{q_{0}|q_{1}}^{\rho^{\prime}_{q_{0}q_{1}}}$ Cerf and Adami (1997,
1999); Wilde (2013). The conditional entropy of an individual qubit for an $N$
qubit quantum system indicates the amount of uncertainty remaining about that
qubit after measuring the whole system. Hence, it is essential to know how
much information the state of qubits provides about the state of the $N$
qubits. For a detailed discussion and notation, see Appendix A.

Figure 5: Relative change in the contribution of qubits 0 and 1 as a function
of change in entanglement. We quantify the relative contribution of individual
qubits to the RL-VQSD ansatz using $\Delta_{q_{0}}^{q_{1}}$ (see Eq. (3)). For
the change in concurrence due to the ansatz, the distribution of
$\Delta_{q_{0}}^{q_{1}}$ is uniform with respect to
$\Delta_{q_{0}}^{q_{1}}=0$. The magnitude of the cumulative weight of all data
points in the region $\Delta_{q_{0}}^{q_{1}}<0$ is $4.854$ while that for data
points in the region $\Delta_{q_{0}}^{q_{1}}>0$ is $5.524$. This indicates
equal contribution from both the qubits to the RL-VQSD ansatz for the range of
change in entanglement.

Let the change in the conditional entropy of the qubit $q_{0}(q_{1})$ after
evolving through the VQSD ansatz be given by $\Delta
S^{\rho\to{\rho^{\prime}}}_{q_{0}|q_{1}}(\Delta
S^{\rho\to{\rho^{\prime}}}_{q_{1}|q_{0}})$. We quantify the relative
contribution of individual qubits via the quantity:

$\displaystyle\Delta_{q_{0}}^{q_{1}}\coloneqq\Delta
S^{\rho\to{\rho^{\prime}}}_{q_{0}|q_{1}}-\Delta
S^{\rho\to{\rho^{\prime}}}_{q_{1}|q_{0}}.$ (3)

We note that $\Delta_{q_{0}}^{q_{1}}>0$ implies a greater change in
conditional entropy of $q_{0}$ as compared to $q_{1}$, and hence the ansatz
has a greater contribution from $q_{0}$ as compared to $q_{1}$. Following a
similar reasoning $\Delta_{q_{0}}^{q_{1}}<0$ implies $q_{1}$ has greater
contribution as compared to $q_{0}$, while $\Delta_{q_{0}}^{q_{1}}=0$ implies
both $q_{0}$ and $q_{1}$ contributes equally to the ansatz.

We present the variation of $\Delta_{q_{0}}^{q_{1}}$ for the change in
concurrence due to the ansatz given by $\Delta_{c}\coloneqq
C^{\rho_{q_{0}q_{1}}}-C^{\rho^{\prime}_{q_{0}q_{1}}}$ in Figure 5. We observe
that the distribution of $\Delta_{q_{0}}^{q_{1}}$ is bounded in the interval
$[-0.7,0.7]$ and is uniform with respect to $\Delta_{q_{0}}^{q_{1}}=0$. The
magnitude of the cumulative weight of all data points in the region
$\Delta_{q_{0}}^{q_{1}}<0$ is $4.854$ while that for data points in the region
$\Delta_{q_{0}}^{q_{1}}>0$ is $5.524$. This indicates equal contribution from
both the qubits to the RL-VQSD ansatz for the range of change in entanglement.
In the region of $\Delta_{q_{0}}^{q_{1}}>0$, the relative contribution of
$q_{0}$ first increases and then decreases. However, no such pattern is
observed for the region of $\Delta_{q_{0}}^{q_{1}}<0$. When $\Delta_{c}>0.4$,
the value of $\Delta_{q_{0}}^{q_{1}}$ monotonically decreases with increase in
$\Delta_{c}$. Also, for $\Delta_{c}<0.2$, $\Delta_{q_{0}}^{q_{1}}$ increases
monotonically with increase in $\Delta_{c}$.

####  III.2.2 Problem with variational quantum state diagonalisation

In this section, we investigate the contribution of individual qubits to the
performance of the RL-VQSD algorithm. Through this investigation, we get a
deeper insight into why the variational quantum state diagonalization
algorithm can not find the smallest eigenvalues.

We present the main observations in Figure 6, where we plot the four
eigenvalues of $21$, $2$-qubit Haar random mixed quantum states with the
correlation in conditional entropy among the qubits. For an individual quantum
state, the RL-agent proposes $\mathbf{s}$ different structures of admissible
ansatzes. Then we get $\rho_{i}^{\prime}$ states where $i\in[0,\mathbf{s}]$.
Afterwards, we calculate the conditional entropy of individual qubits for each
state and take the median.111Calculating the median over the mean is justified
by the broad range of conditional entropy distribution, which makes the median
a more appropriate measure of central tendency. Then, we calculate the
correction in conditional entropy among the qubits using PCC.

In Figure 6, we observe that a mild anti-correlation between the qubits is
required to find the two largest eigenvalues. As the magnitude of the largest
eigenvalues decreases, the mild anti-correlation turns into a strong
correlation between the qubits. This indicates a strong correlation between
the qubits is required to achieve the two largest eigenvalues. Meanwhile, we
require a strong anti-correlation between the qubits to find the smallest
eigenvalues of the same states. Then, as the magnitude of the smallest
eigenvalue increases, the strong anti-correlation turns into mild anti-
correlation.

Figure 6: The conditional entropy correlation between the first and the second
qubit decreases as the magnitude of the largest and the second largest
eigenvalues increases. Meanwhile, the same correlation between the qubits
increases with the magnitude of the third and the smallest eigenvalues. The
RL-ansatz proposes specific structures of admissible ansatzes, and for each
such ansatz, it is not possible to have two different kinds of correlations.
Hence, our investigation suggests that it is not feasible to find the smallest
eigenvalues with the same ansatz that provides a good approximation of the
largest eigenvalues. The correlation of conditional entropy between qubits is
calculated using Pearson correlation coefficient (PCC) and is the median over
all the configuration of RL-ansatz proposed by the agent. The $\bigstar$
signifies the eigenvalues corresponding to the same state.

We recall that the correlation is calculated among the qubits based on the
conditional entropy of individual qubits. As it is not possible to have two
different kinds (such as mild anti-correlation and strong anti-correlation at
the same time) of correlation between the qubits in the same ansatz structure,
we can not find the largest and smallest eigenvalues utilising the same
ansatz. For example, we mark the eigenvalues of a state with $\bigstar$, where
we observe that to find the largest two eigenvalues, the RL-ansatz need to
induce a mild anti-correlation between the two qubits; meanwhile, to obtain
the smallest eigenvalues, we require strong anti-correlation.

On the contrary, for the algorithm, as the agent adds a new gate to the RL-
ansatz, the correlation in conditional entropy among the qubits increases.
That is why it is easier for the RL-ansatz to propose admissible ansatz with
mild anti-correlation. We further certify this observation in Figure 7, where
we observe the correlation in conditional entropy between qubits for $21$
random quantum states. We see that most of the admissible ansatz provide a
mild anti-correlation among the qubits, indicating that the VQSD algorithm
primarily focuses on finding the largest eigenvalues rather than the smallest
ones. Hence, it is easier for the variational diagonalisation algorithm to
find the largest eigenvalues more efficiently than the smallest ones.

Figure 7: The correlation in conditional entropy among qubits is in the range
of mild correlation for $21$ random quantum states. This observation, along
with the observation in Figure 6 certify that RL-agent only proposes a
particular configuration of admissible ansatz that can find only the largest
eigenvalues. The average correlation among all the $21$ states is $-0.4263$
(horizontal black line), which is in the regime of mild anti-correlation.

##  IV Discussion

In this article, we use tools from quantum information theory to analyse the
admissible ansatzes proposed by the RL-agent for solving reinforcement
learning-based quantum architecture search problems, and we particularly
investigate the RL-agent proposed ansatzes in solving recently proposed RL-
VQSD problem.

We observe that the concurrence of the admissible ansatz ranges between an
upper and lower bound, which are independent of the initialisation of the
weights of the deep-Q network. The upper and lower bounds are initially anti-
correlated with respect to the initial state entanglement. The anti-
correlation turns into a mild and eventually strong correlation as the
entanglement of the initial state surpasses beyond a phase change point at
concurrence $0.322$. We also see that the optimal configuration of the
admissible ansatz lies in the upper bound of concurrence, which has a smaller
circuit depth and requires fewer gates than the lower bound. These
observations provide insight regarding the relation between the entanglement
of the initial state and the entanglement generated by the RL-VQSD admissible
ansatzes. We effectively utilise these observations to greatly reduce the RL-
agent and environment interaction, i.e., the computational time needed to
solve the RL-VQSD problem. Specifically, we propose an entanglement enhancing
initial block that adjusts the input state’s concurrence thereby enhancing the
performance of the RL-agent by a factor of $2$ Additionally, the admissible
ansatzes not only diagonalise the random quantum states with high accuracy but
can be used to generate close to maximum entangled states from the vacuum
state.

Furthermore, we quantify the contribution of each qubit in the admissible
ansatz using conditional quantum entropy. For the admissible RL-VQSD ansatz,
we observe an equal contribution from each qubit for the full range of change
in the entanglement of the state due to the ansatz. Focusing on the task of
obtaining the eigenvalues of the starting state, we observe that a strong
correlation between the qubits of the admissible ansatz is required to obtain
the two largest eigenvalues, while strong anti-correlation between the qubits
is needed to find the smallest eigenvalues of the same state. Noting that it
is impossible to have both strong correlation and anti-correlation between the
qubits of the same ansatz, we conclude that we cannot simultaneously obtain
both the largest and smallest eigenvalues from the RL-VQSD admissible ansatz.
From the observation, most of the admissible ansatz has mild anti-correlation
among the qubits, which explains why it is easier for the diagonalisation
algorithm to find the largest eigenvalues than the smallest ones.

For future work, it would be interesting to conduct more comprehensive study
of the implication of introducing entanglement Horodecki  _et al._ (2009) or
nonlocality Brunner  _et al._ (2014); Sadhu and Das (2023)-based reward
functions of the RL-agent in QAS. Specifically, given that most of the
admissible ansatzes have high values of concurrence, it is expected that an
entanglement or non-locality-based reward function may further reduce the time
required to find the optimal configurations. Also, we may expect this to
provide an optimal quantum circuit that provides both the lowest and highest
eigenvalues of the input quantum state.

###### Acknowledgements.

A. Sadhu acknowledges the Ph.D. fellowship from the Raman Research Institute,
Bangalore. AK would like to thank Jarosław Miszczak and Mateusz Ostaszewski
for their valuable comments on the manuscript and discussions. This research
was partially supported by PL-Grid Infrastructure Grant Number
PLG/2023/016205.

##  Appendix A Preliminaries

We assume that every quantum system say $S$ has an associated Hilbert space
$\mathscr{H}_{S}$ of finite dimension $d_{S}$. The state of a quantum system
$S$ is represented by a density operator $\rho_{S}$ defined on
$\mathscr{H}_{S}$ and satisfies the conditions (a) Positivity: $\rho_{S}\geq
0$, (b) Hermitian: $\rho_{S}=\rho_{S}^{\dagger}$ and (c) Trace Preserving:
$\operatorname{Tr}[\rho_{S}]=1$. We denote the set of density operators of $S$
as $\mathscr{D}(\mathscr{H}_{S})$. A pure state is a rank-one density operator
given by $\rho_{S}\coloneqq\outerproduct{\psi}{\psi}_{A}$, where
$\ket{\psi}_{S}\in\mathscr{H}_{S}$ is a unit-norm vector in $\mathscr{H}_{S}$.
We denote the density operator of a composite system $S_{1}S_{2}$ as
$\rho_{S_{1}S_{2}}\in\mathscr{H}_{S_{1}S_{2}}$ with
$\operatorname{Tr}_{S_{2}}\rho_{S_{1}S_{2}}=\rho_{S_{1}}\in\mathscr{H}_{S_{1}}$
being the reduced state of $S_{1}$. The state
$\rho_{S_{1}S_{2}}\in\mathscr{H}_{S_{1}S_{2}}$ is called separable if it
cannot be expressed in the form of
$\rho_{S_{1}S_{2}}=\sum_{x}p_{x}\rho_{S_{1}}^{x}\otimes\rho_{S_{2}}^{x}$,
where $\\{\rho_{S_{1}}^{x}\\}_{x}$ and $\\{\rho_{S_{2}}^{x}\\}_{x}$ are sets
of pure states, $p_{x}\in[0,1]$ and $\sum_{x}p_{x}=1$. The states that cannot
be expressed in the above form are said to be entangled Horodecki  _et al._
(2009). Next, we briefly discuss a relevant measure of entanglement for this
work.

###  A.1 Concurrence

We quantify the amount of entanglement present in a bipartite quantum state
$\rho_{AB}\in\mathscr{D}(\mathscr{H}_{A}\otimes\mathscr{H}_{B})$ using
concurrence Hill and Wootters (1997); Wootters (1998) which is defined as:

$\displaystyle
C^{\rho_{AB}}=\max\\{0,\lambda^{e}_{1}-\lambda^{e}_{2}-\lambda^{e}_{3}-\lambda^{e}_{4}\\}$
(4)

where $\lambda^{e}_{s}$ are the square root of the eigenvalues of
$\rho_{AB}\tilde{\rho}_{AB}$ in descending order with
$\tilde{\rho}_{AB}=(\sigma_{y}\otimes\sigma_{y})\rho^{*}_{AB}(\sigma_{y}\otimes\sigma_{y})$
being the spin-flipped state of $\rho_{AB}$ and $\sigma_{y}$ being a Pauli
spin matrix. For the state $\rho_{AB}=\outerproduct{\psi}{\psi}_{AB}$, the
concurrence is defined as

$\displaystyle\mathcal{C}^{\ket{\psi}_{AB}}=\sqrt{2[1-\operatorname{Tr}\rho_{A}^{2}]},$
(5)

where $\rho_{A}=\operatorname{Tr}_{B}\rho_{AB}$ is a subsystem of the combined
state. It is easy to see that the concurrence of a pure bipartite qubit
maximally entangled state is one, while that of a bipartite qubit separable
state is zero. In the following subsection, we discuss a measure to quantify
the contribution of individual qubits in a quantum circuit.

###  A.2 Conditional quantum entropy

We quantify the contribution of each qubit in a quantum circuit using the
conditional quantum entropy Cerf and Adami (1997, 1999), which is defined as
follows.

######  Definition 1 (Conditional quantum entropy Wilde (2013)).

For the state
$\rho^{\prime}_{AB}\in\mathscr{D}(\mathscr{H}_{A}\otimes\mathscr{H}_{B})$, the
conditional quantum entropy $S_{A|B}^{\rho^{\prime}_{AB}}$ is equal to the
difference of the joint quantum entropy $S^{\rho^{\prime}_{AB}}_{AB}$ and the
marginal entropy $S^{\rho^{\prime}_{AB}}_{B}$:

$S_{A|B}^{\rho^{\prime}_{AB}}\coloneqq
S^{\rho^{\prime}_{AB}}_{AB}-S^{\rho^{\prime}_{AB}}_{B}.$ (6)

Operationally, the conditional entropy $S_{A|B}^{\rho^{\prime}_{AB}}$ of the
state for the state $\rho^{\prime}_{AB}$ signifies how mixed the marginal
state $\rho^{\prime}_{A}$ ($\rho^{\prime}_{B}$) is as compared
$\rho^{\prime}_{AB}$. For more details of the following theorems and
observations, please refer Wilde (2013).

######  Observation 1.

For the state
$\rho^{\prime}_{AB}\in\mathscr{D}(\mathscr{H}_{A}\otimes\mathscr{H}_{B})$,
consider a purification
$\ket{\psi}_{ABE}\in\mathscr{H}_{A}\otimes\mathscr{H}_{B}\mathscr{H}_{E}$. It
then follows that:

$\displaystyle S_{A|B}^{\rho^{\prime}_{AB}}=S_{E}^{\psi}-S_{B}^{\psi}.$ (7)

The conditional quantum information thus measures the difference in the
entropy of the states $\rho_{E}\in\mathscr{D}(\mathscr{H}_{E})$ and
$\rho_{B}\in\mathscr{D}(\mathscr{H}_{B})$.

######  Theorem 1.

For the state
$\rho^{\prime}_{AB}\in\mathscr{D}(\mathscr{H}_{A}\otimes\mathscr{H}_{B})$, the
following inequality applies to the marginal entropy
$S_{A}^{\rho^{\prime}_{AB}}$ and the conditional quantum entropy
$S_{A|B}^{\rho^{\prime}_{AB}}$:

$\displaystyle S_{A}^{\rho^{\prime}_{AB}}\geq S_{A|B}^{\rho^{\prime}_{AB}}.$
(8)

The above theorem states that conditioning cannot increase entropy. The value
of conditional entropy can be positive or negative depending on the quantum
state for which it is evaluated. However, it cannot be arbitrarily large or
arbitrarily small. The following theorem places an upper bound on the absolute
value of the conditional quantum entropy.

######  Theorem 2.

The conditional entropy is bounded as follows:

$\displaystyle|S_{A|B}^{\rho^{\prime}_{AB}}|\leq\log\dim(\mathscr{H}_{A})$ (9)

The bound is saturated for
$\rho^{\prime}_{AB}=\mathbbm{1}_{A}\otimes\sigma_{B}$, where $\mathbbm{1}_{A}$
is the maximally mixed state and $\sigma_{B}\in\mathscr{D}(\mathscr{H}_{B})$,
and also for the family of maximally entangled states.

In the following subsection, we discuss a measure to quantify the amount of
correlation between two random variables.

###  A.3 Correlation coefficient

We quantify the amount of correlation between two random variables $r_{1}$ and
$r_{2}$ using the Pearson correlation coefficient (PCC) Lee Rodgers and
Nicewander (1988) defined as:

$\displaystyle\textrm{PCC}^{r_{1}r_{2}}=\frac{\mathrm{E}[(r_{1}-\mu_{r_{1}})(r_{2}-\mu_{r_{2}})]}{\sigma_{r_{1}}\sigma_{r_{2}}},$
(10)

where $\mu_{r_{j}}$, $\sigma_{r_{j}}$ are the mean and standard deviation of
the distribution $r_{j}$ with $j\in\\{1,2\\}$ and $\mathrm{E}[.]$ is an
operator that returns the expectation value of a random variable. Note that
the Pearson correlation coefficient (PCC) is symmetric:
$\textrm{PCC}^{r_{1}r_{2}}=\textrm{PCC}^{r_{2}r_{1}}$. The value of PCC ranges
in between $-1$ to $+1$. A correlation value of $+1(-1)$ implies perfect
correlation (anti-correlation) between the two random variables $r_{1}$ and
$r_{2}$, while zero implies no linear dependence between the variables.

##  Appendix B The reinforcement learning enhanced variational quantum state
diagonalisation

The VQSD algorithm LaRose  _et al._ (2019) for a quantum state $\rho$
comprises three subroutines: (1) In the training subroutine, the parameters
$\vec{\theta}$ of a quantum gate sequence $U(\vec{\theta})$ are optimised for
a given state $\rho$. Ideally, after optimisation, the sequence
$U(\vec{\theta}_{\text{opt}})$ satisfies

$\rho^{\prime}=U(\vec{\theta}_{\text{opt}})\rho
U(\vec{\theta}_{\text{opt}})^{\dagger}=\rho_{\text{diag}},$

where $\rho_{\text{diag}}$ is $\rho$ diagonalised in its eigenbasis and
$\vec{\theta}_{\text{opt}}$ are the optimal angles. Classical gradient-based
methods such as SPSA and Gradient Descent, or gradient-free optimisation
Powell (1994, 2006), can be utilised in the training process. (2) In the
eigenvalue readout subroutine, using the optimised unitary
$U(\vec{\theta}_{\text{opt}})$ and one copy of state $\rho$, one can extract
all the eigenvalues for low-rank states or the largest eigenvalues for full-
rank states. This is achieved by measuring $\rho^{\prime}$ in the
computational basis, $\mathbf{b}=b_{1}b_{2}\ldots b_{n}$, as follows

$\lambda^{\prime}=\bra{\mathbf{b}}\rho^{\prime}\ket{\mathbf{b}},$

where $\lambda^{\prime}$ are inferred eigenvalues. (3) In the final step, we
prepare the eigenvectors associated with the largest eigenvalues. If
$\mathbf{b}^{\prime}$ is a bit string associated with $\lambda^{\prime}$, then
the inferred eigenvectors $|v_{\mathbf{b}^{\prime}}^{\prime}\rangle$ are
obtained as follows

$|{v}^{\prime}_{{\mathbf{b}}^{\prime}}\rangle=U(\theta_{\textrm{opt}})^{\dagger}\ket{{\mathbf{b}}^{\prime}}=U(\theta_{\textrm{opt}})^{\dagger}\left(X^{b_{1}}\otimes\ldots\otimes
X^{b_{n}}\right)\ket{\mathbf{0}}.$

In the reinforcement learning enhanced variational quantum state
diagonalisation, an RL-agent is utilised to find the optimal configuration of
$U(\vec{\theta})$. The agent contains a double deep-Q network (DDQN) Van
Hasselt  _et al._ (2016) with $\epsilon$-greedy policy. The DDQN optimises the
stability of Q-learning by using two networks to decouple the action selection
$a^{\prime}$ in the subsequent state $s^{\prime}$ from its current reward $r$
evaluation, leading to less overestimated Q-values and more stable and
reliable learning outcomes.

The key update equation is

$Q(s,a;\theta)\leftarrow Q(s,a;\theta)\\\ +\alpha[r+\gamma
Q(s^{\prime},\arg\max_{a^{\prime}}Q(s^{\prime},a^{\prime};\theta);\theta^{\prime})-Q(s,a;\theta)]$
(11)

where, $Q(s,a;\theta)$ is the predicted Q-value for state $s$ and action $a$,
parameterised by weights $\theta$ of the online network. $\alpha$ is the
learning rate and $\gamma$ is the discount factor. The settings of the
parameters are similar to the ref. Kundu  _et al._ (2024).

The optimal policy is obtained by optimising the weights of the DDQN using the
ADAM optimiser. In the RL setup, the RL-state encodes the ansatz and is
defined by a tensor-based encoding presented introduced in Patel  _et al._
(2024). After each environment and agent interaction, the reward function

$R=\left\\{\begin{array}[]{ll}+\mathcal{R}&\text{for
}C_{t}(\vec{\theta})<\zeta+10^{-5}\\\
-\textrm{log}\left(C_{t}(\vec{\theta})-\zeta\right)&\text{for
}C_{t}(\vec{\theta})>\zeta\end{array}\right.,$ (12)

is calculated. The goal of the RL-agent is to reach the minimum error for a
predefined threshold $\zeta$ (which is a hyperparameter of the model) by
optimising the parameters of the $U(\vec{\theta})$ using the COBYLA optimiser.
The cost function

$C_{t}(\vec{\theta})=\text{Tr}(\rho^{2})-\text{Tr}(\rho_{t}(\vec{\theta})^{2}).$
(13)

at each step $t$ is calculated for the ansatz which outputs a state
$\rho_{t}(\vec{\theta})$.

##  Appendix C Analysis of the Haar random input quantum states

In this section, we analyse the 2-qubit quantum states sampled from
random_density_matrix of IBM qiskit’s quantum_info module. Here, we sample
$100000$ Haar random quantum states and observe how the eigenvalues and the
qubits’ conditional entropy change with the state’s entanglement.

Figure 8: The difference in conditional entropy converges towards the zero as
the entanglement of the sampled input state increases. Meanwhile, as the input
state gets closer to the maximally entangled state (with concurrence $1$), the
magnitude of the largest eigenvalue increases while the other eigenvalue
decreases.

In Fig. 8, see that the difference in conditional entropy in qubits of the
input state varies evenly on either side of $S_{AB}^{\rho_{AB}}=0$ where $A$
and $B$ are subsystems containing one qubit each and $\rho_{AB}$ is the
sampled Haar random state. Meanwhile, the area of the distribution of
$S_{AB}^{\rho_{AB}}$ on the ether side of zero entropy converges towards zero
as the input state entanglement increases. Furthermore, the magnitude of the
input quantum state’s largest eigenvalue increases with the increase in
entanglement, and all the other eigenvalues converge towards $0$. It also
shows that even for $100000$ seeds, the probability of sampling states with
concurrence greater than $0.6$ is very low, justifying the low density of
points after $0.6$ in Fig. 1.

##### Sampling a random quantum state

If $n$ is the number of qubits and $s$ is the random seed to sample, then we
can sample a state from random_density_matrix of IBM qiskit’s quantum_info
module (in python) as follows

$\texttt{random\\_density\\_matrix(2**n, seed=s)}.$ (14)

The state used in III.1, is sampled using $n=2$ and $s=27$

## References

  * Harrow and Montanaro (2017) Aram W Harrow and Ashley Montanaro, “Quantum computational supremacy,” Nature 549, 203–209 (2017).
  * Montanaro (2016) Ashley Montanaro, “Quantum algorithms: an overview,” npj Quantum Information 2, 1–8 (2016).
  * Sadhu  _et al._ (2023) Abhishek Sadhu, Meghana Ayyala Somayajula, Karol Horodecki, and Siddhartha Das, “Practical limitations on robustness and scalability of quantum internet,” arXiv preprint arXiv:2308.12739 (2023).
  * Zhang  _et al._ (2022) Shi-Xin Zhang, Chang-Yu Hsieh, Shengyu Zhang, and Hong Yao, “Differentiable quantum architecture search,” Quantum Science and Technology 7, 045023 (2022).
  * Du  _et al._ (2022) Yuxuan Du, Tao Huang, Shan You, Min-Hsiu Hsieh, and Dacheng Tao, “Quantum circuit architecture search for variational quantum algorithms,” npj Quantum Information 8, 62 (2022).
  * Kuo  _et al._ (2021) En-Jui Kuo, Yao-Lung L Fang, and Samuel Yen-Chi Chen, “Quantum architecture search via deep reinforcement learning,” arXiv preprint arXiv:2104.07715 (2021).
  * Zhang  _et al._ (2021) Shi-Xin Zhang, Chang-Yu Hsieh, Shengyu Zhang, and Hong Yao, “Neural predictor based quantum architecture search,” Machine Learning: Science and Technology 2, 045027 (2021).
  * McClean  _et al._ (2016) Jarrod R McClean, Jonathan Romero, Ryan Babbush, and Alán Aspuru-Guzik, “The theory of variational hybrid quantum-classical algorithms,” New Journal of Physics 18, 023023 (2016).
  * Cerezo  _et al._ (2021) Marco Cerezo, Andrew Arrasmith, Ryan Babbush, Simon C Benjamin, Suguru Endo, Keisuke Fujii, Jarrod R McClean, Kosuke Mitarai, Xiao Yuan, Lukasz Cincio, _et al._ , “Variational quantum algorithms,” Nature Reviews Physics 3, 625–644 (2021).
  * Sarkar (2022) Aritra Sarkar, “Automated quantum software engineering: why? what? how?” arXiv e-prints , arXiv–2212 (2022).
  * Ostaszewski  _et al._ (2021) Mateusz Ostaszewski, Lea M Trenkwalder, Wojciech Masarczyk, Eleanor Scerri, and Vedran Dunjko, “Reinforcement learning for optimization of variational quantum circuit architectures,” Advances in Neural Information Processing Systems 34, 18182–18194 (2021).
  * Kundu  _et al._ (2024) Akash Kundu, Przemysław Bedełek, Mateusz Ostaszewski, Onur Danaci, Yash J Patel, Vedran Dunjko, and Jarosław A Miszczak, “Enhancing variational quantum state diagonalization using reinforcement learning techniques,” New Journal of Physics 26, 013034 (2024).
  * Patel  _et al._ (2024) Yash J Patel, Akash Kundu, Mateusz Ostaszewski, Xavier Bonet-Monroig, Vedran Dunjko, and Onur Danaci, “Curriculum reinforcement learning for quantum architecture search under hardware errors,” arXiv preprint arXiv:2402.03500 (2024).
  * Sutton and Barto (2018) Richard S Sutton and Andrew G Barto,  _Reinforcement learning: An introduction_ (MIT press, 2018).
  * Wu  _et al._ (2023) Wenjie Wu, Ge Yan, Xudong Lu, Kaisen Pan, and Junchi Yan, “Quantumdarts: differentiable quantum architecture search for variational quantum algorithms,” in  _International Conference on Machine Learning_ (PMLR, 2023) pp. 37745–37764.
  * Kundu (2024) Akash Kundu, “Reinforcement learning-assisted quantum architecture search for variational quantum algorithms,” arXiv preprint arXiv:2402.13754 (2024).
  * LaRose  _et al._ (2019) Ryan LaRose, Arkin Tikku, Étude O’Neel-Judy, Lukasz Cincio, and Patrick J Coles, “Variational quantum state diagonalization,” npj Quantum Information 5, 57 (2019).
  * Tan and Volkoff (2021) Kok Chuan Tan and Tyler Volkoff, “Variational quantum algorithms to estimate rank, quantum entropies, fidelity, and fisher information via purity minimization,” Physical Review Research 3, 033251 (2021).
  * Cerezo  _et al._ (2020) Marco Cerezo, Alexander Poremba, Lukasz Cincio, and Patrick J Coles, “Variational quantum fidelity estimation,” Quantum 4, 248 (2020).
  * Kundu and Miszczak (2022) Akash Kundu and Jarosław Adam Miszczak, “Variational certification of quantum devices,” Quantum Science and Technology 7, 045017 (2022).
  * Kandala  _et al._ (2017) Abhinav Kandala, Antonio Mezzacapo, Kristan Temme, Maika Takita, Markus Brink, Jerry M Chow, and Jay M Gambetta, “Hardware-efficient variational quantum eigensolver for small molecules and quantum magnets,” nature 549, 242–246 (2017).
  * Mustafa  _et al._ (2022) Hasan Mustafa, Sai Nandan Morapakula, Prateek Jain, and Srinjoy Ganguly, “Variational quantum algorithms for chemical simulation and drug discovery,” in  _2022 International Conference on Trends in Quantum Computing and Emerging Business Technologies (TQCEBT)_ (IEEE, 2022) pp. 1–8.
  * Delgado  _et al._ (2021) Alain Delgado, Juan Miguel Arrazola, Soran Jahangiri, Zeyue Niu, Josh Izaac, Chase Roberts, and Nathan Killoran, “Variational quantum algorithm for molecular geometry optimization,” Physical Review A 104, 052402 (2021).
  * Khairy  _et al._ (2020) Sami Khairy, Ruslan Shaydulin, Lukasz Cincio, Yuri Alexeev, and Prasanna Balaprakash, “Learning to optimize variational quantum circuits to solve combinatorial problems,” in  _Proceedings of the AAAI conference on artificial intelligence_ , Vol. 34 (2020) pp. 2367–2375.
  * Liu  _et al._ (2022) Xiaoyuan Liu, Anthony Angone, Ruslan Shaydulin, Ilya Safro, Yuri Alexeev, and Lukasz Cincio, “Layer vqe: A variational approach for combinatorial optimization on noisy quantum computers,” IEEE Transactions on Quantum Engineering 3, 1–20 (2022).
  * Glos  _et al._ (2022) Adam Glos, Aleksandra Krawiec, and Zoltán Zimborás, “Space-efficient binary optimization for variational quantum computing,” npj Quantum Information 8, 39 (2022).
  * Wilde (2013) Mark M Wilde,  _Quantum information theory_ (Cambridge university press, 2013).
  * Lloyd  _et al._ (2014) Seth Lloyd, Masoud Mohseni, and Patrick Rebentrost, “Quantum principal component analysis,” Nature Physics 10, 631–633 (2014).
  * Hill and Wootters (1997) Sam A. Hill and William K. Wootters, “Entanglement of a pair of quantum bits,” Phys. Rev. Lett. 78, 5022–5025 (1997).
  * Wootters (1998) William K. Wootters, “Entanglement of formation of an arbitrary state of two qubits,” Phys. Rev. Lett. 80, 2245–2248 (1998).
  * Lee Rodgers and Nicewander (1988) Joseph Lee Rodgers and W Alan Nicewander, “Thirteen ways to look at the correlation coefficient,” The American Statistician 42, 59–66 (1988).
  * Cerf and Adami (1997) N. J. Cerf and C. Adami, “Negative entropy and information in quantum mechanics,” Phys. Rev. Lett. 79, 5194–5197 (1997).
  * Cerf and Adami (1999) N. J. Cerf and C. Adami, “Quantum extension of conditional probability,” Phys. Rev. A 60, 893–897 (1999).
  * Horodecki  _et al._ (2009) Ryszard Horodecki, Paweł Horodecki, Michał Horodecki, and Karol Horodecki, “Quantum entanglement,” Rev. Mod. Phys. 81, 865–942 (2009).
  * Brunner  _et al._ (2014) Nicolas Brunner, Daniel Cavalcanti, Stefano Pironio, Valerio Scarani, and Stephanie Wehner, “Bell nonlocality,” Reviews of modern physics 86, 419 (2014).
  * Sadhu and Das (2023) Abhishek Sadhu and Siddhartha Das, “Testing of quantum nonlocal correlations under constrained free will and imperfect detectors,” Physical Review A 107, 012212 (2023).
  * Powell (1994) Michael JD Powell, “A direct search optimization method that models the objective and constraint functions by linear interpolation,” in  _Advances in Optimization and Numerical Analysis. Mathematics and Its Applications_, Vol. 275, edited by S. Gomez and JP. Hennart (Springer, 1994).
  * Powell (2006) Michael JD Powell, “A fast algorithm for nonlinearly constrained optimization calculations,” in  _Numerical Analysis: Proceedings of the Biennial Conference Held at Dundee, June 28–July 1, 1977_ (Springer, 2006) pp. 144–157.
  * Van Hasselt  _et al._ (2016) Hado Van Hasselt, Arthur Guez, and David Silver, “Deep reinforcement learning with double q-learning,” in  _Proceedings of the AAAI conference on artificial intelligence_ , Vol. 30 (2016).