4\,{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}\,\mathbb{E}\big{(}\|\mathcal{X}\|^{4}\big{)}\leq
4\beta_{N}^{2}\,\mathbb{E}h_{\mathcal{G}}(\mathcal{X}).$ (E.13)

Finally, using (E.12) and (E.13), we get

$\displaystyle\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})=\textrm{Var}\big{(}h_{\mathcal{G}}(\mathcal{X})\big{)}+\big{(}\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\big{)}^{2}\leq(6\beta_{N}^{2}+2\gamma_{N}^{2})\mathbb{E}h_{\mathcal{G}}(\mathcal{X}).$
(E.14)

We are now ready to prove the lemma. In our proof, we borrow heavily from the
proof of Lemma 11.4 in Györfi et al. (2002). In what follows,
$\mathcal{X}_{1}^{\prime},\ldots,\mathcal{X}_{N}^{\prime}\overset{{\rm
i.i.d.}}{\sim}\mathcal{X}$ are distributed independently of
$\mathcal{X}_{1},\ldots,\mathcal{X}_{N}$. Also, we denote the data at hand by
$\mathscr{X}_{N}$, i.e.,
$\mathscr{X}_{N}:=\\{\mathcal{X}_{1},\ldots,\mathcal{X}_{N}\\}$. Define
$\mathcal{G}_{N}$ depending on $\mathscr{X}_{N}$ such that

$\mathbb{E}h_{\mathcal{G}_{N}}(\mathcal{X})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n})\geq\epsilon\big{(}a+b+\mathbb{E}h_{\mathcal{G}_{N}}(\mathcal{X})\big{)},$

if such a $\mathcal{G}_{N}$ exists in $\widetilde{\mathscr{F}}_{N}$; otherwise
choose an arbitrary $\mathcal{G}\in\widetilde{\mathscr{F}}_{N}$. Then, using
Chebyshev’s inequality

$\displaystyle\mathbb{P}\Big{\\{}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n}^{\prime})>\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\,|\,\mathscr{X}_{N}\Big{\\}}$
$\displaystyle\leq\frac{\textrm{Var}\Big{\\{}\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n}^{\prime})\,|\,\mathscr{X}_{N}\Big{\\}}}{\Big{\\{}\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{n}\big{)}\Big{\\}}^{2}}=\frac{\textrm{Var}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}}{N\Big{\\{}\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\Big{\\}}^{2}}$
$\displaystyle\leq\frac{4\beta_{N}^{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}}{N\Big{\\{}\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\Big{\\}}^{2}}=\frac{16\beta_{N}^{2}}{N\epsilon^{2}}\times\frac{\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}}{\Big{\\{}(a+b)+\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\Big{\\}}^{2}}$
$\displaystyle\leq\frac{16\beta_{N}^{2}}{N\epsilon^{2}}\times\frac{1}{4(a+b)}~{}~{}\bigg{(}\text{using
}\frac{x}{(\alpha+x)^{2}}\leq\frac{1}{4\alpha}\text{ for all }x\geq 0\text{
and }\alpha\geq 0\bigg{)}$
$\displaystyle=\frac{4\beta_{N}^{2}}{N\epsilon^{2}(a+b)},$

where, we have used (E.13) in the third line. Hence, for $N\geq
32\beta_{N}^{2}/(\epsilon^{2}(a+b))$,

$\mathbb{P}\Big{\\{}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n}^{\prime})\leq\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\,|\,\mathscr{X}_{N}\Big{\\}}\geq\frac{7}{8}.$
(E.15)

Now,

$\displaystyle\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n}^{\prime})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n})\geq\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\bigg{\\}}$
$\displaystyle\kern
21.52771pt\geq\mathbb{P}\bigg{\\{}\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n}^{\prime})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n})\geq\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\bigg{\\}}$
$\displaystyle\kern
21.52771pt\geq\mathbb{P}\bigg{\\{}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n})\geq\epsilon(a+b)+\epsilon\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)},$
$\displaystyle\kern
51.6665pt\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n}^{\prime})\geq\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\bigg{\\}}$
$\displaystyle\kern
21.52771pt\geq\frac{7}{8}\times\mathbb{P}\bigg{\\{}\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}_{N}}(\mathcal{X}_{n})\geq\epsilon(a+b)+\epsilon\mathbb{E}\big{(}h_{\mathcal{G}_{N}}(\mathcal{X})\,|\,\mathscr{X}_{N}\big{)}\bigg{\\}}$
$\displaystyle\kern 258.33252pt(\text{conditioning on }\mathscr{X}_{N}\text{
and using \eqref{eq:lemma_covnet2_type2_general_eq1}})$ $\displaystyle\kern
21.52771pt=\frac{7}{8}\times\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\mathbb{E}h_{\mathcal{G}}(\mathcal{X})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n})\geq\epsilon(a+b)+\epsilon\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\bigg{\\}}.$

So, for $N\geq 32\beta_{N}^{2}/(\epsilon^{2}(a+b))$,

$\displaystyle\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\mathbb{E}h_{\mathcal{G}}(\mathcal{X})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n})\geq\epsilon(a+b)+\epsilon\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\bigg{\\}}$
$\displaystyle\kern
43.05542pt\leq\frac{8}{7}\times\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n}^{\prime})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n})\geq\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\bigg{\\}}.$
(E.16)

Now, for events $E_{1},E_{2},E_{3}$,
$\mathbb{P}(E_{1})\leq\mathbb{P}(E_{1}\cap E_{2}\cap
E_{3})+\mathbb{P}(E_{2}^{c})+\mathbb{P}(E_{3}^{c})$, using which we write

$\displaystyle\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n}^{\prime})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n})\geq\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\bigg{\\}}$
$\displaystyle\kern
21.52771pt\leq\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n}^{\prime})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n})\geq\frac{\epsilon}{2}(a+b)+\frac{\epsilon}{2}\mathbb{E}h_{\mathcal{G}}(\mathcal{X}),$
$\displaystyle\kern
51.6665pt\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})-\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})\leq\epsilon\Big{(}a+b+\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})+\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})\Big{)},$
$\displaystyle\kern
51.6665pt\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n}^{\prime})-\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})\leq\epsilon\Big{(}a+b+\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n}^{\prime})+\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})\Big{)}\bigg{\\}}$
$\displaystyle\kern
21.52771pt+2\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})-\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})}{a+b+\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})+\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})}>\epsilon\bigg{\\}}.$
(E.17)

Using Theorem 11.6 of Györfi et al. (2002), and noting that
$h_{\mathcal{G}}^{2}(\mathcal{X})\leq\eta_{N}^{2}$ almost surely (cf. (E.11)),
we get

$\displaystyle\mathbb{P}\bigg{\\{}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})-\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})}{a+b+\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})+\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})}>\epsilon\bigg{\\}}$
$\displaystyle\kern 43.05542pt\leq
4\times\mathbb{E}\mathcal{N}_{1}\bigg{(}\frac{(a+b)\epsilon}{5},\\{h_{\mathcal{G}}^{2}:\mathcal{G}\in\widetilde{\mathscr{F}}_{N}\\},\mathscr{X}_{N}\bigg{)}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(a+b)N}{40\eta_{N}^{2}}\bigg{\\}},$
(E.18)

where $\mathcal{N}_{1}(\epsilon,F,\mathscr{X}_{N})$ is the _random
$L_{1}$-covering number_ of the class of functions $F$ on $\mathscr{X}_{N}$,
see Györfi et al. (2002, Chapter 9) for details. This provides an upper bound
for the second part in (E.2). For the first part, the second inequality inside
the probability implies

$\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})\geq\frac{1-\epsilon}{1+\epsilon}\times\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})-\frac{\epsilon(a+b)}{1+\epsilon}.$

Also,
$\mathbb{E}h_{\mathcal{G}}^{2}(\mathcal{X})\leq(6\beta_{N}^{2}+2\gamma_{N}^{2})\mathbb{E}h_{\mathcal{G}}(\mathcal{X})$
(cf. (E.14)). So, the first probability can be bounded by

$\displaystyle\mathbb{P}\bigg{[}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n}^{\prime})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X}_{n})\geq\frac{\epsilon}{2}(a+b)$
$\displaystyle\kern
86.11084pt+\frac{\epsilon}{2}\times\frac{1}{4(3\beta_{N}^{2}+\gamma_{N}^{2})}\bigg{\\{}\frac{(1-\epsilon)}{(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n}^{\prime})+\frac{(1-\epsilon)}{(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})-2\frac{\epsilon(a+b)}{(1+\epsilon)}\bigg{\\}}\Bigg{]}$
$\displaystyle=\mathbb{P}\bigg{[}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}\big{(}h_{\mathcal{G}}(\mathcal{X}_{n}^{\prime})-h_{\mathcal{G}}(\mathcal{X}_{n})\big{)}\geq\frac{\epsilon}{2}(a+b)-\frac{\epsilon^{2}(a+b)}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}$
$\displaystyle\kern
215.2771pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}\big{(}h_{\mathcal{G}}^{2}(\mathcal{X}_{n}^{\prime})+h_{\mathcal{G}}^{2}(\mathcal{X}_{n})\big{)}\bigg{]}$
$\displaystyle\leq
2\times\mathbb{P}\bigg{[}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h_{\mathcal{G}}(\mathcal{X}_{n})\bigg{|}\geq\frac{1}{2}\bigg{(}\frac{\epsilon}{2}(a+b)-\frac{\epsilon^{2}(a+b)}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{)}$
$\displaystyle\kern
215.2771pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})\bigg{]}.$
(E.19)

Here, $\zeta_{n}$’s are ${\rm i.i.d.}$ Rademacher random variables, which take
values $\pm 1$ with equal probability and are independent of all other
variables. Next, we derive an upper bound for the probability in (E.2).

Given $\mathscr{X}_{N}$ and $\delta>0$, let
$\mathcal{H}_{\delta}(\mathscr{X}_{N})$ be the smallest subset of
$\\{h_{\mathcal{G}}:\mathcal{G}\in\widetilde{\mathscr{F}}_{N}\\}$ such that
for every $\mathcal{G}\in\widetilde{\mathscr{F}}_{N}$ we can find
$h\in\mathcal{H}_{\delta}(\mathscr{X}_{N})$ satisfying

$\frac{1}{N}\sum_{n=1}^{N}\big{|}h_{\mathcal{G}}(\mathcal{X}_{n})-h(\mathcal{X}_{n})\big{|}<\delta.$

Then, for each $\mathcal{G}\in\widetilde{\mathscr{F}}_{N}$, we can find
$h\in\mathcal{H}_{\delta}(\mathscr{X}_{N})$ such that

$\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h_{\mathcal{G}}(\mathcal{X}_{n})\bigg{|}=\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}\big{\\{}h(\mathcal{X}_{n})+h_{\mathcal{G}}(\mathcal{X}_{n})-h(\mathcal{X}_{n})\big{\\}}\bigg{|}\leq\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h(\mathcal{X}_{n})\bigg{|}+\delta.$

Also,

$\displaystyle\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})$
$\displaystyle=\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})+\frac{1}{N}\sum_{n=1}^{N}\big{\\{}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})-h^{2}(\mathcal{X}_{n})\big{\\}}$
$\displaystyle=\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})+\frac{1}{N}\sum_{n=1}^{N}\big{(}h_{\mathcal{G}}(\mathcal{X}_{n})+h(\mathcal{X}_{n})\big{)}\big{(}h_{\mathcal{G}}(\mathcal{X}_{n})-h(\mathcal{X}_{n})\big{)}$
$\displaystyle\geq\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})-2\,\eta_{N}\,\frac{1}{N}\sum_{n=1}^{N}\big{|}h_{\mathcal{G}}(\mathcal{X}_{n})-h(\mathcal{X}_{n})\big{|}~{}~{}(\text{since
}|h_{\mathcal{G}}(\mathcal{X})|,|h(\mathcal{X})|\leq\eta_{N})$
$\displaystyle\geq\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})-2\delta\eta_{N}.$

Thus,

$\displaystyle\mathbb{P}\bigg{[}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h_{\mathcal{G}}(\mathcal{X}_{n})\bigg{|}\geq\frac{1}{2}\bigg{(}\frac{\epsilon}{2}(a+b)-\frac{\epsilon^{2}(a+b)}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{)}$
$\displaystyle\kern
215.2771pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})\,\Big{|}\,\mathscr{X}_{N}\bigg{]}$
$\displaystyle\leq\mathbb{P}\bigg{[}\exists\,h\in\mathcal{H}_{\delta}(\mathscr{X}_{N}):\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h(\mathcal{X}_{n})\bigg{|}+\delta\geq\frac{1}{2}\bigg{(}\frac{\epsilon}{2}(a+b)-\frac{\epsilon^{2}(a+b)}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,\eta_{N}(1+\epsilon)}\bigg{)}$
$\displaystyle\kern
215.2771pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{\\{}\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})-2\delta\eta_{N}\bigg{\\}}\,\Big{|}\,\mathscr{X}_{N}\bigg{]}$
$\displaystyle=\mathbb{P}\bigg{[}\exists\,h\in\mathcal{H}_{\delta}(\mathscr{X}_{N}):\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h(\mathcal{X}_{n})\bigg{|}\geq\frac{\epsilon}{4}(a+b)-\frac{\epsilon^{2}(a+b)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}-\delta-\delta\frac{\epsilon(1-\epsilon)\eta_{N}}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}$
$\displaystyle\kern
236.80481pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})\,\Big{|}\,\mathscr{X}_{N}\bigg{]}$
$\displaystyle\leq\big{|}\mathcal{H}_{\delta}(\mathscr{X}_{N})\big{|}\sup_{h\in\mathcal{H}_{\delta}(\mathscr{X}_{N})}\mathbb{P}\Bigg{[}\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h(\mathcal{X}_{n})\bigg{|}\geq\frac{\epsilon}{4}(a+b)-\frac{\epsilon^{2}(a+b)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}-\delta-\delta\frac{\epsilon(1-\epsilon)\eta_{N}}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}$
$\displaystyle\kern
236.80481pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})\,\Big{|}\,\mathscr{X}_{N}\Bigg{]}.$
(E.20)

Set $\delta=\epsilon b/5$, so that

$\frac{\epsilon
b}{4}-\frac{\epsilon^{2}b}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}-\delta-\delta\frac{\epsilon(1-\epsilon)\eta_{N}}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\geq
0.$

With this choice, the right side of (E.2) can be bounded by

$\displaystyle\Big{|}\mathcal{H}_{\frac{\epsilon
b}{5}}(\mathscr{X}_{N})\Big{|}\sup_{h\in\mathcal{H}_{\frac{\epsilon
b}{5}}(\mathscr{X}_{N})}\mathbb{P}\Bigg{[}\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h(\mathcal{X}_{n})\bigg{|}\geq\frac{\epsilon}{4}a-\frac{\epsilon^{2}a}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}$
$\displaystyle\kern
215.2771pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})\,\Big{|}\,\mathscr{X}_{N}\Bigg{]}.$

Let $V_{n}=\zeta_{n}h(\mathcal{X}_{n})$ for $n=1,\ldots,N$ and
$\sigma^{2}=N^{-1}\sum_{n=1}^{N}\textrm{Var}\big{(}V_{n}\,|\,\mathscr{X}_{N}\big{)}=N^{-1}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})$.
The probability on the last equation equals
$\mathbb{P}\big{[}|N^{-1}\sum_{n=1}^{N}V_{n}|\geq
C_{1}+C_{2}\sigma^{2}\,|\,\mathscr{X}_{N}\big{]}$, with

$C_{1}=\frac{\epsilon}{4}a-\frac{\epsilon^{2}a}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\text{
and
}C_{2}=\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}.$

Given $\mathscr{X}_{N}$, $V_{1},\ldots,V_{N}$ are independent random variables
with $|V_{n}|=|h_{\mathcal{G}}(\mathcal{X}_{n})|\leq\eta_{N}$ and
$\mathbb{E}(V_{n}\,|\,\mathscr{X}_{N})=0$ for $n=1,\ldots,N$. Also, both
$C_{1}$ and $C_{2}$ are non-negative. So, using Bernstein’s inequality (Györfi
et al., 2002, Lemma A.2), we get

$\displaystyle\mathbb{P}\Bigg{[}\bigg{|}\frac{1}{N}\sum_{n=1}^{n}V_{n}\bigg{|}\geq
C_{1}+C_{2}\sigma^{2}\,\Big{|}\,\mathscr{X}_{N}\Bigg{]}\leq
2\times\exp\bigg{\\{}-18N\frac{C_{1}C_{2}}{(2C_{2}\eta_{N}+3)^{2}}\bigg{\\}},$

see Györfi et al. (2002, pages 217–218). Plugging in the expressions for
$C_{1}$ and $C_{2}$ and noting that

$C_{1}=\frac{\epsilon}{4}a-\frac{\epsilon^{2}a}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\geq\frac{\epsilon}{4}a-\frac{\epsilon}{32}a=\frac{7\epsilon
a}{32},$

we get

$18N\frac{C_{1}C_{2}}{(2C_{2}\eta_{N}+3)^{2}}\geq\frac{3\epsilon^{2}(1-\epsilon)aN}{10(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}.$

All these finally give us

$\displaystyle\mathbb{P}\Bigg{[}\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h(\mathcal{X}_{n})\bigg{|}\geq\frac{\epsilon}{4}a-\frac{\epsilon^{2}a}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h^{2}(\mathcal{X}_{n})\,\Big{|}\,\mathscr{X}_{N}\Bigg{]}$
$\displaystyle\kern 279.86023pt\leq
2\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(1-\epsilon)aN}{10(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{\\}}.$

This upper bound does not depend on $\mathscr{X}_{N}$. So, using
$\mathbb{P}(A)=\mathbb{E}\,\mathbb{P}(A\,|\,B)$, we get

$\displaystyle\mathbb{P}\Bigg{[}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\bigg{|}\frac{1}{N}\sum_{n=1}^{N}\zeta_{n}h_{\mathcal{G}}(\mathcal{X}_{n})\bigg{|}\geq\frac{1}{2}\bigg{(}\frac{\epsilon}{2}(a+b)-\frac{\epsilon^{2}(a+b)}{4(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{)}$
$\displaystyle\kern
236.80481pt+\frac{\epsilon(1-\epsilon)}{8(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}^{2}(\mathcal{X}_{n})\Bigg{]}$
$\displaystyle\leq 2\times\mathbb{E}\Big{|}\mathcal{H}_{\frac{\epsilon
b}{5}}(\mathscr{X}_{N})\Big{|}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(1-\epsilon)aN}{10(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{\\}}.$
(E.21)

Next, we derive upper bounds on
$\mathcal{N}_{1}(\epsilon,\\{h_{\mathcal{G}}^{2}:\mathcal{G}\in\widetilde{\mathscr{F}}_{N}\\},\mathscr{X}_{N})$
and $\big{|}\mathcal{H}_{\delta}(\mathscr{X}_{N})\big{|}$. For any
$\mathcal{G}_{1},\mathcal{G}_{2}\in\widetilde{\mathscr{F}}_{N}$,

$\displaystyle\frac{1}{N}\sum_{n=1}^{N}\big{|}h_{\mathcal{G}_{1}}(\mathcal{X}_{n})-h_{\mathcal{G}_{2}}(\mathcal{X}_{n})\big{|}$
$\displaystyle=\frac{1}{N}\sum_{n=1}^{N}\big{|}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}_{1}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}-{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}_{2}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}-2{\left\langle\kern-2.15277pt\left\langle\kern-2.15277pt\left\langle\mathcal{G}_{1}-\mathcal{G}_{2},\mathcal{X}_{n}\otimes\mathcal{X}_{n}\right\rangle\kern-2.15277pt\right\rangle\kern-2.15277pt\right\rangle}_{2}\big{|}$
$\displaystyle=\frac{1}{N}\sum_{n=1}^{N}\big{|}{\left\langle\kern-2.15277pt\left\langle\kern-2.15277pt\left\langle\mathcal{G}_{1}-\mathcal{G}_{2},\mathcal{G}_{1}+\mathcal{G}_{2}-2\mathcal{X}_{n}\otimes\mathcal{X}_{n}\right\rangle\kern-2.15277pt\right\rangle\kern-2.15277pt\right\rangle}_{2}\big{|}$
$\displaystyle\leq{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}_{1}-\mathcal{G}_{2}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\times\frac{1}{N}\sum_{n=1}^{N}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}_{1}+\mathcal{G}_{2}-2\mathcal{X}_{n}\otimes\mathcal{X}_{n}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}$
$\displaystyle\leq
2(\beta_{N}+\gamma_{N})\times{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}_{1}-\mathcal{G}_{2}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}.$

This shows that a $\delta$-cover for $\widetilde{\mathscr{F}}_{N}$ w.r.t. the
${\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}$
norm is equivalent to a random $L_{1}$-cover for
$\\{h_{\mathcal{G}}:\mathcal{G}\in\widetilde{\mathscr{F}}_{N}\\}$ on
$\mathscr{X}_{N}$ of size $2(\beta_{N}+\gamma_{N})\delta$. Thus,

$\big{|}\mathcal{H}_{\delta}(\mathscr{X}_{N})\big{|}\leq\mathcal{N}\bigg{(}\frac{\delta}{2(\beta_{N}+\gamma_{N})},\widetilde{\mathscr{F}},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}.$
(E.22)

Note that this upper bound does not depend on $\mathscr{X}_{N}$. In fact, it
is not a random quantity. Again,

$\displaystyle\frac{1}{N}\sum_{n=1}^{N}\big{|}h_{\mathcal{G}_{1}}^{2}(\mathcal{X}_{n})-h_{\mathcal{G}_{2}}^{2}(\mathcal{X}_{n})\big{|}$
$\displaystyle=\frac{1}{N}\sum_{n=1}^{N}\big{|}h_{\mathcal{G}_{1}}(\mathcal{X}_{n})-h_{\mathcal{G}_{2}}(\mathcal{X}_{n})\big{|}\,\big{|}h_{\mathcal{G}_{1}}(\mathcal{X}_{n})+h_{\mathcal{G}_{2}}(\mathcal{X}_{n})\big{|}$
$\displaystyle\leq
2\eta_{N}\times\frac{1}{N}\sum_{n=1}^{N}\big{|}h_{\mathcal{G}_{1}}(\mathcal{X}_{n})-h_{\mathcal{G}_{2}}(\mathcal{X}_{n})\big{|}$
$\displaystyle\leq
4\eta_{N}(\beta_{N}+\gamma_{N})\times{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}_{1}-\mathcal{G}_{2}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}.$

This gives us

$\mathcal{N}_{1}\big{(}\delta,\\{h_{\mathcal{G}}^{2}:\mathcal{G}\in\widetilde{\mathscr{F}}_{N}\\},\mathscr{X}_{N}\big{)}\leq\mathcal{N}\bigg{(}\frac{\delta}{4\eta_{N}(\beta_{N}+\gamma_{N})},\widetilde{\mathscr{F}}_{N},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}.$
(E.23)

This, again, is a non-random upper bound. We now assemble all the pieces
together. By (E.2) and (E.2), for every $N\geq
32\beta_{N}^{2}/(\epsilon^{2}(a+b))$,

$\displaystyle\mathbb{P}\bigg{[}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\mathbb{E}h_{\mathcal{G}}(\mathcal{X})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X})\geq\epsilon\big{(}a+b+\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\big{)}\bigg{]}\leq\frac{8}{7}\times(P_{1}+2P_{2}),$

where, by (E.2) and (E.2),

$\displaystyle P_{1}\leq 4\times\mathbb{E}\Big{|}\mathcal{H}_{\frac{\epsilon
b}{5}}(\mathscr{X}_{N})\Big{|}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(1-\epsilon)aN}{10(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{\\}},$

and by (E.2),

$\displaystyle P_{2}\leq
4\times\mathbb{E}\mathcal{N}_{1}\bigg{(}\frac{(a+b)\epsilon}{5},\\{h_{\mathcal{G}}^{2}:\mathcal{G}\in\widetilde{\mathscr{F}}_{N}\\},\mathscr{X}_{N}\bigg{)}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(a+b)N}{40\eta_{N}^{2}}\bigg{\\}}.$

Using these, with the bounds on the covering numbers (E.22), (E.23), we get

$\displaystyle\mathbb{P}\bigg{[}\exists\,\mathcal{G}\in\widetilde{\mathscr{F}}_{N}:\mathbb{E}h_{\mathcal{G}}(\mathcal{X})-\frac{1}{N}\sum_{n=1}^{N}h_{\mathcal{G}}(\mathcal{X})\geq\epsilon\big{(}a+b+\mathbb{E}h_{\mathcal{G}}(\mathcal{X})\big{)}\bigg{]}$
$\displaystyle\kern 21.52771pt\leq\frac{8}{7}\times
4\times\mathbb{E}\Big{|}\mathcal{H}_{\frac{\epsilon
b}{5}}(\mathscr{X}_{N})\Big{|}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(1-\epsilon)aN}{10(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{\\}}$
$\displaystyle\kern 64.58313pt+\frac{16}{7}\times
4\times\mathbb{E}\mathcal{N}_{1}\bigg{(}\frac{(a+b)\epsilon}{5},\\{h_{\mathcal{G}}^{2}:\mathcal{G}\in\widetilde{\mathscr{F}}_{N}\\},\mathscr{X}_{N}\bigg{)}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(a+b)N}{40\eta_{N}^{2}}\bigg{\\}}$
$\displaystyle\kern
21.52771pt\leq\frac{32}{7}\times\mathcal{N}\bigg{(}\frac{\epsilon
b}{10(\beta_{N}+\gamma_{N})},\widetilde{\mathscr{F}}_{N},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(1-\epsilon)aN}{10(3\beta_{N}^{2}+\gamma_{N}^{2})\,(1+\epsilon)}\bigg{\\}}$
$\displaystyle\kern
64.58313pt+\frac{64}{7}\times\mathcal{N}\bigg{(}\frac{(a+b)\epsilon}{20\eta_{N}(\beta_{N}+\gamma_{N})},\widetilde{\mathscr{F}}_{N},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(a+b)N}{40\eta_{N}^{2}}\bigg{\\}}$
$\displaystyle\kern 21.52771pt\leq 14\times\mathcal{N}\bigg{(}\frac{\epsilon
b}{20\eta_{N}(\beta_{N}+\gamma_{N})},\widetilde{\mathscr{F}}_{N},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}\bigg{)}\times\exp\bigg{\\{}-\frac{3\epsilon^{2}(1-\epsilon)aN}{40\eta_{N}^{2}\,(1+\epsilon)}\bigg{\\}}$
$\displaystyle\kern 21.52771pt\leq 14\times\mathcal{N}\bigg{(}\frac{\epsilon
b}{80(\beta_{N}+\gamma_{N})^{3}},\widetilde{\mathscr{F}}_{N},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}\bigg{)}\times\exp\bigg{\\{}-\frac{\epsilon^{2}(1-\epsilon)aN}{214(\beta_{N}+\gamma_{N})^{4}\,(1+\epsilon)}\bigg{\\}},$

where we have used that
$\eta_{N}:=\max\\{4\beta_{N}^{2},2(\beta_{N}^{2}+\gamma_{N}^{2})\\}\leq
4(\beta_{N}+\gamma_{N})^{2}$. This proves the result for $N\geq
32\beta_{N}^{2}/(\epsilon^{2}(a+b))$. For
$N<32\beta_{N}^{2}/(\epsilon^{2}(a+b))$,

$\displaystyle\exp\bigg{\\{}-\frac{\epsilon^{2}(1-\epsilon)aN}{214(\beta_{N}+\gamma_{N})^{4}\,(1+\epsilon)}\bigg{\\}}\geq\frac{1}{14},$

so the inequality holds trivially. This completes the proof. ∎

######  Remark 14.

When $\beta_{N}=\gamma_{N}$, we can mimic the same proof to get the following
upper bound with slightly better constants:

$14\times\mathcal{N}\bigg{(}\frac{\epsilon
b}{160\beta_{N}^{3}},\widetilde{\mathscr{F}}_{N},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}\bigg{)}\times\exp\bigg{\\{}-\frac{\epsilon^{2}(1-\epsilon)aN}{214\beta_{N}^{4}(1+\epsilon)}\bigg{\\}}.$

We use (E.2) and Lemma 11 with $\epsilon=1/2,a=b=t/2$, to get

$\displaystyle\mathbb{P}\big{(}\widetilde{V}_{N}>t\big{)}\leq
14\times\mathcal{N}\bigg{(}\frac{t}{320(\beta_{N}+\gamma_{N})^{3}},\widetilde{\mathscr{F}}_{N},{|\kern-1.07639pt|\kern-1.07639pt|\cdot|\kern-1.07639pt|\kern-1.07639pt|}_{2}\bigg{)}\times\exp\bigg{\\{}-\frac{Nt}{5136(\beta_{N}+\gamma_{N})^{4}}\bigg{\\}}.$

Now, for any non-negative random variable $Y$,

$\displaystyle\mathbb{E}(Y)=\int_{0}^{\infty}\mathbb{P}(Y>t)\,\mathrm{d}t\leq
u+\int_{u}^{\infty}\mathbb{P}(Y>t)\,\mathrm{d}t,$

for every $u>0$. Using this, we get that for all $u>0$,

$\displaystyle\mathbb{E}\widetilde{V}_{N}\leq
u+\int_{u}^{\infty}\mathbb{P}\big{(}\widetilde{V}_{N}>t\big{)}\,\mathrm{d}t$
$\displaystyle\kern 38.74988pt\leq
u+14\times\int_{u}^{\infty}\mathcal{N}\bigg{(}\frac{t}{320(\beta_{N}+\gamma_{N})^{3}},\widetilde{\mathscr{F}},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}\times\exp\bigg{\\{}-\frac{Nt}{5136(\beta_{N}+\gamma_{N})^{4}}\bigg{\\}}$
$\displaystyle\kern 38.74988pt\leq
u+14\times\mathcal{N}\bigg{(}\frac{u}{320(\beta_{N}+\gamma_{N})^{3}},\widetilde{\mathscr{F}},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}\times\int_{u}^{\infty}\exp\bigg{\\{}-\frac{Nt}{5136(\beta_{N}+\gamma_{N})^{4}}\bigg{\\}}$
$\displaystyle\leq u+\frac{14\times
5136(\beta_{N}+\gamma_{N})^{4}}{N}\times\mathcal{N}\bigg{(}\frac{u}{320(\beta_{N}+\gamma_{N})^{3}},\widetilde{\mathscr{F}},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}\times\exp\bigg{\\{}-\frac{Nu}{5136(\beta_{N}+\gamma_{N})^{4}}\bigg{\\}}.$

To obtain the rate of convergence of $\mathbb{E}\widetilde{V}_{N}$, we
(approximately) minimize the above quantity w.r.t. $u$. In particular, by
choosing

$u=\frac{5136(\beta_{N}+\gamma_{N})^{4}}{N}\Bigg{\\{}\log(14)+\log\mathcal{N}\bigg{(}\frac{5136(\beta_{N}+\gamma_{N})}{320N},\widetilde{\mathscr{F}}_{N},{|\kern-1.07639pt|\kern-1.07639pt|\cdot|\kern-1.07639pt|\kern-1.07639pt|}_{2}\bigg{)}\Bigg{\\}},$

we get that

$\displaystyle\mathbb{E}\widetilde{V}_{N}$
$\displaystyle\leq\frac{5146(\beta_{N}+\gamma_{N})^{4}}{N}\Bigg{\\{}1+\log(14)+\log\mathcal{N}\bigg{(}\frac{5136(\beta_{N}+\gamma_{N})}{320N},\widetilde{\mathscr{F}}_{N},{|\kern-1.07639pt|\kern-1.07639pt|\cdot|\kern-1.07639pt|\kern-1.07639pt|}_{2}\bigg{)}\Bigg{\\}}$
$\displaystyle=\mathcal{O}\bigg{(}\frac{(\beta_{N}+\gamma_{N})^{4}}{N}\times\log\mathcal{N}\Big{(}\frac{\beta_{N}+\gamma_{N}}{N},\widetilde{\mathscr{F}}_{N},{|\kern-1.07639pt|\kern-1.07639pt|\cdot|\kern-1.07639pt|\kern-1.07639pt|}_{2}\Big{)}\bigg{)}.$
(E.24)

Finally, combining (E.2) and (E.2) with (E.2), we get the following lemma.

######  Lemma 12.

Let $\widetilde{\mathscr{F}}_{N}$ be a class of operators with
${\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\leq\gamma_{N}$
for every $\mathcal{G}\in\widetilde{\mathscr{F}}_{N}$. Let
$\mathcal{X}_{1},\ldots,\mathcal{X}_{N}\overset{{\rm
i.i.d.}}{\sim}\mathcal{X}$, with
$\mathbb{P}(\|\mathcal{X}\|^{2}\leq\beta_{N})=1$, $\mathbb{E}(\mathcal{X})=0$
and $\textrm{Var}(\mathcal{X})=\mathcal{C}$. Define
$\widehat{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}=\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{N}-\mathcal{G}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}$,
where
$\widehat{\mathcal{C}}_{N}=N^{-1}\sum_{n=1}^{N}\mathcal{X}_{n}\otimes\mathcal{X}_{n}$
is the empirical covariance operator. Then,

$\displaystyle\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}$
$\displaystyle\leq
2\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}+\mathcal{O}\bigg{(}\frac{\Delta_{N}^{4}}{N}\times\log\mathcal{N}\bigg{(}\frac{\Delta_{N}}{N},\widetilde{\mathscr{F}}_{N},{|\kern-1.07639pt|\kern-1.07639pt|\cdot|\kern-1.07639pt|\kern-1.07639pt|}_{2}\bigg{)}\bigg{)},$

where $\Delta_{N}=\max\\{\beta_{N},\gamma_{N}\\}$.

As in the case of consistency, we use this lemma in conjunction with the fact
that
${|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}|\kern-1.07639pt|\kern-1.07639pt|}_{2}\leq|\mathcal{Q}|R\lambda_{N}$
for an CovNet operator $\mathcal{G}$ from any of the three CovNet classes
(shallow, deep or deepshared) and with the bounds on the covering numbers of
these classes to get the rates given in Theorem 4.

###  E.3 Consistency without boundedness

We now extend the consistency results by removing the boundedness condition on
$\mathcal{X}$. Recall that our estimators are re-defined in this case. As
before, we prove a general result and then show the particular cases of
shallow, deep and deepshared CovNet models. To this effect, consider a kernel
of the form

$g(\text{\bf u},\text{\bf
v})=\sum_{r=1}^{R}\sum_{s=1}^{R}\lambda_{r,s}\,g_{r}(\text{\bf
u})\,g_{s}(\text{\bf v}),~{}~{}\text{\bf u},\text{\bf v}\in\mathcal{Q},$
(E.25)

where $\Lambda:=(\lambda_{r,s})$ is positive semi-definite. We denote the
corresponding integral operator by $\mathcal{G}$. For $\lambda_{N}>0$, define
$\mathscr{P}_{\lambda_{N}}\mathcal{G}$ to be the operator obtained by
thresholding the eigenvalues of $\Lambda$ to $\lambda_{N}$. That is, if
$\Lambda=\sum_{i=1}^{R}\eta_{i}\,\mathbf{e}_{i}\,\mathbf{e}_{i}^{\top}$ is the
eigendecomposition of $\Lambda$, then we define
$\Lambda_{\lambda_{N}}=\sum_{i=1}^{R}\max\\{\eta_{i},\lambda_{N}\\}\,\mathbf{e}_{i}\,\mathbf{e}_{i}^{\top}$
to be the $\lambda_{N}$-thresholded version of $\Lambda$. We define
$\mathscr{P}_{\lambda_{N}}\mathcal{G}$ as the integral operator with kernel
$g_{\lambda_{N}}(\text{\bf u},\text{\bf
v})=\sum_{r=1}^{R}\sum_{s=1}^{R}\widetilde{\lambda}_{r,s}\,g_{r}(\text{\bf
u})\,g_{s}(\text{\bf v})$, where $\widetilde{\lambda}_{r,s}$ is the $(r,s)$-th
element of the thresholded matrix $\Lambda_{\lambda_{N}}$. By construction,
$\mathrm{0}\preceq\Lambda_{\lambda_{N}}\preceq\lambda_{N}\,\mathrm{I}_{R}$.
Let $\widetilde{\mathscr{F}}_{R}$ be a class of operators with kernels of the
form (E.25) and we denote the corresponding class of restricted operators by
$\widetilde{\mathscr{F}}_{R,\lambda_{N}}$. Define

$\widehat{\mathcal{C}}_{R,N}=\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R}}{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{N}-\mathcal{G}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2},$

to be the estimator without any restriction on the underlying class. Now, for
a constant $\lambda_{N}>0$, our modified estimator is defined as

$\widetilde{\mathcal{C}}_{R,N}=\mathscr{P}_{\lambda_{N}}\widehat{\mathcal{C}}_{R,N}.$
(E.26)

The following theorem illustrates the conditions for consistency of the
modified estimator. The result is similar to Theorem 10.2 of Györfi et al.
(2002).

######  Theorem 8.

Let $\mathcal{X}_{1},\ldots,\mathcal{X}_{N}\overset{{\rm
i.i.d.}}{\sim}\mathcal{X}$, where $\mathbb{E}(\|\mathcal{X}\|^{4})<\infty$,
$\mathbb{E}(\mathcal{X})=0$ and $\textrm{Var}(\mathcal{X})=\mathcal{C}$.
Suppose that $R_{N},\lambda_{N}\to\infty$ as $N\to\infty$. Also, suppose that
the underlying class of operators $\widetilde{\mathscr{F}}_{R,\lambda_{N}}$ is
an universal approximator, i.e.,
$\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}\to
0$ as $N\to\infty$. If for every $u>0$,

$\frac{R_{N}^{4}\lambda_{N}^{4}}{N}\times\log\mathcal{N}\bigg{(}\frac{u}{R_{N}\lambda_{N}},\widetilde{\mathscr{F}}_{R,\lambda_{N}},{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\cdot\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}\bigg{)}\to
0\text{ as }N\to\infty,$

then
${|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}$
converges in probability to $0$ as $N\to\infty$. If in addition
$(R_{N}\lambda_{N})^{4}/N^{1-\delta}\to 0$ for some $\delta\in(0,1)$, then the
previous convergence holds almost surely.

###### Proof.

Note that
${|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}=\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\,|\,\mathscr{X}_{N}\big{)}-\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}$,
where $\widetilde{\mathcal{C}}_{N}$ is distributed identically to
$\widehat{\mathcal{C}}_{N}$, independently of $\mathscr{X}_{N}$ (cf. (E.1)).
Now,
$\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}=N^{-1}\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\mathcal{X}\otimes\mathcal{X}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}$
converges to $0$ as $N\to\infty$. Thus, it is enough to show that
$\Big{\\{}\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\,|\,\mathscr{X}_{N}\big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}\Big{\\}}^{1/2}$
converges to $0$ as $N\to\infty$ in the appropriate notion (i.e., in
probability or almost surely). We write

$\displaystyle 0$
$\displaystyle\leq\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{|}\,\mathscr{X}_{N}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}$
$\displaystyle=\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{|}\,\mathscr{X}_{N}\Big{)}\Big{\\}}^{1/2}-\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}$
$\displaystyle\kern
43.05542pt+\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}$
$\displaystyle=:A_{N}+B_{N}.$

For the second term,

$\displaystyle B_{N}$
$\displaystyle=\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}$
$\displaystyle\leq\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\bigg{|}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}\bigg{|}$
$\displaystyle\leq\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\Big{\\{}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{\\}}^{1/2}~{}~{}(\text{by
\eqref{eq:HS_equality_covnet1}})$
$\displaystyle=\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2},$

which by assumption converges to $0$ as $N\to\infty$. So, for convergence in
probability, it is enough to show that $\mathbb{P}\big{(}A_{N}\leq 0\big{)}\to
1$ as $N\to\infty$. Similarly, for almost sure convergence, it is enough to
show that $\mathbb{P}\big{(}\limsup_{N\to\infty}A_{N}\leq 0\big{)}=1$.

Recall that for $\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}$,
${|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}|\kern-1.07639pt|\kern-1.07639pt|}_{2}\leq\gamma_{N}:=R\lambda_{N}|\mathcal{Q}|M^{2}$,
where $M=\sup_{g\in\mathcal{H}}\|g\|$. Let $L>0$ be arbitrary. Since
$R_{N},\lambda_{N}\to\infty$, we can assume w.l.o.g. that $L\leq\gamma_{N}$.
Define $\mathcal{X}_{L}$ to be the projection of $\mathcal{X}$ onto
$\\{y\in\mathcal{L}_{2}(\mathcal{Q}):\|y\|^{2}\leq L\\}$. Similarly, for
$n=1,2,\ldots$, define $\mathcal{X}_{n,L}$ to be the projection of
$\mathcal{X}_{n}$ onto $\\{y\in\mathcal{L}_{2}(\mathcal{Q}):\|y\|^{2}\leq
L\\}$. We define
$\mathcal{C}_{L}=\mathbb{E}(\mathcal{X}_{L}\otimes\mathcal{X}_{L})$ to be the
covariance operator of $\mathcal{X}_{L}$ and
$\widehat{\mathcal{C}}_{N,L}=N^{-1}\sum_{n=1}^{N}\mathcal{X}_{n,L}\otimes\mathcal{X}_{n,L}$
to be its empirical counterpart based on
$\mathcal{X}_{1,L},\ldots,\mathcal{X}_{N,L}$. Then,

$\displaystyle A_{N}$
$\displaystyle=\bigg{(}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\,\big{|}\,\mathscr{X}_{N}\Big{)}\Big{\\}}^{1/2}-\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}\bigg{)}$
$\displaystyle=\sup_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\bigg{(}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\,\big{|}\,\mathscr{X}_{N}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}\bigg{)}$
$\displaystyle=\sup_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\bigg{(}\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\,\big{|}\,\mathscr{X}_{N}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\,\big{|}\,\mathscr{X}_{N}\Big{)}\Big{\\}}^{1/2}$
$\displaystyle\kern
55.97205pt+\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widetilde{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\,\big{|}\,\mathscr{X}_{N}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{\\}}^{1/2}$
$\displaystyle\kern
55.97205pt+{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}-{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{R,N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}$
$\displaystyle\kern
55.97205pt+{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{R,N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}-{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{R,N}-\widehat{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}$
$\displaystyle\kern
55.97205pt+{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{R,N}-\widehat{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}-{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}$
$\displaystyle\kern
55.97205pt+{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}-{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}$
$\displaystyle\kern
55.97205pt+\Big{\\{}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}$
$\displaystyle\kern
55.97205pt+\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}-\Big{\\{}\mathbb{E}\Big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\Big{)}\Big{\\}}^{1/2}\bigg{)}.$

By definition, the fifth term is non-positive. Also, by construction, the
third term is non-positive. Of the remaining, the first and the eighth terms
are bounded above by
$\big{\\{}\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}\big{\\}}^{1/2}$,
while the fourth and the sixth terms are bounded above by
${|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}$.
Finally, the second and the seventh terms are bounded above by
$\sup_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\Big{|}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}-\big{\\{}\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}\big{\\}}^{1/2}\Big{|}$.
Under the assumption,
$\sup_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\big{|}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}-\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}\big{|}\to
0$ in probability as $N\to\infty$ (cf. Lemma 8). Using this and the uniform
continuity of $x\mapsto\sqrt{x}$ on $[0,\infty)$, we get

$\sup_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\bigg{|}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}-\Big{\\{}\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}\Big{\\}}^{1/2}\bigg{|}\overset{P}{\to}0\text{
as }N\to\infty.$

For the other two terms, observe that
$\widehat{\mathcal{C}}_{N}-\widehat{\mathcal{C}}_{N,L}=N^{-1}\sum_{n=1}^{N}Z_{n,L}$,
where
$Z_{n,L}=\mathcal{X}_{n}\otimes\mathcal{X}_{n}-\mathcal{X}_{n,L}\otimes\mathcal{X}_{n,L}$.
Note that $\mathbb{E}(Z_{n,L})=\mathcal{C}-\mathcal{C}_{L}$. Now,
${|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}\leq
N^{-1}\sum_{n=1}^{N}{|\kern-1.07639pt|\kern-1.07639pt|Z_{n,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}$,
which converges almost surely to
$\mathbb{E}({|\kern-1.07639pt|\kern-1.07639pt|Z_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2})=\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{X}\otimes\mathcal{X}-\mathcal{X}_{L}\otimes\mathcal{X}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}\big{)}$
as $N\to\infty$. On the other hand,

$\displaystyle\mathbb{E}{|\kern-1.07639pt|\kern-1.07639pt|\widehat{\mathcal{C}}_{N}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}=\mathbb{E}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\frac{1}{N}\sum_{n=1}^{N}Z_{n,L}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle=\frac{1}{N}\mathbb{E}{|\kern-1.07639pt|\kern-1.07639pt|Z_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}+\frac{N(N-1)}{N^{2}}{|\kern-1.07639pt|\kern-1.07639pt|\mathbb{E}(Z_{L})|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}$
$\displaystyle=\frac{1}{N}\mathbb{E}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{X}\otimes\mathcal{X}-\mathcal{X}_{L}\otimes\mathcal{X}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}+\frac{N(N-1)}{N^{2}}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\mathcal{C}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}.$

Since $\mathbb{E}\|\mathcal{X}\|^{4}<\infty$, this last term converges to
${|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\mathcal{C}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}$
as $N\to\infty$ for all $L>0$. Thus, for all $L>0$,

$\displaystyle\mathbb{P}\Big{(}A_{N}\leq
2\mathbb{E}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{X}\otimes\mathcal{X}-\mathcal{X}_{L}\otimes\mathcal{X}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}+2{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\mathcal{C}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}\Big{)}\to
1\text{ as }N\to\infty.$

Note that
${|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}-\mathcal{C}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}\leq\mathbb{E}({|\kern-1.07639pt|\kern-1.07639pt|\mathcal{X}\otimes\mathcal{X}-\mathcal{X}_{L}\otimes\mathcal{X}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2})$.
Now, using (B.5),

$\displaystyle\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{X}\otimes\mathcal{X}-\mathcal{X}_{L}\otimes\mathcal{X}_{L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}\big{)}$
$\displaystyle\leq
2\mathbb{E}\big{(}\|\mathcal{X}\|\,\|\mathcal{X}-\mathcal{X}_{L}\|\big{)}+\mathbb{E}\big{(}\|\mathcal{X}-\mathcal{X}_{L}\|^{2}\big{)}$
$\displaystyle\leq
2\Big{\\{}\mathbb{E}\big{(}\|\mathcal{X}\|^{2}\big{)}\,\mathbb{E}\big{(}\|\mathcal{X}-\mathcal{X}_{L}\|^{2}\big{)}\Big{\\}}^{1/2}+\mathbb{E}\big{(}\|\mathcal{X}-\mathcal{X}_{L}\|^{2}\big{)}.$

By the dominated convergence theorem, the last quantity converges to $0$ as
$L\to\infty$. Thus, taking limit as $L\to\infty$, we get that
$\mathbb{P}(A_{N}\leq 0)\to 1$ as $N\to\infty$, proving the convergence of
${|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{R,N}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}$
to $0$ in probability.

For the almost sure convergence, all the steps remain the same, except now
with the additional condition

$\sup_{\mathcal{G}\in\widetilde{\mathscr{F}}_{R,\lambda_{N}}}\bigg{|}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}-\Big{\\{}\mathbb{E}\big{(}{|\kern-1.07639pt|\kern-1.07639pt|\mathcal{G}-\widehat{\mathcal{C}}_{N,L}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\big{)}\Big{\\}}^{1/2}\bigg{|}\overset{a.s.}{\to}0\text{
as }N\to\infty,$

see Lemma 8. Thus, by the same steps we get
$\mathbb{P}\big{(}\limsup_{N\to\infty}A_{N}\leq 0)=1$, completing the proof. ∎

###  E.4 The case of discretely observed data

Here, we derive the asymptotic properties of our estimators in the discrete
measurement regime (Section 6.2). Again, we will derive the results for a
general class of operators $\widetilde{\mathscr{F}}_{N}$. We define
$\widetilde{\mathcal{X}}_{n}^{K}=(\widetilde{X}_{n}^{K}(\text{\bf
u}):\text{\bf u}\in[0,1]^{d})$ to be the voxel-wise continuation of the $n$-th
measurement
$\widetilde{\mathbf{X}}_{n}^{K}=(\widetilde{X}_{n}^{K}[i_{1},\ldots,i_{d}])$,
with

$\widetilde{X}_{n}^{K}(\text{\bf
u})=\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\widetilde{X}_{n}^{K}[i_{1},\ldots,i_{d}]\,\textrm{{1}}\big{\\{}\text{\bf
u}\in V_{i_{1},\ldots,i_{d}}^{K}\big{\\}}.$

Our estimator in this case is defined as

$\widehat{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}^{K}\in\operatornamewithlimits{arg\,min}_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{|\kern-1.07639pt|\kern-1.07639pt|\widetilde{\mathcal{C}}_{N}^{K}-\mathcal{G}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2},$
(E.27)

where
$\widetilde{\mathcal{C}}_{N}^{K}=N^{-1}\sum_{n=1}^{N}\widetilde{\mathcal{X}}_{n}^{K}\otimes\widetilde{\mathcal{X}}_{n}^{K}$
is the empirical covariance based on
$\widetilde{\mathcal{X}}_{1}^{K},\ldots,\widetilde{\mathcal{X}}_{N}^{K}$. In
the following, for an operator $\mathcal{A}$, we write
$\mathbf{A}^{K}\in\mathbb{R}^{K_{1}\times\cdots\times K_{d}\times
K_{1}\times\cdots\times K_{d}}$ to denote its discretized version w.r.t. the
voxels $V_{i_{1},\ldots,i_{d}}^{K},1\leq i_{1}\leq K_{1},\ldots,1\leq
i_{d}\leq K_{d}$, and $\mathcal{A}^{K}$ to denote the voxel-wise continuation
of $\mathbf{A}^{K}$. The Frobenius norm of $\mathbf{A}^{K}$ is defined as

$\|\mathbf{A}^{K}\|_{\rm
F}^{2}=\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\sum_{j_{1}=1}^{K_{1}}\cdots\sum_{j_{d}=1}^{K_{d}}A^{2}[i_{1},\ldots,i_{d};j_{1},\ldots,j_{d}].$

It is easy to verify that $\|\mathbf{A}^{K}\|_{\rm F}^{2}=(K_{1}\cdots
K_{d})^{2}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{A}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$.
Thus, for a class of covariance operators $\widetilde{\mathscr{F}}$,

$\operatornamewithlimits{arg\,min}_{\mathcal{G}\in\widetilde{\mathscr{F}}}\big{\|}\widehat{\mathbf{C}}_{N}^{K}-\mathbf{G}^{K}\big{\|}_{\rm
F}^{2}=\operatornamewithlimits{arg\,min}_{\mathcal{G}\in\widetilde{\mathscr{F}}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widehat{\mathcal{C}}_{N}^{K}-\mathcal{G}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}\approx\operatornamewithlimits{arg\,min}_{\mathcal{G}\in\widetilde{\mathscr{F}}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widehat{\mathcal{C}}_{N}^{K}-\mathcal{G}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2},$
(E.28)

where the last assertion holds when $K$ is large. This justifies the
definition of our estimator in (E.27).

By (6.4), it follows that
$\widetilde{\mathcal{X}}_{n}^{K}=\mathcal{X}_{n}^{K}+\mathcal{E}_{n}^{K}$,
where $\mathcal{X}_{n}^{K}=(X_{n}^{K}(\text{\bf u}):\text{\bf u}\in[0,1]^{d})$
and $\mathcal{E}_{n}^{K}=(E_{n}^{K}(\text{\bf u}):\text{\bf u}\in[0,1]^{d})$
are the voxel-wise continuations of
$\mathbf{X}_{n}^{K}=(X_{n}^{K}[i_{1},\ldots,i_{d}])$ and
$\bm{\mathcal{E}}_{n}^{K}=(\epsilon_{n}^{K}[i_{1},\ldots,i_{d}])$,
respectively, i.e.,

$\displaystyle X_{n}^{K}(\text{\bf
u})=\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}X_{n}^{K}[i_{1},\ldots,i_{d}]\,\textrm{{1}}\big{\\{}\text{\bf
u}\in V_{i_{1},\ldots,i_{d}}^{K}\big{\\}}\,\text{ and }\,E_{n}^{K}(\text{\bf
u})=\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}E_{n}^{K}[i_{1},\ldots,i_{d}]\,\textrm{{1}}\big{\\{}\text{\bf
u}\in V_{i_{1},\ldots,i_{d}}^{K}\big{\\}}.$

It is easy to see that
$\widetilde{\mathcal{X}}_{1}^{K},\ldots,\widetilde{\mathcal{X}}_{N}^{K}$ are
i.i.d. with mean zero. Let
$\widetilde{\mathcal{C}}^{K}=\mathbb{E}(\widetilde{\mathcal{X}}_{1}^{K}\otimes\widetilde{\mathcal{X}}_{1}^{K})$
be the covariance of $\widetilde{\mathcal{X}}_{1}^{K}$ (the existence of
$\mathcal{C}^{K}$ will be proved shortly). Since $\mathbf{X}_{n}^{K}$ and
$\bm{\mathcal{E}}_{n}^{K}$ are uncorrelated, it follows that
$\widetilde{\mathcal{C}}^{K}=\mathcal{C}^{K}+\Sigma^{K}$, where
$\mathcal{C}^{K}=\mathbb{E}(\mathcal{X}^{K}\otimes\mathcal{X}^{K})$ is the
covariance of $\mathcal{X}^{K}$ and
$\Sigma^{K}=\mathbb{E}(\mathcal{E}^{K}\otimes\mathcal{E}^{K})$ is the
covariance of $\mathcal{E}^{K}$. To quantify the rate of convergence of the
estimator (E.27), we use the following decomposition

$\displaystyle{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widehat{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}^{K}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle={\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widehat{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}^{K}-\widetilde{\mathcal{C}}^{K}+\widetilde{\mathcal{C}}^{K}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle={\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widetilde{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}^{K}-\widetilde{\mathcal{C}}^{K}+\mathcal{C}^{K}-\mathcal{C}+\Sigma^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle\leq
3{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widetilde{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}^{K}-\widetilde{\mathcal{C}}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}+3{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{C}^{K}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}+3{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\Sigma^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}.$
(E.29)

We will separately bound each term in the above.

Firstly, note that $\|\widetilde{\mathcal{X}}_{n}^{K}\|^{2}\leq
2\|\mathcal{X}_{n}^{K}\|^{2}+2\|\mathcal{E}_{n}^{K}\|^{2}$. Now,

$\|\mathcal{X}_{n}^{K}\|^{2}=\int_{[0,1]^{d}}\big{(}X_{n}^{K}(\text{\bf
u})\big{)}^{2}\,\mathrm{d}\text{\bf u}=\frac{1}{K_{1}\cdots
K_{d}}\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\big{(}X_{n}^{K}[i_{1},\ldots,i_{d}]\big{)}^{2}.$

Under (M1), since $\|\mathcal{X}\|_{\infty}^{2}=\sup_{\text{\bf
u}\in[0,1]^{d}}X^{2}(\text{\bf u})\leq\beta_{N}$ almost surely, it follows
that $\|\mathcal{X}_{n}^{K}\|^{2}\leq\beta_{N}$ almost surely. Under (M2), if
we define $\mathcal{I}_{i_{1},\ldots,i_{d}}^{K}(\text{\bf
u})=|V_{i_{1},\ldots,i_{d}}^{K}|^{-1/2}\textrm{{1}}\big{\\{}\text{\bf u}\in
V_{i_{1},\ldots,i_{d}}^{K}\big{\\}}$, then it follows that
$X_{n}^{K}[i_{1},\ldots,i_{d}]=\sqrt{K_{1}\cdots
K_{d}}\,\langle\mathcal{X}_{n},\mathcal{I}_{i_{1},\ldots,i_{d}}^{K}\rangle$.
Thus,

$\|\mathcal{X}_{n}^{K}\|^{2}=\frac{1}{K_{1}\cdots
K_{d}}\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\big{(}X_{n}^{K}[i_{1},\ldots,i_{d}]\big{)}^{2}=\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\langle\mathcal{X},\mathcal{I}_{i_{1},\ldots,i_{d}}^{K}\rangle^{2}.$

Now, the functions $\mathcal{I}_{i_{1},\ldots,i_{d}}^{K},1\leq i_{1}\leq
K_{1},\ldots,1\leq i_{d}\leq K_{d}$ are orthonormal, and hence by Parseval’s
inequality it follows that
$\|\mathcal{X}_{n}^{K}\|^{2}\leq\|\mathcal{X}\|^{2}\leq\beta_{N}$ almost
surely. In a similar way, it follows that

$\|\mathcal{E}_{n}^{K}\|^{2}=\frac{1}{K_{1}\cdots
K_{d}}\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\big{(}E_{n}^{K}[i_{1},\ldots,i_{d}]\big{)}^{2}\leq\beta_{N,K}^{\rm
e}\text{ almost surely}.$

Thus, under both measurement schemes, we get that
$\|\widetilde{\mathcal{X}}_{n}^{K}\|^{2}\leq\widetilde{\beta}_{N,K}:=2(\beta_{N}+\beta_{N,K}^{\rm
e})$ almost surely. This also ensures that
$\widetilde{\mathcal{C}}^{K},\mathcal{C}^{K}$ and $\Sigma^{K}$ are well-
defined. Now, since $\widetilde{\mathcal{C}}_{N}^{K}$ is the empirical
covariance based on
$\widetilde{\mathcal{X}}_{1}^{K},\ldots,\widetilde{\mathcal{X}}_{N}^{K}$,
which are i.i.d. with $\mathbb{E}(\mathcal{X}_{1}^{K})=0$,
$\textrm{Var}(\mathcal{X}_{1}^{K})=\widetilde{\mathcal{C}}^{K}$ and
$\|\widetilde{\mathcal{X}}_{n}^{K}\|^{2}\leq\widetilde{\beta}_{N,K}$ almost
surely, it follows from Lemma 12 that

$\mathbb{E}\Big{(}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widetilde{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}^{K}-\widetilde{\mathcal{C}}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}\Big{)}\leq
2\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\widetilde{\mathcal{C}}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}+\mathcal{O}\bigg{(}\frac{\widetilde{\Delta}_{N,K}^{4}}{N}\times\log\mathcal{N}\bigg{(}\frac{\widetilde{\Delta}_{N,K}}{N},\widetilde{\mathscr{F}}_{N},{|\kern-1.07639pt|\kern-1.07639pt|\cdot|\kern-1.07639pt|\kern-1.07639pt|}_{2}\bigg{)}\bigg{)},$
(E.30)

where $\widetilde{\Delta}_{N,K}=\max\\{\widetilde{\beta}_{N,K},\gamma_{N}\\}$.
For the first term on the right-hand side, we use the upper-bound

$\displaystyle\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\widetilde{\mathcal{C}}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle=\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\mathcal{C}+\mathcal{C}-\widetilde{\mathcal{C}}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle=\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\mathcal{C}+\mathcal{C}-\mathcal{C}^{K}-\Sigma^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle\leq
3\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}+3{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{C}-\mathcal{C}^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}+3{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\Sigma^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}.$
(E.31)

Secondly, because of the discretized nature of $\mathcal{C}^{K}$, we get that

${|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}^{K}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}=\frac{1}{(K_{1}\cdots
K_{d})^{2}}\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\sum_{j_{1}=1}^{K_{1}}\cdots\sum_{j_{d}=1}^{K_{d}}\iint_{V_{i_{1},\ldots,i_{d}}^{K}\times
V_{j_{1},\ldots,j_{d}}^{K}}\big{\\{}c^{K}(\text{\bf u},\text{\bf
v})-c(\text{\bf u},\text{\bf v})\big{\\}}^{2}\,\mathrm{d}\text{\bf
u}\,\mathrm{d}\text{\bf v},$

where $c^{K}$ is the kernel corresponding to $\mathcal{C}^{K}$, defined as
$c^{K}(\text{\bf u},\text{\bf v})=\mathrm{Cov}\\{\mathcal{X}_{1}^{K}(\text{\bf
u}),\mathcal{X}_{1}^{K}(\text{\bf v})\\}$. It follows that, under (M1),
$c^{K}(\text{\bf u},\text{\bf
v})=c(u_{i_{1}},\ldots,u_{i_{d}};u_{j_{1}},\ldots,u_{j_{d}})$ for $\text{\bf
u}\in V_{i_{1},\ldots,i_{d}}^{K},\text{\bf v}\in V_{j_{1},\ldots,j_{d}}^{K}$.
Thus, for $\text{\bf u}\in V_{i_{1},\ldots,i_{d}}^{K},\text{\bf v}\in
V_{j_{1},\ldots,j_{d}}^{K}$, using the Lipschitz property of $c$, we get

$\displaystyle\big{|}c^{K}(\text{\bf u},\text{\bf v})-c(\text{\bf u},\text{\bf
v})\big{\|}^{2}$ $\displaystyle\leq\rho^{2}\|(\text{\bf u},\text{\bf
v})-(u_{i_{1}},\ldots,u_{i_{d}};u_{j_{1}},\ldots,u_{j_{d}})\big{\|}^{2}$
$\displaystyle\leq\rho^{2}\bigg{(}\frac{1}{K_{1}^{2}}+\cdots+\frac{1}{K_{d}^{2}}\bigg{)}.$

Again, under (M2), $\mathcal{X}_{1}^{K}(\text{\bf u})=\langle
X,|V_{i_{1},\ldots,i_{d}}^{K}|^{-1}\textrm{{1}}\big{\\{}\text{\bf u}\in
V_{i_{1},\ldots,i_{d}}^{K}\big{\\}}\rangle$ for $\text{\bf u}\in
V_{i_{1},\ldots,i_{d}}^{K}$, so that

$\displaystyle c^{K}(\text{\bf u},\text{\bf v})$
$\displaystyle=\frac{1}{|V_{i_{1},\ldots,i_{d}}^{K}|\,|V_{j_{1},\ldots,j_{d}}^{K}|}\langle\mathcal{C}\,\textrm{{1}}\big{\\{}\text{\bf
u}\in V_{i_{1},\ldots,i_{d}}^{K}\big{\\}},\big{\\{}\text{\bf v}\in
V_{j_{1},\ldots,j_{d}}^{K}\big{\\}}\rangle_{2}$
$\displaystyle=\frac{1}{|V_{i_{1},\ldots,i_{d}}^{K}|\,|V_{j_{1},\ldots,j_{d}}^{K}|}\iint_{V_{i_{1},\ldots,i_{d}}^{K}\times
V_{j_{1},\ldots,j_{d}}^{K}}c(\text{\bf u}^{\prime},\text{\bf
v}^{\prime})\,\mathrm{d}\text{\bf u}^{\prime}\,\mathrm{d}\text{\bf
v}^{\prime}.$

Thus, under this scheme again, for $\text{\bf u}\in
V_{i_{1},\ldots,i_{d}}^{K},\text{\bf v}\in V_{j_{1},\ldots,j_{d}}^{K}$, we get

$\displaystyle\big{|}c^{K}(\text{\bf u},\text{\bf v})-c(\text{\bf u},\text{\bf
v})\big{\|}^{2}$
$\displaystyle=\bigg{|}\frac{1}{|V_{i_{1},\ldots,i_{d}}^{K}|\,|V_{j_{1},\ldots,j_{d}}^{K}|}\iint_{V_{i_{1},\ldots,i_{d}}^{K}\times
V_{j_{1},\ldots,j_{d}}^{K}}c(\text{\bf u}^{\prime},\text{\bf
v}^{\prime})\,\mathrm{d}\text{\bf u}^{\prime}\,\mathrm{d}\text{\bf
v}^{\prime}-c(\text{\bf u},\text{\bf v})\bigg{|}^{2}$
$\displaystyle=\bigg{|}\frac{1}{|V_{i_{1},\ldots,i_{d}}^{K}|\,|V_{j_{1},\ldots,j_{d}}^{K}|}\iint_{V_{i_{1},\ldots,i_{d}}^{K}\times
V_{j_{1},\ldots,j_{d}}^{K}}\underbrace{\big{|}c(\text{\bf
u}^{\prime},\text{\bf v}^{\prime})-c(\text{\bf u},\text{\bf
v})\big{|}}_{\leq\rho\sqrt{\frac{1}{K_{1}^{2}}+\cdots+\frac{1}{K_{d}^{2}}}}\,\mathrm{d}\text{\bf
u}^{\prime}\,\mathrm{d}\text{\bf v}^{\prime}\bigg{|}$
$\displaystyle\leq\rho^{2}\bigg{(}\frac{1}{K_{1}^{2}}+\cdots+\frac{1}{K_{d}^{2}}\bigg{)}.$

Hence, under both (M1) and (M2), we get that

${|\kern-1.07639pt|\kern-1.07639pt|\mathcal{C}^{K}-\mathcal{C}|\kern-1.07639pt|\kern-1.07639pt|}_{2}^{2}\leq\rho^{2}\bigg{(}\frac{1}{K_{1}^{2}}+\cdots+\frac{1}{K_{d}^{2}}\bigg{)}.$
(E.32)

Thirdly, by the assumptions of Theorem 6, it follows that

$\displaystyle{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\Sigma^{K}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}$
$\displaystyle=\frac{1}{(K_{1}\cdots
K_{d})^{2}}\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\sum_{j_{1}=1}^{K_{1}}\cdots\sum_{j_{d}=1}^{K_{d}}\big{\\{}\mathrm{Cov}(E_{n}^{K}[i_{1},\ldots,i_{d}],E_{n}^{K}[j_{1},\ldots,j_{d}])\big{\\}}^{2}$
$\displaystyle=\frac{1}{(K_{1}\cdots
K_{d})^{2}}\sum_{i_{1}=1}^{K_{1}}\cdots\sum_{i_{d}=1}^{K_{d}}\big{\\{}\textrm{Var}(E_{n}^{K}[i_{1},\ldots,i_{d}])\big{\\}}^{2}$
$\displaystyle=\frac{\sigma_{K}^{4}}{K_{1}\cdots K_{d}}.$ (E.33)

Combining (E.30)–(E.4) with (E.4), we get that

$\displaystyle\mathbb{E}\Big{(}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\widehat{\mathcal{C}}_{\widetilde{\mathscr{F}}_{N}}^{K}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}\Big{)}$
$\displaystyle\leq
18\inf_{\mathcal{G}\in\widetilde{\mathscr{F}}_{N}}{\left|\kern-1.07639pt\left|\kern-1.07639pt\left|\mathcal{G}-\mathcal{C}\right|\kern-1.07639pt\right|\kern-1.07639pt\right|}_{2}^{2}+21\rho^{2}\bigg{(}\frac{1}{K_{1}^{2}}+\cdots+\frac{1}{K_{d}^{2}}\bigg{)}+\frac{21\sigma_{K}^{4}}{K_{1}\cdots
K_{d}}$ $\displaystyle\kern
129.16626pt+\mathcal{O}\bigg{(}\frac{\widetilde{\Delta}_{N,K}^{4}}{N}\times\log\mathcal{N}\bigg{(}\frac{\widetilde{\Delta}_{N,K}}{N},\widetilde{\mathscr{F}}_{N},{|\kern-1.07639pt|\kern-1.07639pt|\cdot|\kern-1.07639pt|\kern-1.07639pt|}_{2}\bigg{)}\bigg{)},$

where $\widetilde{\Delta}_{N,K}=\max\\{2(\beta_{N}+\beta_{N,K}^{\rm
e}),\gamma_{N}\\}$. Using this, with $\gamma_{N}=R\lambda_{N}$ and the derived
bounds on the covering numbers, we get the rates as given in Theorem 6.

##  Appendix F Additional simulation results

In this section, we provide some additional simulation results which were
skipped in the main text. We start with the estimation errors for Ex 1–5 in
the main text in 2D for the fixed resolution and varying sample size regime,
which are shown in Figure 14. The results are as expected – the estimation
error for all the methods decreases as the sample size increases.

(a) Brownian sheet | (b) Rotated Brownian sheet
---|---
|
(c) Integrated Brownian sheet | (d) Rotated integrated Brownian sheet
|
(e) Matern ($\nu=0.01$)
Legend: Empirical  Best separable  Shallow  Deep  DeepShared
Figure 14:  Relative errors of different methods for different examples in 2D.
Results are reported for a fixed resolution of $25\times 25$ and varying
sample size. The numbers are averages based on $25$ simulation runs.

In Ex 1 and 3, when the true covariance is separable, although all the methods
perform almost similarly, the CovNet estimators seem to have an edge over the
others. In Ex 2 and 4, the separability of the true covariance is broken by
rotation, which has a dire consequence on the performance of the best
separable estimator, but not on the CovNet estimators. The advantage of the
CovNet estimators is more prominent for larger sample sizes, as can be seen in
the results for the integrated Brownian motion (both the usual and the rotated
versions) and the Matérn covariance models (Figures 14(c)–(e)).

(a) Rotated Brownian sheet: $N=500$ | (b) Rotated integrated Brownian sheet: $N=500$
---|---
|
(c) Matérn covariance: $N=250$, resolution $15\times 15\times 15$ | (d) Matérn covariance: $N=500$, $\nu=0.01$
|
Legend: Empirical  Best separable  Shallow  Deep  DeepShared
Figure 15:  Relative errors of different methods for different examples in 3D.
In (a) and (b), results are for a fixed sample size of $500$ and varying
resolutions. In (c), results are for sample size $250$ and resolution
$15\times 15\times 15$, with varying smoothness parameter $\nu$. In (d),
results are for sample size $500$ and $\nu=0.01$ with varying resolutions. The
numbers are averages based on $20$ simulation runs.

Next, we consider the results in 3D, which are reported in Figure 15. Here, we
only report the results for the non-separable examples (Ex 2, 4 and 5). For
the rotated examples (Ex 2 and 4), we take $\mathrm{O}$ to be the composition
of the basic rotations by $45^{\circ}$ along the $x,y$, and $z$-axes,
respectively. Formally,
$\mathrm{O}=\mathrm{O}_{z}\mathrm{O}_{y}\mathrm{O}_{x}$, where

$\mathrm{O}_{x}=\begin{pmatrix}1&0&0\\\ 0&1/\sqrt{2}&-1/\sqrt{2}\\\
0&1/\sqrt{2}&1/\sqrt{2}\end{pmatrix},\mathrm{O}_{y}=\begin{pmatrix}1/\sqrt{2}&0&1/\sqrt{2}\\\
0&1&0\\\ -1/\sqrt{2}&0&1/\sqrt{2}\end{pmatrix}\text{ and
}\mathrm{O}_{z}=\begin{pmatrix}1/\sqrt{2}&-1/\sqrt{2}&0\\\
1/\sqrt{2}&1/\sqrt{2}&0\\\ 0&0&1\end{pmatrix}.$

The best separable estimator is designed specifically for problems which have
the 2D structure (specifically, for spatio-temporal problems), with no
straightforward extension to the 3D scenario. To accommodate for this, for the
best separable estimator, we combined two of the three dimensions together to
transform the data into 2D. This gives us three different results depending on
which two dimensions are combined. In Figure 15, we report the best (minimum
error) among these three results.

In 3D, our basic findings remain the same as in 2D. The CovNet estimators
outperform the empirical and the best separable estimators, especially when
the resolution is low. Also, the smoothness of the integrated Brownian sheet
enhances the performance of the CovNet operators, as opposed to the other two
estimators. For the Matérn example, the empirical estimator performs very
poorly for smaller values of $\nu$. One interesting observation here is that
the effect of non-separability is not so severe on the best separable
estimator, particularly for the integrated Brownian sheet. This may be due to
the fact that during the construction of the best separable estimator in 3D,
we merged two dimensions together, which somehow caters for the non-
separability. Among the different CovNet estimators, the deepshared model had
the best performance.

Table 3: Relative errors (in %) of the CovNet models with hyperparameters chosen using $5$-fold cross-validation. Difference from the least observed error over the range of hyperparameters is shown in parentheses. Relative errors for the empirical and the best separable estimators are also reported. The reported numbers are based on one simulation run with $N=500$ and $K=15$. Example | Empirical | Best separable | Shallow | Deep | Deepshared
---|---|---|---|---|---
Rotated Brownian sheet 3D | $15.38$ | $33.83$ | $12.84\,(0.00)$ | $11.64\,(0.00)$ | $10.67\,(0.35)$
Rotated integrated Brownian sheet 3D | $15.35$ | $19.70$ | $16.41\,(1.54)$ | $12.57\,(0.73)$ | $11.16\,(0.81)$
Matern 3D $\nu=0.001$ | $47.52$ | $12.38$ | $10.01\,(0.00)$ | $10.94\,(1.23)$ | $10.10\,(0.00)$
Matern 3D $\nu=0.01$ | $49.14$ | $12.38$ | $11.13\,(0.00)$ | $11.83\,(0.36)$ | $11.88\,(0.98)$
Matern 3D $\nu=0.1$ | $18.94$ | $11.65$ | $12.76\,(1.72)$ | $12.21\,(0.09)$ | $11.99\,(1.07)$
Matern 3D $\nu=1$ | $9.94$ | $9.66$ | $9.64\,(0.00)$ | $9.61\,(0.00)$ | $9.45\,(0.55)$

Finally, in Table 3, we show the results for the proposed cross-validation
strategy on the examples in 3D. As before, for each example, we report
relative errors for the three CovNet models (shallow, deep and deepshared)
with hyperparameters selected via cross-validation based on a single
simulation run with $500$ observations at a resolution of $15\times 15\times
15$. For each model, we also show the difference from the least observed
relative error over the range of hyperparameters. The relative errors for the
empirical and the best separable estimators are also reported to facilitate
comparison. Similar to 2D, in all the examples, the average difference from
the best result was less than $1\%$ for all the CovNet models, while the
maximum difference was less than $1.75\%$. So, here too, the cross-validation
method identifies a good set of hyperparameters.

## References

  * Adler and Taylor (2007) [author] Adler, Robert J.R. J. and Taylor, Jonathan E.J. E. (2007). Random fields and geometry. Springer, New York.
  * Anthony and Bartlett (1999) [author] Anthony, MartinM. and Bartlett, Peter L.P. L. (1999). Neural network learning: theoretical foundations. Cambridge University Press, Cambridge.
  * Aston and Kirch (2012) [author] Aston, John ADJ. A. and Kirch, ClaudiaC. (2012). Evaluating Stationarity via Change-Point Alternatives with Applications to fMRI Data. The Annals of Applied Statistics 1906–1948.
  * Aston, Pigoli and Tavakoli (2017) [author] Aston, John A. D.J. A. D., Pigoli, DavideD. and Tavakoli, ShahinS. (2017). Tests for separability in nonparametric covariance operators of random surfaces. The Annals of Statistics 45 1431–1461.
  * Bagchi and Dette (2020) [author] Bagchi, PramitaP. and Dette, HolgerH. (2020). A test for separability in covariance operators of random surfaces. The Annals of Statistics 48 2303–2322.
  * Bauer and Kohler (2019) [author] Bauer, BenediktB. and Kohler, MichaelM. (2019). On deep learning as a remedy for the curse of dimensionality in nonparametric regression. The Annals of Statistics 47 2261–2285.
  * Baydin et al. (2018) [author] Baydin, Atilim GunesA. G., Pearlmutter, Barak A.B. A., Radul, Alexey AndreyevichA. A. and Siskind, Jeffrey MarkJ. M. (2018). Automatic Differentiation in Machine Learning: a Survey. Journal of Machine Learning Research 18 1-43.
  * Bengio (2012) [author] Bengio, YoshuaY. (2012). Practical Recommendations for Gradient-Based Training of Deep Architectures, In Neural Networks: Tricks of the Trade second ed. 437–478. Springer, Berlin, Heidelberg.
  * Buduma and Locascio (2017) [author] Buduma, NikhilN. and Locascio, NicholasN. (2017). Fundamentals of deep learning: Designing next-generation machine intelligence algorithms. O’Reilly Media, Inc.
  * Choromanska et al. (2015) [author] Choromanska, AnnaA., Henaff, MikaelM., Mathieu, MichaelM., Arous, Gérard BenG. B. and LeCun, YannY. (2015). The loss surfaces of multilayer networks. In Artificial intelligence and statistics 192–204. PMLR.
  * Constantinou, Kokoszka and Reimherr (2017) [author] Constantinou, PanayiotisP., Kokoszka, PiotrP. and Reimherr, MatthewM. (2017). Testing separability of space-time functional processes. Biometrika 104 425–437.
  * Dette, Dierickx and Kutta (2020) [author] Dette, HolgerH., Dierickx, GauthierG. and Kutta, TimT. (2020). Quantifying deviations from separability in space-time functional processes. arXiv preprint arXiv:2003.12126.
  * Dick, Kuo and Sloan (2013) [author] Dick, JosefJ., Kuo, Frances Y.F. Y. and Sloan, Ian H.I. H. (2013). High-dimensional integration: the quasi-Monte Carlo way. Acta Numerica 22 133–288.
  * Eldan and Shamir (2016) [author] Eldan, RonenR. and Shamir, OhadO. (2016). The power of depth for feedforward neural networks. In Conference on learning theory 907–940.
  * Funahashi (1989) [author] Funahashi, Ken-IchiK.-I. (1989). On the approximate realization of continuous mappings by neural networks. Neural networks 2 183–192.
  * Gneiting, Genton and Guttorp (2006) [author] Gneiting, TilmannT., Genton, Marc GM. G. and Guttorp, PeterP. (2006). Geostatistical Space-Time Models, Stationarity, Separability, and Full Symmetry. In Statistical Methods for Spatio-Temporal Systems 151-175. Chapman and Hall/CRC.
  * Golub and Van Loan (2013) [author] Golub, Gene H.G. H. and Van Loan, Charles F.C. F. (2013). Matrix computations, Fourth ed. Johns Hopkins University Press, Baltimore, MD.
  * Györfi et al. (2002) [author] Györfi, LászlóL., Kohler, MichaelM., Krzyżak, AdamA. and Walk, HarroH. (2002). A distribution-free theory of nonparametric regression. Springer-Verlag, New York.
  * Hsing and Eubank (2015) [author] Hsing, TailenT. and Eubank, RandallR. (2015). Theoretical foundations of functional data analysis, with an introduction to linear operators. John Wiley & Sons.
  * Kaipio and Somersalo (2005) [author] Kaipio, JariJ. and Somersalo, ErkkiE. (2005). Statistical and computational inverse problems. Springer-Verlag, New York.
  * Kingma and Ba (2014) [author] Kingma, Diederik PD. P. and Ba, JimmyJ. (2014). Adam: A method for stochastic optimization. arXiv preprint arXiv:1412.6980.
  * Langer (2021) [author] Langer, SophieS. (2021). Approximating smooth functions by deep neural networks with sigmoid activation function. Journal of Multivariate Analysis 182 104696\. https://doi.org/10.1016/j.jmva.2020.104696
  * Liang and Srikant (2017) [author] Liang, ShiyuS. and Srikant, RR. (2017). Why deep neural networks for function approximation? In 5th International Conference on Learning Representations.
  * Mhaskar (1996) [author] Mhaskar, Hrushikesh NH. N. (1996). Neural networks for optimal approximation of smooth and analytic functions. Neural Computation 8 164–177.
  * Ohn and Kim (2019) [author] Ohn, IlsangI. and Kim, YongdaiY. (2019). Smooth function approximation by deep neural networks with general activation functions. Entropy 21 627\.
  * Pigoli et al. (2018) [author] Pigoli, DavideD., Hadjipantelis, Pantelis ZP. Z., Coleman, John SJ. S. and Aston, John ADJ. A. (2018). The statistical analysis of acoustic phonetic data: exploring differences between spoken Romance languages. Journal of the Royal Statistical Society: Series C (Applied Statistics) 67 1103–1145.
  * Pinkus (1999) [author] Pinkus, AllanA. (1999). Approximation theory of the MLP model in neural networks. Acta Numerica 8 143–195.
  * Poggio et al. (2017) [author] Poggio, TomasoT., Mhaskar, HrushikeshH., Rosasco, LorenzoL., Miranda, BrandoB. and Liao, QianliQ. (2017). Why and when can deep-but not shallow-networks avoid the curse of dimensionality: a review. International Journal of Automation and Computing 14 503–519.
  * Raemaekers et al. (2018) [author] Raemaekers, MathijsM., Schellekens, WouterW., Petridou, NataliaN. and Ramsey, Nick F.N. F. (2018). Knowing Left from Right: Asymmetric Functional Connectivity during Resting State. Brain Structure & Function 223 1909–1922.
  * Ramsay and Silverman (2002) [author] Ramsay, J. O.J. O. and Silverman, B. W.B. W. (2002). Applied Functional Data Analysis: Methods and Case Studies. Springer-Verlag, New York.
  * Rasmussen and Williams (2006) [author] Rasmussen, Carl EdwardC. E. and Williams, Christopher K. I.C. K. I. (2006). Gaussian processes for machine learning. Adaptive Computation and Machine Learning. MIT Press, Cambridge, MA.
  * Rougier (2017) [author] Rougier, JonathanJ. (2017). A representation theorem for stochastic processes with separable covariance functions, and its implications for emulation. arXiv preprint arXiv:1702.05599.
  * Schmidt-Hieber (2020) [author] Schmidt-Hieber, JohannesJ. (2020). Nonparametric regression using deep neural networks with ReLU activation function. The Annals of Statistics 48 1875–1897.
  * Stoehr, Aston and Kirch (2021) [author] Stoehr, ChristinaC., Aston, John A DJ. A. D. and Kirch, ClaudiaC. (2021). Detecting Changes in the Covariance Structure of Functional Time Series with Application to fMRI Data. Econometrics and Statistics 18 44–62. 10.1016/j.ecosta.2020.04.004
  * Wainwright (2019) [author] Wainwright, Martin J.M. J. (2019). High-dimensional statistics: A non-asymptotic viewpoint. Cambridge University Press, Cambridge.
  * Wang, Cao and Shang (2021) [author] Wang, ShuoyangS., Cao, GuanqunG. and Shang, ZuofengZ. (2021). Estimation of the Mean Function of Functional Data via Deep Neural Networks. Stat 10 e393. For the Alzheimer’s Disease Neuroimaging Initiative. 10.1002/sta4.393
  * Wang, Chiou and Müller (2016) [author] Wang, Jane-LingJ.-L., Chiou, Jeng-MinJ.-M. and Müller, Hans-GeorgH.-G. (2016). Functional data analysis. Annual Review of Statistics and Its Application 3 257–295.
  * Wang, Wong and Zhang (2020) [author] Wang, JiayiJ., Wong, Raymond K. W.R. K. W. and Zhang, XiaokeX. (2020). Low-Rank Covariance Function Estimation for Multidimensional Functional Data. Journal of the American Statistical Association 1–14. 10.1080/01621459.2020.1820344