%- \frac{1}{\beta} \frac{1}{\alpha(1-\alpha)} \nonumber
\end{align}

The conjugate representation of $D_{\alpha}[\pi_0(a|s):\pi(a|s)]$ then follows by definition.

In this section, we show that a state-action occupancy measure which satisfies the Bellman flow constraints is normalized as a joint distribution, assuming that the initial state $\nu_0(s)$ and transition $P(s\tick|a,s)$ distributions are normalized.   More formally, if $\sum_s \nu_0(s) = 1$ and $\sum_{s\tick} P(s\tick|a,s) = 1$  $\forall a \in \mathcal{A},s \in \mathcal{S}$, then $\mu(a,s) \in \mathcal{M} \implies \mu(a,s) \in \Delta^{|\mathcal{A}| \times |\mathcal{S}|}$.

Starting from the Bellman flow constraints

\begin{align}
\sum \limits_{a} \mu(a,s) = (1-\gamma) \nu_0(s) + \gamma \sum \limits_{a\tick,s\tick} P(s|a\tick,s\tick) \mu(a\tick, s\tick)
\end{align}

we consider taking the summation over states $s \in \mathcal{S}$,

\begin{align}
\sum \limits_{a, s} \mu(a,s) &= (1-\gamma) \sum \limits_{s} \nu_0(s) + \gamma \sum \limits_{s} \sum \limits_{a\tick,s\tick} P(s|a\tick,s\tick) \mu(a\tick, s\tick) \\
\sum \limits_{a, s} \mu(a,s) &\overset{(1)}{=} (1-\gamma) \cdot 1 + \gamma \sum \limits_{s} P(s|a\tick,s\tick) \sum \limits_{a\tick,s\tick}  \mu(a\tick, s\tick) \\
\sum \limits_{a, s} \mu(a,s) &\overset{(2)}{=} (1-\gamma) \cdot 1 + \gamma \sum \limits_{a\tick,s\tick}  \mu(a\tick, s\tick)
\end{align}

where $(1)$ uses the normalization assumption on $\nu_0(s)$ and the distributive law, and $(2)$ uses the normalization assumption on $P(s|a\tick,s\tick)$.  Finally, we rearrange to obtain

\begin{align}
(1-\gamma)  \sum \limits_{a, s} \mu(a,s)  = (1-\gamma) \quad \implies \quad \sum \limits_{a, s} \mu(a,s) = 1
\end{align}

which shows that $\mu(a,s)$ is normalized as a joint distribution over $a \in \mathcal{A},s \in \mathcal{S}$, as desired.

\begin{align}
\alphaconjm &= \max \limits_{\mu} \blangle \mu(a,s), \pr(a,s) \brangle - \alphaomegamu \label{eq:conj_mu} \\
&= \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a,s} \mu_0(a,s)\big[1+\beta(\alpha-1)\pr(a,s)\big]^{\frac{\alpha}{\alpha-1}} - \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a,s}\mu_0(a,s) \label{eq:conj_mu_result}
\end{align}

\begin{align}
\alphaconjm &= \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a,s} \mu_0(a,s)\big[1+\beta(\alpha-1)\pr(a,s)\big]^{\frac{\alpha}{\alpha-1}} - \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a,s}\mu_0(a,s) \label{eq:conj_mu_result}
\end{align}

\begin{align}
% \alphaconjm &= \max \limits_{\mu} \blangle \mu(a,s), \pr(a,s) \brangle - \alphaomegamu \\
\pr(a,s) &= \frac{1}{\beta} \frac{1}{\alpha(1-\alpha)} \nabla_{\mu} \left( (1-\alpha) \sum \limits_{a,s} \mu_0(a,s) + \alpha \sum \limits_{a,s} \mu(a,s) - \sum \limits_{a,s} \mu_0(a,s)^{1-\alpha} \mu(a,s)^{\alpha} \right) \\
&= \frac{1}{\beta} \frac{1}{1-\alpha} - \frac{1}{\beta} \frac{1}{1-\alpha} \mu_0(a,s)^{1-\alpha} \mu(a,s)^{\alpha-1} \\
\implies \mu_{\pr}(a,s) &= \mu_0(a,s) [1 + \beta(1-\alpha)\pr(a,s)]^{ \frac{1}{\alpha-1}}
\end{align}

Plugging this back into the conjugate function <ref>,

\begin{align}
\alphaconjm &= \blangle \mu_{\pr}(a,s), \pr(a,s) \brangle - \frac{1}{\beta} \frac{1}{\alpha(1-\alpha)}\bigg( (1-\alpha) \sum \limits_{a,s} \mu_0(a,s) + \alpha \sum \limits_{a,s} \mu_{\pr}(a,s)  - \sum \limits_{a,s} \mu_{\pr}(a,s)  \underbrace{\left(\frac{\mu_{\pr}(a,s) }{\mu_0(a,s)}\right)^{\alpha-1}}_{1 + \beta(\alpha-1) \pr(a,s)} \bigg) \nonumber \\
&= \left(1- \frac{1}{\alpha} \right) \blangle \mu_{\pr}(a,s), \pr(a,s) \brangle - \frac{1}{\beta} \frac{1}{\alpha}\sum \limits_{a,s} \mu_0(a,s) +  \left( \frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} - \frac{1}{\beta}\frac{1}{1-\alpha} \right) \sum \limits_{a,s} \mu_{\pr}(a,s) \\
&= \text{ \footnotesize $\frac{\alpha -1}{\alpha}  \blangle \mu_{0}(a,s)
\big[1+\beta(\alpha-1)\pr(a,s)\big]^{\frac{1}{\alpha-1}}, \pr(a,s) \brangle + \frac{1}{\beta} \frac{1}{\alpha} \sum \limits_{a,s} \mu_{0}(a,s)\big[1+\beta(\alpha-1)\pr(a,s)\big]^{\frac{1}{\alpha-1}} - \frac{1}{\beta} \frac{1}{\alpha} \sum \limits_{a,s} \mu_{0}(a,s)$} \nonumber \\
% &= \frac{\alpha -1}{\alpha}  \blangle \mu_{0}(a,s)[1+\beta(\alpha-1)\pr(a,s)]^{\frac{1}{\alpha-1}}, \pr(a,s) \brangle \\[1.5ex]
% &\phantom{====} + \frac{1}{\beta} \frac{1}{\alpha} \sum \limits_{a,s} \mu_{0}(a,s)[1+\beta(\alpha-1)\pr(a,s)]^{\frac{1}{\alpha-1}} - \frac{1}{\beta} \frac{1}{\alpha} \sum \limits_{a,s} \mu_{0}(a,s) \nonumber \\
&=  \sum \limits_{a,s} \mu_0(a,s)\mu_{0}(a,s)\big[1+\beta(\alpha-1)\pr(a,s)\big]^{\frac{1}{\alpha-1}}\cdot \frac{1}{\beta}\frac{1}{\alpha} \big( 1 + \beta (\alpha-1) \pr(a,s) \big) - \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a,s}\mu_0(a,s) \\
&=  \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a,s} \mu_0(a,s)\big[1+\beta(\alpha-1)\pr(a,s)\big]^{\frac{\alpha}{\alpha-1}} - \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a,s}\mu_0(a,s)
\end{align}

where, in the last line, note that $\frac{1}{\alpha-1}+1=\frac{\alpha}{\alpha-1}$.

§.§ KL Divergence Conjugate with $\mu(a,s)$ Optimization

The conjugate function for divergence regularization of the state-action occupancy $\mu(a,s)$ to a reference $\mu_0(a,s)$ has the following closed form.

\begin{align}
\frac{1}{\beta} \Omega^{*}(\pr) &= \frac{1}{\beta} \sum \limits_{a,s} \mu(s)\pi_0(a|s) \expof{\beta \cdot \pr(a,s)}  - \frac{1}{\beta} \sum \limits_{a,s} \mu(s) \pi_0(a|s)
\end{align}

\begin{align}
\frac{1}{\beta} \Omega^{*}(\pr) = \max \limits_{\mu(a,s)} \blangle \mu(a,s), \pr(a,s) \brangle - \frac{1}{\beta} \blangle \mu(a,s) \log \frac{\mu(a,s)}{\mu(s) \pi_0(a|s)} \brangle + \frac{1}{\beta} \sum \limits_{a,s} \mu(a,s) - \frac{1}{\beta} \sum \limits_{a,s} \mu(s) \pi_0(a|s)
\end{align}

\begin{align}
\pr(a,s) &\overset{(1)}{=} \frac{1}{\beta} \log \frac{\mu(a,s)}{\mu(s) \pi_0(a|s)} - \frac{1}{\beta}
% \sum \limits_{a,s} \frac{d \mu(a,s)}{d \mu(a^\prime, s^\prime)}
+ \frac{1}{\beta}  \sum \limits_{a,s} \frac{\mu(a,s)}{\mu(s)} \underbrace{
\frac{d \sum_{a^{\prime\prime}} \mu(a^{\prime\prime},s)}{d \mu(a^\prime, s^\prime)}}
+ \frac{1}{\beta} -  \frac{1}{\beta}  \sum \limits_{a,s}
\underbrace{
\frac{d \sum_{a^{\prime\prime}} \mu(a^{\prime\prime},s)}{d \mu(a^\prime, s^\prime)}
}_{\delta(s=s^{\prime})} \pi_0(a|s) \\
&= \frac{1}{\beta} \log \frac{\mu(a,s)}{\mu(s) \pi_0(a|s)} +  \frac{1}{\beta}  \sum \limits_{a} \frac{\mu(a,s)}{\mu(s)}
% \textcolor{red}{\pi(a|s) \footnote{Notation?}}
- \frac{1}{\beta} \sum \limits_{a} \pi_0(a|s) \\
&\overset{(2)}{=} \frac{1}{\beta} \log \frac{\mu(a,s)}{\mu(s) \pi_0(a|s)}
\end{align}

where in $(1)$ we take care to differentiate $\mu(s)$ with respect to $\mu(a,s)$.
In $(2)$ we assume that $\sum_a \pi_0(a|s) = 1$ and note that $\sum_a \frac{\mu(a,s)}{\mu(s)} = \frac{\sum_a \mu(a,s)}{\mu(s)} = \frac{\mu(s)}{\mu(s)} =1 $.

Defining $\pi_{\pr}(a|s) = \frac{\mu(a,s)}{\mu(s)}$,

\begin{align}
\pi_{\pr}(a|s) = \pi_0(a|s) \expof{\beta \cdot \pr(a,s)}
\end{align}

Plugging this back into the conjugate optimization, we have

\begin{align}
\frac{1}{\beta} \Omega^{*}(\pr) &=  \blangle \mu(s) \pi_{\pr}(a,s), \pr(a,s) \brangle - \frac{1}{\beta} \blangle \mu(s) \pr(a,s)
\cancel{\log \exp} \{ \beta \cdot \pr(a,s) \} \brangle \\
&\phantom{= \mu(s) \pr(a,s), \pr(a,s) \pr(} + \frac{1}{\beta} \sum \limits_{a,s} \mu(s)\pi_0(a|s) \expof{\beta \cdot \pr(a,s)}  - \frac{1}{\beta} \sum \limits_{a,s} \mu(s) \pi_0(a|s) \nonumber \\
&= \frac{1}{\beta} \sum \limits_{a,s} \mu(s)\pi_0(a|s) \expof{\beta \cdot \pr(a,s)}  - \frac{1}{\beta} \sum \limits_{a,s} \mu(s) \pi_0(a|s)
\end{align}

§.§ Feasible Set for State-Action Occupancy Regularization

In this section, we provide background on soft value aggregation [Fox et al., 2016, Haarnoja et al., 2017] and the regularized Bellman optimality operator [Neu et al., 2017, Geist et al., 2019], which rely on the convex conjugate functions derived in conj_examples and conjugates.

We give a non-standard derivation of the regularized Bellman optimality operator.   Factorizing $\mu(a,s) = \mu(s) \pi(a|s)$, we can imagine optimizing over $\mu(s)$ and $\pi(a|s) \in \simplex$ separately,

\begin{align}
%   \mathcal{L}:=
\max \limits_{\mu(s)\rightarrow \mathcal{M}} \max \limits_{\pi(a|s)} \min \limits_{V(s)}  \,
(1-\gamma) \big\langle \nu_0(s), V(s) \big  \rangle
+ &\blangle \mu(a,s), r(a,s) +  \gamma \transitionv -V(s)\brangle - \frac{1}{\beta} \Omega_{\pi_0}(\mu)  \label{eq:dual_lagrangian_app}
\end{align}

Eliminating $\mu(s)$ by setting $d/d\mu(s) = 0$, we obtain the regularized Bellman operator

\begin{align}
V(s)  = \blangle \pi(a|s), \, r(s,a) + \gamma \transitionv \brangle - \frac{1}{\beta} \Omega_{\pi_0}(\pi) \nonumber . % \label{eq:reg_bell}
\end{align}

Eliminating $\mu(s)$ thus leads to a constraint on the form of $V(s)$, since both may be viewed as enforcing the Bellman flow constraints.  Plugging this expression back into the Lagrangian <ref> and removing the minimization over $V(s)$, we are left with only the initial state term and a maximization over $\pi(a|s) \in \Delta^{|\mathcal{A}|}$.

Recalling the definition of the optimal state-action value, or $Q$ function, ${Q_*(s,a)= r(s,a) + \gamma \transitionv}$, which is recursively defined in terms of value function estimates $V(s\tick)$, we obtain the following optimization

\begin{align}
\mathcal{R}_{\Omega, \beta}^{*}(r) &= \, \max \limits_{\pi(a|s) \in \Delta^{|\mathcal{A}|}} (1-\gamma) \, \blangle \nu_0(s) , V(s) \brangle  \,\, \text{subj. $ V(s)= \blangle \pi(a|s), r(s,a) + \gamma \transitionv \brangle -  \klomegapi$} \label{eq:bellman_reg} \\
&=(1-\gamma) \, \mathbb{E}_{\nu_0} \big[\max \limits_{\pi \in \Delta^{|\mathcal{A}|}} \langle \pi(a|s), Q_*(s,a)  \rangle - \klomegapi \big]  \label{eq:soft_bellman}
% \frac{1}{\beta} \Omega_{\pi_0}(\pi)
\end{align}

This conveniently involves expectations under $\nu_0(s)$, and leads to `soft' or regularized versions of algorithms such as value iteration (Fox et al., 2016, Haarnoja et al., 2017) and modified policy iteration [Geist et al., 2019].

Maximizing over $\pi \in \simplex$ in <ref>, we can recognize the optimal value function $V_*(s) = \alphaconjn$ as the conjugate function, also known as the regularized Bellman optimality operator [Geist et al., 2019].
We derived closed forms for $V_*(s)$ and the optimal policy $\piopt(a|s)$ in conjugates and conj_examples.

§.§ Soft Value Function by $\alpha$-Divergence

In <ref>, we visualize the optimal value function $V_*(s) = \frac{1}{\beta}\Omega^{*}_{\pi_0,\beta}(Q)$, for kl or $\alpha$-divergence regularization and different choices of regularization strength $1/\beta$.
The choice of divergence particularly affects the aggregated value for low regularization, although we do not observe a clear pattern with respect to $\alpha$.[See Belousov & Peters, 2019, Lee et al., 2018, Lee et al., 2019, or <ref> for additional discussion of the effect of $\alpha$-divergence regularization on learned policies.]
In all cases, the value function ranges between $\max_a Q(a,s)$ for an unregularized deterministic policy as $\beta \rightarrow \infty$, and the expectation under the reference policy $\mathbb{E}_{\pi_0}[Q(a,s)]$ for strong regularization as $\beta \rightarrow 0$.   We further discuss this property in entropy_vs_div.

In this section, we derive the worst-case reward perturbations for the adversary associated with a particular policy (<ref>, optimal_perturbations) and show that they preserve the value of the regularized objective in <ref>.  In contrast to conj_examples and conjugates, we consider the conjugate $\alphaconjmu$ with optimization over the full state-action occupancy measure $\mu(a,s)$.
However, when analyzing the optimal policy $\piopt(a|s)$ in advantage_pf, we will find that this approach sheds insight into the relationship between $V_*(s)$ and $\normalizeralpha$, which both appear in the conjugate with optimization over $\pi(a|s)$ in
<ref> and alpha_conj_app.

§.§ Derivation of Worst-Case Reward Perturbations (Proof of optimal_perturbations)

Compared with the main text, note that we include a Lagrange multiplier $\lambdazero$ enforcing $\mu(a,s) \geq 0$.   For $\mu(a,s)>0$ or $\pi(a|s)>0$, we have $\lambdazero= 0$ by complementary slackness [Boyd & Vandenberghe, 2004].

Recall from the conjugacy conditions in conj_intro that $\propt(a,s) = \nabla \omegamu = (\nabla \alphaconjmu)^{-1}(\mu)$.   We will derive the optimal reward perturbations via the first equality  $\propt(a,s) = \nabla \omegamu$ for convenience, since we will see that the conjugate $\alphaconjmu$ can be difficult to manipulate directly.

We consider the conjugate optimization over $\mu(a,s)$, and include a Lagrange multiplier $\lambdazero$ to enforce $\mu(a,s) \geq 0$. We justify ignoring normalization constraints below, and write the optimization as

\begin{align}
\alphaconjmu(\pr) &= \max \limits_{\mu(a,s)} \blangle \mu(a,s), \pr(a,s) \brangle - \alphanmu - \lambdazero \label{eq:mu_as_conj}
\end{align}

where we treat the conditional regularization $\alphanmu= \mathbb{E}_{\mu(s)}\big[ D_{\alpha}[\pi_0(a|s): \frac{\mu(a,s)}{\mu(s)}] \big]$ as a function of the full state-action occupancy measure $\mu(a,s)$, with $\pi(a|s) = \mu(a,s) / \mu(s)$ and

\begin{align}
\alphanmu
% &= \mathbb{E}_{\mu(s)}\big[ D_{\alpha}[\pi_0(a|s): \underbrace{\frac{\mu(a,s)}{\mu(s)}}_{\pi(a|s)}] \big] \nonumber \\
&= \frac{1}{\beta}\frac{1}{\alpha (1-\alpha)} \left( (1-\alpha) \sum \limits_{a,s} \mu(s) \pi_0(a|s) + \alpha \sum \limits_{a,s} \mu(a,s) - \sum \limits_{a,s} \mu(s) \pi_0(a|s)^{1-\alpha} \left(\frac{\mu(a,s)}{\mu(s)}\right)^{\alpha} \right) \label{eq:joint_div}
% &= \frac{1}{\beta}\frac{1}{\alpha (1-\alpha)} \left( 1 - \sum \limits_{a,s} \mu(s) \pi_0(a|s)^{1-\alpha} \left(\frac{\mu(a,s)}{\mu(s)}\right)^{\alpha} \right) \label{eq:joint_div}
\end{align}

After rearranging the exponents in the second term of  <ref>,
we can solve for the optimal $\mu(a,s)$ by differentiating <ref> with respect to $\mu$.
We explicitly denote the arguments of differentiation and summation to highlight the care needed to differentiating the marginal $\mu(s\tick)$ with respect to $\mu(a,s)$, with $\sum_{s\tick} \frac{d}{d\mu(a,s)}\mu(s\tick) = \sum_{s\tick, a\tick} \frac{d}{d\mu(a,s)} \mu(a\tick, s\tick) = \sum_{s\tick, a\tick} \delta(a\tick,s\tick = a,s) = 1$.  See Neu et al., 2017, Lee et al., 2019 for similar derivations.

\begin{align}
\pr&(a,s) = \nabla_{\mu} \alphanmu  - \lambdazero \label{eq:pr_equals_grad} \\
% &= \frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} \sum \limits_{a\tick,s\tick}  \frac{d}{d\mu(a,s)} \mu(s\tick)^{1-\alpha} \pi_0(a\tick|s\tick)^{1-\alpha} \mu(a\tick,s\tick)^{\alpha} \\
&= \frac{1}{\beta}\frac{1}{\alpha(1-\alpha)}  \sum \limits_{a\tick,s\tick}  \frac{d}{d\mu(a,s)} \bigg( (1-\alpha) \mu(s\tick)\pi_0(a\tick|s\tick) + \alpha \mu(a\tick, s\tick) - \mu(s\tick)^{1-\alpha} \pi_0(a\tick|s\tick)^{1-\alpha} \mu(a\tick,s\tick)^{\alpha} \bigg)  - \lambdazero \nonumber \\
&=  - \lambdazero + \frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} \bigg(
(1-\alpha) \sum \limits_{s\tick} \underbrace{\frac{d \sum_{a\tick} \mu(a\tick, s\tick)}{d\mu(a,s)}}_{\delta(s = s\tick)} \sum \limits_{a\tick} \pi_0(a\tick|s)  + \alpha
\sum \limits_{a\tick,s\tick} \underbrace{\frac{d\mu(a\tick,s\tick)}{d\mu(a,s)}}_{\delta(a\tick,s\tick = a,s)}  -
\\ %phantom{= \frac{1}{\beta}\frac{1}{(1-\alpha)} \bigg(}
&\phantom{==} -\sum \limits_{a\tick,s\tick}  \alpha \mu(a\tick,s\tick)^{\alpha-1} \underbrace{\frac{d\mu(a\tick,s\tick)}{d\mu(a,s)}}_{\delta(a\tick,s\tick = a,s)} \mu(s\tick)^{1-\alpha} \pi_0(a\tick|s\tick)^{1-\alpha}- (1-\alpha) \sum \limits_{a\tick,s\tick} \mu(s\tick)^{-\alpha} \underbrace{\frac{d \sum_{a\tick} \mu(a\tick, s\tick)}{d\mu(a,s)}}_{\delta(s = s\tick)} \cdot \pi_0(a\tick|s\tick)^{1-\alpha} \mu(a\tick,s\tick)^{\alpha} \bigg) \nonumber \\
&= \frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} \bigg( (1-\alpha) \textcolor{red}{\sum_{a} \pi_0(a|s)} + \alpha - \alpha \left(\frac{\mu(a,s)}{\mu(s) \pi_0(a|s)} \right)^{\alpha-1} - (1-\alpha) \sum \limits_{a\tick}   \pi_0(a|s)^{1-\alpha}\left(\frac{\mu(a,s)}{\mu(s)} \right)^{\alpha}  \bigg)  - \lambdazero \nonumber
\end{align}

Finally, with this optimality condition, we substitute $\pi_*(a|s) = \frac{\mu(a,s)}{\mu(s)}$ to obtain

\begin{align}
\pr(a,s) =
\textcolor{red}{\frac{1}{\beta}\frac{1}{\alpha} \sum_{a} \pi_0(a|s) + \frac{1}{\beta}\frac{1}{1-\alpha}}
%   \frac{1}{\beta}\frac{1}{\alpha (1-\alpha)}
- \frac{1}{\beta}\frac{1}{1-\alpha} \left(\frac{\pi_*(a|s)}{\pi_0(a|s)} \right)^{\alpha-1} - \frac{1}{\beta}\frac{1}{\alpha} \sum \limits_{a} \pi_0(a|s)^{1-\alpha} \pi_*(a|s)^{\alpha}  - \lambdazero\label{eq:propt_intermediate}
\end{align}

Using $\frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} = -\frac{1}{\beta}\frac{1}{\alpha(\alpha-1)} = \frac{1}{\beta}\frac{1}{(\alpha-1)} - \frac{1}{\beta}\frac{1}{\alpha}$, we split the constant into two terms, which allows us to recognize a $\log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)}$ term and $-\myconst$ term in <ref> for $\sum \limits_{a} \pi_0(a|s) = 1$.
We first notice that <ref> contains terms which combine to yield $\frac{1}{\beta} \log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)}$.   Further, if we assume that $\sum \limits_{a} \pi_0(a|s) = 1$ and notice that $\sum \limits_{a} \pi(a|s) = \sum \limits_{a} \frac{\mu(a,s)}{\sum \limits_a \mu(a,s)} = 1$, then we can also recognize a term of $-\myconst$ term,

\begin{align}
\pr(a,s) = \frac{1}{\beta}\underbrace{ \frac{1}{\alpha-1}  \bigg( \left(\frac{\pi(a|s)}{\pi_0(a|s)} \right)^{\alpha-1} - 1 }_{\log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)}}\bigg) -  \frac{1}{\beta}\bigg( \underbrace{\frac{1}{\alpha}  \sum \limits_{a} \pi_0(a|s)^{1-\alpha} \pi(a|s)^{\alpha} - \frac{1}{\alpha}\sum \limits_{a} \pi_0(a|s)}_{-\myconst} \bigg)  - \lambdazero \, .\label{eq:worst_case_reward_perturba}
\end{align}

As desired, we obtain

\begin{align}
\pr(a,s) = \frac{1}{\beta}\log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)} + \frac{1}{\beta} \myconst  - \lambdazero\, . \label{eq:worst_case_reward_perturb}
\end{align}

By complementary slackness, we know that $\lambdazero=0$ when $\mu(a,s) > 0$, so that the perturbations are not affected for state-action pairs with non-zero probability.   In subsequent derivations, we will frequently consider expectations of the form $\langle \pi(a|s), \propt(a,s) \rangle$.   Note that we can ignore the contribution of $\lambdazero$ in these expressions, since either $\lambdazero = 0$ or $\mu(a,s) = 0$ and $\pi(a|s) = 0$.

§.§.§ Conjugate Function using $\mu(a,s)$ Optimization

We can use the above derivations to obtain the conjugate function $\alphaconjmu$, which treats the $\alpha$-divergence as a function of $\mu(a,s)$.   This proposition will allow us to show that the conjugate preserves the value of the objective in preserve.

[Conjugate Function using $\mu(a,s)$ Optimization]propositionconjugatemu
The conjugate $\alphaconjmu(\dualvar)$ of the $\alpha$-divergence $\omegamu = \mathbb{E}_{\mu(s)}\big[ D_{\alpha}[\pi_0(a|s): \pi(a|s)] \big]$ in <ref> as a function of $\mu(a,s)$, can be written

\begin{align}
\hspace*{-.2cm} \alphaconjmu(\pr) = \frac{\alpha-1}{\alpha} \blangle \mur(s) \pir(a|s), \pr(a,s) \brangle + \frac{1}{\alpha} \normalizerpr \label{eq:conj_alpha_mu_app}
% - \frac{\alpha-1}{\alpha}\blangle \mur(s), D_{\alpha}[\pi_0:\pir] \brangle= \frac{\alpha-1}{\alpha} \mathbb{E}_{\mu(s)}\bigg[ \alphaconjn(\pr)\bigg] \nonumber
\end{align}

RESTATE / CLARIFY
where $\pir(a|s)=\frac{\mur(a,s)}{\mur(s)}$, which is a function of $\pr(a,s)$, is the policy derived from the state-action occupancy measure that corresponds to $\pr$ via the conjugacy conditions
$\mur(a,s) = \nabla \alphaconjmu(\dualvar) = (\nabla \Omega^{(\alpha)}_{\pi_0, \mu})^{-1}(\dualvar)$

\begin{align}
\pir(a|s) = \pi_0(a|s) \big[ 1 +  \beta (\alpha-1) \cdot \big( \pr(a,s) -\normalizerpr \big) \big]_+^{\frac{1}{\alpha-1}} \label{eq:policy_for_perturbed}
% \pir(a|s) = \pi_0(a|s) \bigg[ 1 +  \beta (\alpha-1) \cdot \big( \pr(a,s) - \frac{1}{\beta}\mydiv{\pir} \big) \bigg] \label{eq:policy_for_perturbed}
\end{align}

Starting from <ref>, we can solve for $\pir(a|s)$ in self-consistent fashion, since $\mydiv{\pir}$ depends on $\pir(a|s)$, to obtain (<ref>).   Note that we substitute $\normalizerpr = \frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pir]$ in order to emphasize that $\normalizerpr$ should be treated as a constant and solved for numerically, thus avoiding recursive substitution of $\pir$ terms.

This notation also highlights the role of $\normalizerpr = \frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pir]$ as a normalization constant that is different than $\normalizeralpha$ (which normalizes the policy for the conjugate $\alphaconjn(Q)$ with $Q$-values as inputs, as in conj_examples and conjugates).    REDO THIS... TOTALLY UNCLEAR

If $\mur(a,s)$ is normalized, note that $\pir(a|s)= \mur(a,s) / \sum_a \mur(a,s)$ is normalized by construction (see proof that $\mu \in \mathcal{M}$ is normalized).  When plotting the feasible sets in feasible, we confirm that <ref> is normalized for the single-step setting, with $\normalizerpr$ matching $\frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pir]$ and the normalization constant of <ref> obtained by numerical optimization.

Plugging this optimizing argument $\pir(a|s)$ back into <ref>, we obtain an expression for the conjugate $\alphaconjmu$, which matches the form of the conjugate $\alphaconjn$ with optimization over $\pi(a|s)\in \simplex$ in <ref> and <ref>.

\begin{align}
\alphaconjmu(\pr) &= \blangle \mur(s) \pir(a|s), \pr(a,s) \brangle - \frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} +\frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} \blangle \pir(a|s),  \left( \frac{\pir(a|s)}{\pi_0(a|s)}\right)^{\alpha-1} \brangle \\
&=\blangle \mur(s) \pir(a|s), \pr(a,s) \brangle - \frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} +\frac{1}{\beta}\frac{1}{\alpha(1-\alpha)} \blangle \pir(a|s), 1\brangle \\
&\phantom{====} +  \frac{1}{\alpha} \frac{\beta}{\beta}\frac{(\alpha-1)}{(1-\alpha)}  \blangle \pir(a|s), \pr(a,s) \brangle - \frac{1}{\alpha} \frac{\beta}{\beta}\frac{(\alpha-1)}{(1-\alpha)} \normalizerpr \nonumber \\
&= \frac{\alpha-1}{\alpha} \blangle \mur(s) \pir(a|s), \pr(a,s) \brangle + \frac{1}{\alpha} \normalizerpr
\end{align}

§.§ Worst-Case Reward Perturbations Preserve Value of the Objective

We now confirm that the conjugate optimization matches the value of the regularization function $\omegamu = \max_{\pr} \langle \mu(a,s), \pr(a,s) \rangle - \alphaconjmu(\pr)$ when evaluated with the worst-case reward perturbations $\propt(a,s)$, as we would expect from the definition in <ref>.

We first show that $\alphaconjmu(\propt)=0 \, \forall \alpha$ in the following proposition, which will also be useful in deriving the feasible set in feasible.

For the worst-case reward perturbation $\propt(a,s)$ associated with a given policy $\pi(a|s)$ or occupancy measure $\mu(a,s)$, evaluating the conjugate function, for any $\alpha$, yields

\begin{align}
\alphaconjmu = 0
\end{align}

Treating the conjugate function $\alphaconjmu(\pr)$ as a function of $\pr$, we substitute the worst-case reward perturbations $\propt(a,s)$ from optimal_perturbations and <ref>.  Note that $\propt(a,s)$ is a function of the original $\pi(a|s)$.  We would like to show that $\omegamu = \langle \mu(s)\pir(a|s), \propt(a,s) \rangle - \alphaconjmu(\propt)$.

First, we can simplify the expression for $\pir(a,s)$ in <ref>, using

\begin{align*}
\pir(a|s) &= \pi_0(a|s) \big[ 1 +  \beta (\alpha-1) \cdot \big( \pr(a,s) -\normalizerpr \big) \big]_+^{\frac{1}{\alpha-1}} \nonumber \\
&= \pi_0(a|s) \big[ 1 +  \beta (\alpha-1) \cdot \big( \frac{1}{\beta}\frac{1}{\alpha-1}\big( \frac{\pi(a|s)}{\pi_0(a|s)} \big)^{\alpha-1} - \frac{1}{\beta}\frac{1}{\alpha-1} + \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\pi] - \underbrace{\frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\pi]}_{\normalizerpr} \big) \big]_+^{\frac{1}{\alpha-1}} \nonumber \\
&= \pi(a|s) \nonumber
\end{align*}

which is to be expected since $\pi(a|s)$ (or $\pir(a|s)$) and $\propt(a|s)$ are in dual correspondence.
Plugging this into conjugate_mu <ref>,

\begin{align*}
\hspace*{-.2cm} \alphaconjmu(\pr) &= \frac{\alpha-1}{\alpha} \blangle \mu(s) \pi(a|s), \pr(a,s) \brangle + \frac{1}{\alpha} \normalizerpr \\
&= \frac{\alpha-1}{\alpha} \blangle \mu(s) \pi(a|s), \frac{1}{\beta}\frac{1}{\alpha-1} \frac{\pi(a|s)}{\pi_0(a|s)}^{\alpha-1} - \frac{1}{\beta}\frac{1}{\alpha-1}\brangle  +  \frac{\alpha-1}{\alpha}\frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pi] + \frac{1}{\alpha} \normalizerpr \nonumber \\
&\overset{(1)}{=}  -\frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pi] +  \left( \frac{\alpha-1}{\alpha} +\frac{1}{\alpha} \right) \frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pi]  \\
&= 0
\end{align*}

where, in $(1)$, we have used $\normalizerpr:=\frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pi]$ and noted that a negative sign is necessary to obtain $(1-\alpha) D_{\alpha}[\pi_0:\pi] = \frac{1}{\alpha}(1 - \sum \pi_0(a|s)^{1-\alpha}\pi(a|s)^{\alpha}$ in the first term.

For the worst-case reward perturbations $\propt(a,s)$ associated with a given policy $\pi(a|s)$, the conjugate expansion does not change the value of the objective function in <ref>.  In other words, we confirm

\begin{align}
\omegamu  = \langle \pi(a|s), \propt(a,s) \rangle - \alphaconjmu(\propt) \, .
\end{align}

We have shown in conjugatezero that $\alphaconjmu(\propt) = 0$.   Substituting the reward perturbations $\propt(a,s)$, we can follow similar derivations to confirm

\begin{align*}
\omegamu  &= \langle \mu(s) \pi(a|s), \propt(a,s) \rangle - \langle \mu(s), \alphaconjmu(\propt) \rangle \\
&= \blangle \mu(s) \pi(a|s), \frac{1}{\beta}\frac{1}{\alpha-1} \frac{\pi(a|s)}{\pi_0(a|s)}^{\alpha-1} - \frac{1}{\beta}\frac{1}{\alpha-1}\brangle  +  \blangle \mu(s), \frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pi] \brangle - 0 \\
&= \blangle \mu(s), \frac{1}{\beta}\,\alpha\, D_{\alpha}[\pi_0:\pi] \brangle + \blangle \mu(s), \frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pi] \brangle \\
&=  \blangle \mu(s), \frac{1}{\beta} D_{\alpha}[\pi_0:\pi] \brangle = \omegamu \, .
\end{align*}

§.§ Relationship with Advantage Function (Proof of advantage)

We now prove the relationship between the worst-case reward perturbations and the advantage function.   Our derivations assume optimal values, with $Q_*(a,s) = r(a,s) + \gamma \transitionvstar$ and $V_*(s) = \max \limits_{\pi \in \simplex} \langle \pi(a|s), Q(a,s) \blangle - \omegamu$.   We prove the following proposition from optimal_policy.

We begin by considering the Lagrangian dual of the regularized objective in <ref>

\begin{align}
\max \limits_{\mu(a,s)} \min \limits_{V(s)} & \, (1-\gamma) \big\langle \nu_0(s), V(s) \big  \rangle
+ \blangle \mu(a,s), r(a,s) +  \gamma \transitionv -V(s)\brangle - \omegamu \, . \nonumber
% \label{eq:dual_lagrangian}
\end{align}

Eliminating $\mu(a,s)$ by setting $\frac{d}{d\mu}=0$, we can reuse our derivations in <ref>-(<ref>) to show that

\begin{align}
r(a,s) +  &\gamma \transitionv -V(s) - \nabla \omegamu = 0 \label{eq:kkt} \\
r(a,s) +  & \gamma \transitionv -V(s) - \frac{1}{\beta} \frac{1}{\alpha-1} \big( \frac{\pi(a|s)}{\pi_0(a|s)}^{\alpha-1} - 1 \big) - \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\pi] = 0 \\
\implies \pi(a|s) &= \pi_0(a|s) [ 1 + \beta (\alpha-1) ( r(a,s) +  \gamma \transitionv -V(s) - \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\pi]) ]_+^{\frac{1}{\alpha-1}} \label{eq:opt_alpha_pol_app}
% \implies \mu(a,s)
\end{align}

Recall that the optimal Lagrange multipliers $V_*(s)$ enforce the Bellman flow constraints in <ref> which, for normalized $\nu_0(s) \in \Delta^{|\mathcal{S}|}$, implies that $\mu(a,s) \in \Delta^{|\mathcal{S}| \times |\mathcal{A}|}$ and thus $\pi(a|s) \in \Delta^{|\mathcal{A}|}$ are normalized.
Plugging in the optimal $V_*(s)$ and $Q_*(a,s) = r(a,s) + \gamma \transitionvstar$, we can thus consider the second two terms to be the normalization constant for the optimal policy $\piopt(a|s)$.    As in conj_examples and conjugates, we denote this normalizer as $\normalizeralpha$, with

\begin{align}
\normalizeralpha =  V_*(s) + \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\piopt] \, . \label{eq:psidentity_app}
\end{align}

See Lee et al., 2019 App. D2 for similar reasoning.   Note that $\normalizeralpha$ also corresponds to the normalization constant for the optimal policy in the conjugate optimization $\alphaconjn(Q) = \max_{\pi \in \simplex} \langle \pi, Q \rangle - \omegapi$.
Further, note that <ref> is a self-consistent equation since all three terms interact via the policy $\piopt(a|s)$.

While we found this identity useful in our simplified exposition in optimal_policy, it is not strictly necessary in the below proof.

Substituting the optimal values into the policy in <ref>, we have

\begin{align}
\piopt(a|s) &= \pi_0(a|s) [ 1 + \beta (\alpha-1) ( Q_{*}(a,s) \underbrace{- V_*(s) - \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\piopt])}_{-\normalizeralpha} ]_+^{\frac{1}{\alpha-1}}  \\
&= \pi_0(a|s) \exp_{\alpha}\big\{ \beta \cdot \big(  Q_{*}(a,s) - V_*(s) - \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\piopt] \big) \big\}
\end{align}

Finally, we can rewrite the worst-case reward perturbations $\propt(a,s)$ (<ref> or <ref>) for this optimal policy as follows

\begin{align}
\propt(a,s) &= \frac{1}{\beta}\log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)} + \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\piopt] \\
&= Q_{*}(a,s) - V_*(s) - \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\piopt]  +\frac{1}{\beta} D_{\alpha}[\pi_0:\piopt] \\
&= Q_{*}(a,s) - V_*(s)
\end{align}

which demonstrates that the worst-case reward perturbations $\propt(a,s)$ for the optimal policy $\piopt(a|s)$ correspond to the (optimal) advantage function $\propt(a,s) =  Q_{*}(a,s) - V_*(s)$.

§.§ Confirming the Relationship between Normalization and Value Function

It is known that $V_*(s) = \normalizerexp$ in the case of kl divergence regularization [Ortega & Braun, 2013, Fox et al., 2016, Haarnoja et al., 2017].  However, the relationship between $V_*(s)$ and $\normalizeralpha$ for general $\alpha$-divergences is less well-understood.   In conj_by_alpha_app, we plot both $V_*(s)$ and $\normalizeralpha$ for various values of $\alpha$ (x-axis) and $\beta$ (in each panel).     We also plot the value of $\frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\piopt]$ in the third row, and confirm the identity in <ref> in the fourth row.

As we also observed in value_agga, the soft value function or certainty equivalent $V_*(s) = \alphaconjn(Q)$ is not monotonic in $\alpha$ for this set of single-step rewards (the same $r(a,s)$ as in perturb_opt or value_agga).

While it can be shown that $\normalizeralpha$ is convex as a function of $\beta$ <cit.>, we see that $\normalizeralpha$ is not necessarily convex in $\alpha$ and appears to be monotonically decreasing in $\alpha$.   Finally, we find that the identity in <ref> holds empirically, with only small numerical optimization issues.

We rewrite the derivations of Husain et al., 2021 where $\Omega \big( \mu(a,s) \big)$ represents a convex regularizer

\begin{align}
\obj_{\Omega, \beta}(r) = \max \limits_{\mu(a,s)} \obj_{\Omega, \beta}(\mu) &= \max \limits_{\mu(a,s)} \,\,  \langle \mu(a,s), r(a,s) \rangle - \frac{1}{\beta} \Omega \big( \mu(a,s) \big)  \label{eq:reg_rl} \\
\text{subj. to: } \, \, & \int \mu(a,s) \, da = (1-\gamma) \nu_0(s) + \gamma \int p(s|s',a') \, \mu(s',a')  ds \, da \, \, \forall s \in \mathcal{S} \nonumber
\end{align}

We can let $\Omega$ regularize the full state-action distribution, or the conditional policy $\pi(a|s) = \mu(a,s) / \sum_a \mu(a,s)$), and denote the feasible set for the above constraints as $\mudomain$, so that we can write

\begin{align}
\max \limits_{\mu(a,s) \in \mudomain} \obj_{\Omega, \beta}(\mu) &= \max \limits_{\mu(a,s) \in \mudomain} \,\,  \big\langle \mu(a,s), r(a,s)\rangle - \frac{1}{\beta} \Omega \big( \mu(a,s) \big) := \Omega^{*}\big( r(a,s) \big) \label{eq:reg_rl2}
\end{align}

Note that the objective is concave, as the sum of a linear term and the concave $-\Omega$.  Since the conjugate is an involution for convex functions, we can rewrite  $ \obj_{\Omega, \beta}(r) = -(-\obj_{\Omega, \beta}(r)) = - ((-\obj_{\Omega, \beta})^*)^* $, which yields

\begin{align}
\obj_{\Omega, \beta}(r) &= \sup \limits_{\mu(a,s) \in \mudomain} - ((-\obj_{\Omega, \beta})^*)^* \nonumber \\
&\overset{(1)}{=}\sup \limits_{\mu(a,s) \in \mudomain} -\bigg( \sup \limits_{r^\prime(a,s) \in \mathcal{F}} \langle \mu(a,s), r^{\prime}(a,s) \rangle -  (-\obj_{\Omega, \beta})^*(r^{\prime}) \bigg) \nonumber \\[1.25ex]
%- \big(-\langle \mu(a,s), r(a,s) \rangle + \obj \big) \bigg) \\
&= \sup \limits_{\mu(a,s) \in \mudomain}   \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \langle \mu(a,s), -r^{\prime}(a,s) \rangle +  (-\obj_{\Omega, \beta})^*(r^{\prime})  \nonumber \\
&\overset{(2)}{=}  \sup \limits_{\mu(a,s) \in \mudomain} \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \langle \mu(a,s), r^{\prime}(a,s) \rangle +  (-\obj_{\Omega, \beta})^*(-r^{\prime})  \nonumber\\
&\overset{(3)}{=} \sup \limits_{\mu(a,s) \in \mudomain}  \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \langle \mu(a,s), r^{\prime}(a,s) \rangle +  \bigg( \sup \limits_{\mu^{\prime}} \, \langle \mu^{\prime}(a,s), -r^{\prime}(a,s) \rangle + \obj_{\Omega, \beta}(\mu^{\prime}) \bigg) \nonumber \\
&\overset{(4)}{=}  \sup \limits_{\mu(a,s) \in \mudomain}  \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \langle \mu(a,s), r^{\prime}(a,s) \rangle +  \bigg( \sup \limits_{\mu^{\prime}} \, \langle \mu^{\prime}(a,s), -r^{\prime}(a,s) \rangle + \langle \mu^{\prime}(a,s), r(a,s)\rangle - \frac{1}{\beta} \Omega(\mu^{\prime})   \bigg) \nonumber \\
&=  \sup \limits_{\mu(a,s) \in \mudomain}  \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \langle \mu(a,s), r^{\prime}(a,s) \rangle + \frac{1}{\beta}  \bigg( \sup \limits_{\mu^{\prime}}  \, \langle \mu^{\prime}(a,s), \beta \cdot \big( r(a,s) - r^{\prime}(a,s) \big)  \rangle -  \Omega(\mu^{\prime})   \bigg) \nonumber \\[1.25ex]
&\overset{(5)}{=} \sup \limits_{\mu(a,s) \in \mudomain}  \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \langle \mu(a,s), r^{\prime}(a,s) \rangle +  \frac{1}{\beta} \Omega^{*} \big( \beta \cdot ( r - r^{\prime} ) \big)   \\
&\overset{(6)}{=}  \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \sup \limits_{\mu(a,s) \in \mudomain}  \langle \mu(a,s), r^{\prime}(a,s) \rangle +  \frac{1}{\beta}  \Omega^{*} \big( \beta \cdot ( r - r^{\prime} ) \big)  \label{eq:end}
\end{align}

where $(1)$ applies the definition of the conjugate of $(-\obj_{\Omega, \beta})^*$, $(2)$ reparameterizes the optimization in terms of $r^{\prime} \rightarrow -r^{\prime}$, $(3)$ is the conjugate for $(-\obj_{\Omega, \beta})$, and $(4)$ uses the definition of the regularized RL objective for measure $\mu^{\prime}(a,s)$ with the reward $r(a,s)$.  Finally, $(5)$ recognizes the inner maximization as the conjugate function for a modified reward and $(6)$ swaps the order of $\inf$ and $\sup$ assuming the problem is feasible.

To derive the optimal reward perturbations, consider introducing $V(s)$ to enforce the flow constraints.

\begin{align}
\obj_{\Omega, \beta}(r)=   \inf \limits_{r^\prime(a,s) \in \mathcal{F}}  \sup \limits_{\mu(a,s)} \inf \limits_{V(s)} \langle \mu(a,s), r^{\prime}(a,s) + \gamma \transitionv - V(s) \rangle +  \frac{1}{\beta}  \Omega^{*} \big( \beta \cdot ( r - r^{\prime} ) \big) + (1-\gamma) \langle \nu_0(s), V(s) \rangle \label{eq:husain_lagr}
\end{align}

Now, eliminating $\mu(a,s)$ yields the condition

\begin{align}
r^{\prime}(a,s) + \gamma \transitionv - V(s) = 0 \quad \implies \quad r^{\prime}(a,s) = V(s) - \gamma \transitionv \label{eq:husain_result}
\end{align}

which matches Husain et al., 2021.

Recognizing that $r^{\prime}(a,s) = r(a,s)- \pr(a,s)$, we can rearrange <ref> to obtain $\pr(a,s) = r(a,s) + \gamma \transitionv - V(s)$ which matches the advantage function.

Finally, note that the term involving $\mu(a,s)$ vanishes when substituting the expression for $\mr(a,s)$ in <ref> into <ref>.  This yields Theorem 1 of Husain et al., 2021,

\begin{align}
\obj_{\Omega, \beta}(r)=    \inf \limits_{V(s)}  \, (1-\gamma) \blangle \nu_0(s), V(s) \brangle + \frac{1}{\beta}  \Omega^{*} \big( \beta \cdot ( r(a,s) + \gamma \transitionv - V(s) ) \big)
\end{align}

In this section, we explore further implications of our adversarial interpretation for the optimal policy $\piopt(a|s)$, including connections with the path consistency optimality conditions [Nachum et al., 2017, Chow et al., 2018] and game-theoretic indifference condition [Ortega & Lee, 2014].     We derive path consistency conditions for general $\alpha$-divergences with arbitrary reference distributions, which extend the previous work of [Nachum et al., 2017, Chow et al., 2018].

§.§ Path Consistency

Using the kkt conditions, Nachum et al., 2017 note that the condition

\begin{align}
r(a,s) + \gamma \transitionvstar - \frac{1}{\beta} \log \frac{\pi(a|s)}{\pi_0(a|s)}  =  V_*(s)
\end{align}

is fulfilled at optimality of the policy $\piopt(a,s)$ and value function $V_*(s)$.   This condition is used to provide an alternative learning objective, with the squared error $\mathbb{E}_{a, s, s\tick}\left[ \left( r(a,s) + \gamma \transitionv - \frac{1}{\beta} \log \frac{\pi(a|s)}{\pi_0(a|s)}  - V(s) \right)^2 \right] $ used as a loss for learning $\pi(a|s)$ and $V(s)$ (or simply $Q(a,s)$, Nachum et al., 2017 Sec. 5.1) using neural networks.

Similarly, Chow et al., 2018 consider a (scaled) Tsallis entropy regularizer, $\frac{1}{\beta}\Omega(\pi) = \frac{1}{\beta}\frac{1}{\alpha(\alpha-1)}(\sum_a \pi(a|s) - \sum_a \pi(a|s)^{\alpha})$.   For $\alpha = 2$, the optimal policy and value function satisfy

\begin{align}
r(a,s) + \gamma \transitionv  +  \frac{1}{\beta} \frac{1}{\alpha(\alpha-1)}  - \frac{1}{\beta} \frac{1}{\alpha-1} \pi(a|s)^{\alpha-1} &= V_*(s)  + \Lambda(s)  \label{eq:path1}
% \log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)}
\end{align}

where $\Lambda(s)$ is a Lagrange multiplier whose value is learned in Chow et al., 2018.

However, inspecting the proof of Theorem 3 in Chow et al., 2018, we see that this multiplier is obtained via the identity $\Lambda(s) := \normalizeralpha - V_*(s)$ (see

Our notation in <ref> differs from Chow et al., 2018 in that we use $\frac{1}{\beta}$ as the regularization strength (compared with their $\alpha$).  We have also written <ref> using the constant factors appearing in the $\alpha$-divergence, as we will show below.

In generalizing the path consistency equations,  we will consider the $\alpha$-divergence, with $\Omega(\pi) = \frac{1}{\alpha(\alpha-1)}((1-\alpha) \sum_a \pi_0(a|s) + \alpha \sum_a \pi(a|s) - \sum_a \pi_0(a|s)^{1-\alpha} \pi(a|s)^{\alpha})$.   Note that this includes an additional $\alpha$ factor which multiplies the $\sum_a \pi(a|s)$ term.
In particular, this scaling will change the $\frac{1}{\beta} \frac{1}{\alpha(\alpha-1)}$ constant factor in <ref> and Chow et al., 2018, to a factor of $\frac{1}{\beta} \frac{1}{\alpha-1}$ (which equals $\frac{1}{\beta}$ for $\alpha =2$).

To generalize these path consistency conditions, we consider the kkt conditions for the convex optimization problem in <ref> or <ref>, with $\alpha$-divergence regularization.[We focus on the differences in our approach, and direct the reader to Chow et al., 2018 for detailed derivations.]

The kkt condition involving the gradient of the Lagrangian <ref> with respect to $\mu(a,s)$ suggest that, at optimality,

\begin{align}
r(a,s) +  \gamma \transitionvstar -V_*(s) - \nabla \omegamu = 0 \label{eq:kkt_path}
\end{align}

Using the identity $\normalizeralpha =  V_*(s) + \frac{1}{\beta}(1-\alpha) D_{\alpha}[\pi_0:\piopt]$
in <ref>, we can recognize that $\Lambda(s) := \normalizeralpha - V_*(s) = \myconst$ at optimality.

Substituting this into
optimal_perturbations, we have $\nabla \omegamu = \propt(a,s) = \frac{1}{\beta}\log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)} - \myconst = \frac{1}{\beta}\log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)} - \Lambda(s)$. Double check nonnegativity and switch in notation to use optimal $V_*$.   Substituting into <ref> and rearranging, we can write

\begin{align}
r(a,s) +  \gamma \transitionv - \frac{1}{\beta}\log_{\alpha}\frac{\pi(a|s)}{\pi_0(a|s)} = V_*(s) + \Lambda(s)
% \frac{1}{\beta}(1-\alpha)D_{\alpha}[\pi_0:\pi] = 0
\end{align}

which is directly comparable to <ref> and Chow et al., 2018.  Finally, using $\Lambda(s) := \normalizeralpha - V_*(s)$, we obtain <ref> in the main text

\begin{align}
\hspace*{-.2cm}  r(a,s) + \gamma \transitionvstar - \frac{1}{\beta} \log_{\alpha}\frac{\pi_*(a|s)}{\pi_0(a|s)} = \normalizeralpha  \, . \label{eq:path_app}
\end{align}

§.§ Indifference Condition

In the single step setting with kl divergence regularization, Ortega & Lee, 2014 argue that the perturbed reward for the optimal policy is constant with respect to actions

\begin{align}
r(a) - \propt(a) = c \, \, \, \forall a \in \mathcal{A} \, , \label{eq:single_step_indifference}
\end{align}

when $\propt(a)$ are obtained using the optimal policy $\pi_*(a|s)$.
Ortega & Lee, 2014 highlight that this is a well-known property of Nash equilibria in game theory where, for the optimal policy and worst-case adversarial perturbations, the agent obtains equal (perturbed) reward for each action and  thus is indifferent between them.

In the sequential setting, we can consider $Q(a,s) = r(a,s) + \gamma \transitionv$ as the analogue of the single-step reward or utility function.   Using our advantage function interpretation for the optimal policy, we directly obtain an indifference condition for the sequential setting

\begin{align}
Q_*(a,s) - \propt(a,s) = V_*(s) \, \, \,
\end{align}

where $V_*(s)$ is a constant with respect to $a \in \mathcal{A}$ for a given state $s \in \mathcal{S}$.
Recall that our notation for $V_*(s)$ omits its dependence on $\beta$ and $\alpha$.
This indifference condition indeed holds for arbitrary regularization strengths $\beta$ and choices of $\alpha$-divergence, since our proof of the advantage function interpretation in advantage_pf is general.
Finally, we emphasize that the indifference condition holds only for the optimal policy with a given reward $r(a,s)$.

§.§ Confirming Optimality using Path Consistency and Indifference

In <ref> of the main text, we plotted the regularized policies and worst-case reward perturbations for various regularziation strength $\beta$ and kl divergence regularization.  We now seek to certify the optimality of each policy using the path consistency or indifference conditions in pyco_indifference.   In particular, we confirm that the following equalities hold

\begin{align}
r(a,s) + \gamma \transitionv - \frac{1}{\beta} \log \frac{\pi(a|s)}{\pi_0(a|s)}  = \normalizerexp = V_*(s) \label{eq:confirming_path}
\end{align}

which aligns with the path consistency condition in [Nachum et al., 2017].
This equality also confirms the indifference condition since the right hand side, $\normalizerexp$ or $V_*(s)$, is a constant with respect to actions.       Finally, we can recognize our advantage function interpretation $Q_*(a,s) - \propt(a,s) = V_*(s)$ in <ref>, by direct substituting $Q_*(a,s) = r(a,s) + \gamma \transitionv$  and $\propt(a,s) = \frac{1}{\beta} \log  \frac{\pi(a|s)}{\pi_0(a|s)}$ for kl divergence regularization.

In pyco_indifference, we plot $r(a,s) + \gamma \transitionv - \frac{1}{\beta} \log \frac{\pi(a|s)}{\pi_0(a|s)} = Q_*(a,s) - \propt(a,s)$ for each state-action pair to confirm that it yields a constant value.   Note that this value is different across states based on the soft value function $V_*(s)$, which also depends on the regularization strength.

In this section, we derive a constrained optimization corresponding to the Lagrangian relaxation in <ref> of the main text.  This allows us to characterize the feasible set of reward perturbations, which represents both the possible strategies of the adversary and the reward function changes to which a particular policy is robust.   Finally, we detail our approach to construct the feasible set plots in feasible_set_main and feasible_set_uniform_app-<ref>.

The proof [Our proof differs from the intuitive arguments given in the main text. Note that $\alphaconjmu(\pr)$ increasing does not imply the constraint $\alphaconjmu(\pr)\leq 0$ based on the optimization in <ref>.     The fact that the conjugate function $\alphaconjmu$ is increasing in $\pr(a,s)$, in the sense that if $\pr_1(a,s) \geq \pr_2(a,s)$ for all $a \in \mathcal{A}, s \in \mathcal{S}$ then $\alphaconjmu(\pr_1) \geq \alphaconjmu(\pr_2)$, is shown for general $f$-divergences in Husain et al., 2021 App. 7.8.1, which includes the $\alpha$-divergence as a special case.] is closely related to the result in  preserve conjugatezero that $\alphaconjmu(\propt)=0 \, \forall \alpha$ for the worst-case reward perturbations $\propt(a,s)$ associated with a particular policy $\pi(a|s)$ or occupancy measure $\mu(a,s)$.

Considering the unconstrained optimization over $\pr(a,s)$ in <ref>

\begin{align}
\hspace*{-.2cm} & \quad \max \limits_{\mu(a,s)} \min \limits_{V(s)} \min \limits_{\textcolor{\highlight}{\perturbr(a,s)}}  \big \langle {\mu(a,s)}, {r(a,s) +\gamma \transitionv - V(s) - \textcolor{\highlight}{\perturbr(a,s)}} \brangle + \alphaconjmu \big( \textcolor{\highlight}{\perturbr} \big) + (1-\gamma) \blangle \nu_0(s), V(s) \brangle \, ,  \, \label{eq:pedro_form4}
\end{align}

Informed by the result in conjugatezero, we can recognize $\frac{1}{\beta}$ as a Lagrange multiplier enforcing a constraint on the conjugate $\alphaconjmu( {\perturbr}) \leq 0$.   Note that no additional constants are required to enforce the inequality constraint.   Since $\frac{1}{\beta} > 0$ by assumption, we can see that $\alphaconjmu( {\perturbr}) > 0$ incurs a penalty in the minimization objective.   Finally, the complementary slackness condition [Boyd & Vandenberghe, 2004] suggests that, at optimality, we will have $\alphaconjmu( {\perturbr}) = 0$ as in conjugatezero.
Thus, the constrained optimization corresponding to <ref> becomes

\begin{align}
\quad \max \limits_{\mu(a,s)} \min \limits_{V(s)} \min \limits_{\textcolor{\highlight}{\perturbr(a,s)}}  \big \langle {\mu(a,s)}, &{r(a,s) +\gamma \transitionv - V(s) - \textcolor{\highlight}{\perturbr(a,s)}} \brangle  + (1-\gamma) \blangle \nu_0(s), V(s) \brangle \quad \label{eq:pedro_form_constrained} \\
&\text{subj. to } \,\, \alphaconjmu \big( \textcolor{\highlight}{\perturbr} \big) \leq 0 \nonumber \,
\end{align}

For the kl divergence, we confirm that $\klconjmu(\pr)$ takes a form which is analogous to  the conjugate derived using optimization over $\pi(a|s)$ in kl_conj_all.
To do so, we plug the policy $\pir(a|s) = \pi_0 \exp \{ \beta \cdot (\pr(a,s) - \psi(\pr, s;\beta)) \}$ that corresponds a particular reward perturbation into the conjugate optimization

\begin{align}
\klconjmu &= \blangle \mu(s) \pir(a|s), \pr(a,s) \brangle - \frac{1}{\beta} \blangle \mu(s) \pir(a|s), \cancel{\log \exp} \{ \beta \cdot (\pr(a,s) - \psi(\pr, s;\beta)) \}\brangle \\
&= \blangle \mu(s), \psi(\pr, s;\beta) \brangle = \blangle \mu(s), \frac{1}{\beta} \log \sum \limits_{a \in \mathcal{A}} \pi_0(a|s) \exp \{ \beta \cdot \pr(a,s) \} \brangle
\end{align}

We consider plotting the feasible set for the single step case in the next section, <ref>, and feasible_set_uniform_app-<ref>.   In this case, we can ignore the possibility that we have both positive and negative $\log \sum_{a} \pi_0(a|s) \exp \{ \beta \cdot \pr(a,s) \} \lessgtr 0$ in various $s \in \mathcal{S}$, but that averaging over $\mu(s)$ would still fulfill the constraint.
Simplifying the constraint $\klconjpi \leq 0$ at each $s \in \mathcal{S}$, we remove the $\frac{1}{\beta}$ constant factor and exponentiate both sides to obtain <ref>

\begin{align}
\sum\limits_{a \in \mathcal{A}} \pi_0(a|s) \expof{\beta \cdot \pr(a,s)} \leq 1 \, .
\end{align}

§.§ Plotting the $\alpha$-Divergence Feasible Set

To plot boundary of the feasible set for the kl divergence regularization in two dimensions, we can simply solve for the $\pr(a_2,s)$ which satisfies the constraints for a given $\pr(a_1,s)$

\begin{align}
\pr(a_2,s) = \frac{1}{\beta} \log \frac{1}{\pi_0(a_2|s)}(1- \pi_0(a_1|s) \expof{\beta \cdot \pr(a_1,s)}) \, .
\end{align}

The interior of the feasible set contains $\pr(a_1,s)$ and $\pr(a_2,s)$ that are greater than or equal to these values.

However,  we cannot analytically solve for the feasible set boundary for general $\alpha$-divergences, since the conjugate function $\alphaconjmu$ in <ref> is defined in self-consistent fashion due to the normalization constant of $\pir(a,s)$.
Instead, we perform exhaustive search over a range of $\pr(a_1, s)$ and $\pr(a_2,s)$ values.  For each pair of candidate reward perturbations, we use cvx-py [Diamond & Boyd, 2016] to solve the optimization

\begin{align}
\alphaconjn(\pr) = \max \limits_{\pi(a|s)} \blangle \pi(a|s), \pr(a,s) \brangle - \omegapi \,  \label{eq:feasible_set_opt}
\end{align}

which yields the policy $\pir(a|s)$ corresponding to this choice of $\pr(a,s)$.   We also obtain a value for the conjugate function  $\alphaconjn(\pr)$ by evaluating  <ref> at this $\pir(a|s)$.
We can terminate our exhaustive search and record the boundary of the feasible set when we find that $\alphaconjn(\pr) = 0$ within appropriate precision.
Note that, in the single step case, we do not have to consider the marginal state visitation distribution $\mu(s)$ for either kl or $\alpha$-divergence regularization.

In this section, we provide proofs and discussion to support our observations in <ref> on the convenience of divergence regularization over entropy regularization.

§.§ Non-positivity of $\propt(a,s)$ for Entropy Regularization

We first derive the worst-case reward perturbations for entropy regularization, before analyzing the sign of these reward perturbations for various values of $\alpha$ in nonpositive.
In particular, we have $\propt(a,s) \leq 0$ for entropy regularization with $0 < \alpha \leq 1$, which includes Shannon entropy regularization at $\alpha = 1$.  This implies that the modified reward $\mropt(a,s) \geq r(a,s)$.

The worst-case reward perturbations for Tsallis entropy regularization correspond to

\begin{align}
\propt(a,s) = \frac{1}{\beta} \log_{\alpha} \pi(a|s) + \frac{1}{\beta} \frac{1}{\alpha} (1 - \sum \limits_{a \in \mathcal{A}} \pi(a|s)^{\alpha}) \,. \label{eq:ptwise_perturb_ent_app}
\end{align}

with limiting behavior of $\propt(a,s) = \frac{1}{\beta} \log \pi(a|s)$ for Shannon entropy regularization as $\alpha \rightarrow 1$.

We can write the Tsallis entropy using an additional constant $k$, with $k=\frac{1}{\alpha}$ mirroring the $\alpha$-divergence

\begin{align}
H^T_{\alpha}[\pi] = \frac{k}{\alpha-1} \bigg( \sum \limits_{a \in \mathcal{A}} \pi(a|s) - \sum \limits_{a \in \mathcal{A}} \pi(a|s)^{\alpha} \bigg) \, .  \label{eq:tsallis2}
%  =   - \sum \limits_{a \in \mathcal{A}} \pi(a|s) \cdot \log_{\alpha} \big(\pi(a|s) \big)
\end{align}

Note that we use negative Tsallis entropy for regularization since the entropy is concave.   Thus, the worst case reward perturbations correspond to the condition $\propt(a,s)= \nabla \frac{1}{\beta}\Omega^{(H_{\alpha})}_{\pi_0}(\mu) = - \nabla \mathbb{E}_{\mu(s)}\big[ H^T_{\alpha}[\pi]\big]$. Differentiating with respect to $\mu(a,s)$???  Following similar steps as in perturb_pf_sub <ref>-(<ref>), we obtain

\begin{align}
\propt(a,s) = k \cdot \frac{1}{\beta} \cdot \alpha \cdot \log_{\alpha} \pi(a|s) + k \cdot \frac{1}{\beta} (\alpha-1) H^T_{\alpha}[\pi]
\end{align}

For $k = \frac{1}{\alpha}$ and $(\alpha-1)H^T_{\alpha}[\pi] = ( \sum_a \pi(a|s) - \sum_a \pi(a|s)^{\alpha} )$, we obtain <ref>.

For $0 < \alpha \leq 1$ and $\beta > 0$,
the worst-case reward perturbations with Tsallis entropy regularization from ent_reg are non-positive, with $\propt(a,s) \leq 0$.  This implies that the entropy-regularized policy is robust to only pointwise reward increases for these values of $\alpha$.

We first show that $\log_{\alpha} \pi(a|s) \leq 0 $ for $0 < \pi(a|s) \leq 1$ and any $\alpha$.  Note that we may write (Naudts, 2011 Ch. 7)

\begin{align}
\log_{\alpha} \pi(a|s) = \int \limits_1^{\pi(a|s)} u^{\alpha-2} du = \frac{1}{\alpha-1}u^{\alpha-1}\bigg|^{\pi(a|s)}_1 = \frac{1}{\alpha-1}(\pi(a|s)^{\alpha-1}-1)
\end{align}

Since $u^{\alpha-2}$ is a non-negative function for $0 \leq \pi(a|s) \leq 1$, then $\int_1^{\pi(a|s)}  u^{\alpha-2} du \leq 0$.

To analyze the second term, consider $0 < \alpha \leq 1$.  We know that $\pi(a|s)^{\alpha} \geq \pi(a|s)$ for any $0 < \pi(a|s)\leq 1$, so that $\sum_a \pi(a|s)^{\alpha} \geq \sum_a \pi(a|s) = 1$.  Thus, we have $( \sum_a \pi(a|s) - \sum_a \pi(a|s)^{\alpha} ) \leq 0$, from which $\alpha >0$ implies $ \frac{1}{\alpha}( \sum_a \pi(a|s) - \sum_a \pi(a|s)^{\alpha}) \leq 0$.  Since both terms are non-positive, we have shown that $\propt(a,s) \leq 0$ for $0 < \alpha \leq 1$ as desired.

However, for  $\alpha > 1$ or $\alpha < 0$, we cannot guarantee the reward perturbations are non-positive.   Writing the second term in <ref> as $\frac{\alpha-1}{\alpha} H^T_{\alpha}[\pi]$, we first observe that that $H^T_{\alpha}[\pi] \geq 0$.   Now, $\alpha > 1$ or $\alpha < 0$ implies that  $\frac{\alpha-1}{\alpha} > 0$, so that the second term is non-negative.

§.§ Bounding the Conjugate Function

We follow similar derivations as Lee et al., 2019 to bound the value function for general $\alpha$-divergence regularization instead of entropy regularization.
To upper bound the conjugate function, we can consider the optimum over each term separately

\begin{align}
\alphaconjn(\dualvar) &= \max \limits_{\pi} \blangle \pi(a|s), \dualvar(a,s) \brangle - \frac{1}{\beta} D_{\alpha}[\pi_0(a|s) || \pi(a|s)] \\
&\leq   \max \limits_{\pi} \blangle \pi(a|s), \dualvar(a,s) \brangle - \min \limits_{\pi} \frac{1}{\beta} D_{\alpha}[\pi_0(a|s) || \pi(a|s)] \\
&=  \max \limits_{\pi} \blangle \pi(a|s), \dualvar(a,s) \brangle  - 0 \\
%&= \max \limits_{a} \dualvar(a,s)
&= \dualvar(a_{\max},s) \,.
\end{align}

where we let $a_{\max}:= \argmax \limits_{a} \dualvar(a,s)$.

We can also lower bound the conjugate function in terms of $\max_a \dualvar(a,s)$.  Noting that any policy $\pi(a|s)$ provides a lower bound on the value of the maximization objective, we can consider $\pi_{\max}(a|s) = \delta( a = a_{\max})$.  For the $\alpha$-divergence,

\begin{align*}
\alphaconjn(\dualvar) &= \max \limits_{\pi} \blangle \pi(a|s), \dualvar(a,s) \brangle - \frac{1}{\beta} D_{\alpha}[\pi_0(a|s) || \pi(a|s)] \\
&\geq    \blangle \pi_{\max}(a|s), \dualvar(a,s) \brangle - \frac{1}{\beta} D_{\alpha}[\pi_0(a|s) || \pi_{\max}(a|s)(a|s)] \\
&=  \dualvar(a_{\max},s) - \frac{1}{\beta} (1 -  \frac{1}{\alpha(1-\alpha)} \pi_0(a_{\max}|s)^{1-\alpha} 1^{\alpha})\\
&= \dualvar(a_{\max},s) + \frac{1}{\beta} \frac{1}{\alpha} \log_{\alpha} \pi_0(a_{\max}|s)
\end{align*}

Similarly, for $D_{KL}[\pi||\pi_0]$ at $\alpha = 1$, we obtain

\begin{align*}
\frac{1}{\beta}\Omega^{*}_{\pi_0,\beta}(\dualvar) \geq \dualvar(a_{\max},s)- \big(-\frac{1}{\beta} \log \pi_0(a_{\max}|s) \big)
\end{align*}

Combining these bounds, we can write

\begin{align}
\dualvar(a_{\max},s) + \frac{1}{\beta}\frac{1}{\alpha} \log_{\alpha} \pi_0(a_{\max}|s) \, \, \leq   \, \, \alphaconjn(\dualvar) \, \, \leq   \, \,  \dualvar(a_{\max},s)
\end{align}

Finally, we can analyze the conjugate as a function of $\beta$.   As $\beta \rightarrow 0$ and $1/\beta \rightarrow \infty$, the divergence regularization forces $\pi(a|s) = \pi_0(a|s)$ for any $\alpha$.   As $\beta \rightarrow \infty$ and $1/\beta \rightarrow 0$, the unregularized objective yields a deterministic optimal policy with $\pi(a|s) = \max_a r(a,s)$.  For negative values of $\beta$, the optimization becomes a minimization, with the optimal solution approaching a deterministic policy with $\pi(a|s) = \min_a r(a,s)$ as $\beta \rightarrow -\infty$,  $1/\beta \rightarrow 0$.

Fig. 2 or Fig. 7 (left) of
Eysenbach & Levine, 2021
Constraint: $\sum \limits_{a \in \mathcal{A}} \exp\{\beta  \Delta r(a)\} \leq 1$

Shifted $r^{\prime}= r - \Delta r \textcolor{blue}{- \ln(2)}$
Constraint: $\sum\limits_{a \in \mathcal{A}} \exp\{\beta  \Delta r(a)\} \leq 1$

With reference $\pi_0(a|s) = u(a) = \frac{1}{2}$
Constraint: $\small \sum\limits_{a \in \mathcal{A}} \textcolor{blue}{\pi_0(a)} \exp\{\beta \Delta r(a)\} \leq 1$
Analyzing the feasible set for entropy regularization (a) versus divergence regularization (b,c).

§.§ Feasible Set for Entropy Regularization

In their App. A5, Eysenbach & Levine, 2021 make the surprising conclusion that policies with higher entropy regularization are less robust, as they lead to smaller feasible or robust sets than for lower regularization.   This plot is depicted in the left plots of their main text Fig. 2 or appendix Fig. 7, as recreated below in <ref>.
In this section, we compare feasible sets derived from entropy regularization ( <ref>) with those derived from the kl divergence, as in <ref> and <ref>-<ref>, and argue that entropy regularization should be analyzed as a special case of the kl divergence to avoid misleading conclusoins.

To translate from Shannon entropy to kl divergence, with $ - \frac{1}{\beta} D_{KL}[\pi:\pi_0] = \frac{1}{\beta} H(\pi) - \frac{1}{\beta} \log |\mathcal{A}|$,  we include the additional additive constant of $\log |\mathcal{A}|$ corresponding to the entropy of the uniform distribution.  Note that this constant is multiplied by the regularization strength $1/\beta$ in the objective.
This highlights the delicate dependence on the feasible set on the exact form of the objective function, as this constant shifts the robust reward set in two dimensions by $(r^{\prime}(a_1), r^{\prime}(a_2)) \leftarrow (r^{\prime}(a_1) - \frac{1}{\beta}\log 2, r^{\prime}(a_2)-\frac{1}{\beta}\log 2)$.   In this case, the feasible set includes the original reward function, and we can see that policies with higher regularization strength are more robust with larger feasible sets.

An alternative approach to correct the constraint set is to include the uniform reference distribution as in feasible and <ref>, so that we calculate $\sum_a \pi_0(a|s) \expof{\beta \cdot \pr} \leq 1$.   Similarly, the constraint in Eysenbach & Levine, 2021 can be modified to have $\sum_a \expof{\beta \cdot \pr} \leq |\mathcal{A}|$ in the case of entropy regularization.

Our modifications to the feasible set constraints correct this apparently contradictory conclusion.   We do not give detailed consideration to the other question briefly posed in Eysenbach & Levine, 2021 App. A5, of `if a reward function $r^{\prime}$
is included in a robust set, what other reward functions are included in that robust set?', whose solution is visualized in Eysenbach & Levine, 2021 Fig. 7 (right panel) without detailed derivations.   However,  this plot matches our <ref> and <ref>.   This suggests that
the constraint arising from kl divergence regularization with strength $\beta$,
$\sum \pi_0(a|s) \expof{\beta \cdot \pr(a,s)} = 1$,
is sufficient to define both the robust set for a given reward function and to characterize the other reward functions to which an optimal policy is robust.
The key observation is that the original reward function is included in the robust set when explicitly including the reference distribution $\pi_0(a|s)$ as in divergence regularization.

In feasible_set_uniform_app and feasible_set_nonuniform_app, we provide additional feasible set plots for the $\alpha$-divergence with $\alpha \in \{-1, 1, 2, 3\}$ and $\beta \in \{ 0.1, 1.0, 10 \}$ with $r(a,s) = [1.1, 0.8]$.   As in feasible_set_main, we show the feasible set corresponding to the single-step optimal policy for $Q(a,s) = r(a,s)$ for various regularization schemes.   Since each policy is optimal, we can confirm the indifference condition, with $\mropt(a,s) =[V_*(s), V_*(s)]$ constant across actions and equal to the soft value function, certainty equivalent, or conjugate function $\alphaconjpi(Q)$.   As in feasible_set_main, we can obtain the ratio of action probabilities by taking the slope of the tangent line to the feasible set boundary at $\mropt(a,s)$.

For $\alpha=-1$ and low regularization strength ($\beta=10$), we observe a wider feasible set boundary.   For $\alpha=2$ and $\alpha=3$, the boundary appears slightly more restricted.  However, as in value_agg_main, we do not observe notable differences in the robust sets at lower regularization strengths based on the choice of $\alpha$-divergence.

§ CONSISTENCY IMPLIES OPTIMALITY

Consistency equation implies

\begin{align}
r(a,s) + \gamma \transitionv - V(s) = \frac{1}{\beta} \log_{\alpha}\frac{\piopt(a|s)}{\pi_0(a|s)} + \psi_{\prpi}
\end{align}

Recall that $\frac{1}{\beta} \nabla_{\mu} \Omega_{\pi_0}(\mu) = \log_{\alpha}\frac{\piopt(a|s)}{\pi_0(a|s)} + \psi_{\prpi}$.
Thus, we can rewrite the consistency condition as

\begin{align}
r(a,s) + \gamma \transitionv  = V(s) + \frac{1}{\beta} \nabla_{\mu} \Omega_{\pi_0}(\mu)
\end{align}

This condition is identical to showing that that the maximization in the Bellman optimality operator is tight,

\begin{align}
\mathcal{T}V(s) &= \max \limits_{\pi(a|s) \in \Delta^{|\mathcal{A}|}} \blangle \pi(a|s), r(a,s) + \gamma \transitionv \brangle - \omegapi \\
&= \blangle \piopt(a|s), r(a,s) + \gamma \transitionv \brangle - \omegapiopt
\end{align}