From 70ef40aca19c850dd6b25499acc9ff324bd4895b Mon Sep 17 00:00:00 2001 From: Yi Su Date: Fri, 6 Aug 2021 06:58:21 +0800 Subject: [PATCH 01/18] implement GAIL policy --- README.md | 1 + docs/api/tianshou.policy.rst | 5 + docs/index.rst | 1 + examples/atari/README.md | 16 +++ examples/atari/atari_gail.py | 172 ++++++++++++++++++++++++++++++ tianshou/policy/__init__.py | 2 + tianshou/policy/imitation/gail.py | 117 ++++++++++++++++++++ tianshou/utils/net/common.py | 2 + tianshou/utils/net/discrete.py | 36 ++++++- 9 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 examples/atari/atari_gail.py create mode 100644 tianshou/policy/imitation/gail.py diff --git a/README.md b/README.md index 1144a0cab..11f7b51b9 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ - [Discrete Batch-Constrained deep Q-Learning (BCQ-Discrete)](https://arxiv.org/pdf/1910.01708.pdf) - [Discrete Conservative Q-Learning (CQL-Discrete)](https://arxiv.org/pdf/2006.04779.pdf) - [Discrete Critic Regularized Regression (CRR-Discrete)](https://arxiv.org/pdf/2006.15134.pdf) +- [Generative Adversarial Imitation Learning (GAIL)](https://arxiv.org/pdf/1606.03476.pdf) - [Prioritized Experience Replay (PER)](https://arxiv.org/pdf/1511.05952.pdf) - [Generalized Advantage Estimator (GAE)](https://arxiv.org/pdf/1506.02438.pdf) - [Posterior Sampling Reinforcement Learning (PSRL)](https://www.ece.uvic.ca/~bctill/papers/learning/Strens_2000.pdf) diff --git a/docs/api/tianshou.policy.rst b/docs/api/tianshou.policy.rst index c3063665c..a0d9bed99 100644 --- a/docs/api/tianshou.policy.rst +++ b/docs/api/tianshou.policy.rst @@ -134,6 +134,11 @@ Imitation :undoc-members: :show-inheritance: +.. autoclass:: tianshou.policy.GAILPolicy + :members: + :undoc-members: + :show-inheritance: + Model-based ----------- diff --git a/docs/index.rst b/docs/index.rst index 15ac74037..b131ff402 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Welcome to Tianshou! * :class:`~tianshou.policy.DiscreteBCQPolicy` `Discrete Batch-Constrained deep Q-Learning `_ * :class:`~tianshou.policy.DiscreteCQLPolicy` `Discrete Conservative Q-Learning `_ * :class:`~tianshou.policy.DiscreteCRRPolicy` `Critic Regularized Regression `_ +* :class:`~tianshou.policy.GAILPolicy` `Generative Adversarial Imitation Learning `_ * :class:`~tianshou.policy.PSRLPolicy` `Posterior Sampling Reinforcement Learning `_ * :class:`~tianshou.policy.ICMPolicy` `Intrinsic Curiosity Module `_ * :class:`~tianshou.data.PrioritizedReplayBuffer` `Prioritized Experience Replay `_ diff --git a/examples/atari/README.md b/examples/atari/README.md index 561255b20..820d2f57a 100644 --- a/examples/atari/README.md +++ b/examples/atari/README.md @@ -121,3 +121,19 @@ One epoch here is equal to 100,000 env step, 100 epochs stand for 10M. | MsPacmanNoFrameskip-v4 | 1930 | ![](results/ppo/MsPacman_rew.png) | `python3 atari_ppo.py --task "MsPacmanNoFrameskip-v4"` | | SeaquestNoFrameskip-v4 | 904 | ![](results/ppo/Seaquest_rew.png) | `python3 atari_ppo.py --task "SeaquestNoFrameskip-v4" --lr 2.5e-5` | | SpaceInvadersNoFrameskip-v4 | 843 | ![](results/ppo/SpaceInvaders_rew.png) | `python3 atari_ppo.py --task "SpaceInvadersNoFrameskip-v4"` | +Note that CRR itself does not work well in Atari tasks but adding CQL loss/regularizer helps. + +# GAIL + +To running GAIL algorithm on Atari, you need to do the following things: + +- Train an expert, by using the command listed in the above QRDQN section; +- Generate buffer with noise: `python3 atari_qrdqn.py --task {your_task} --watch --resume-path log/{your_task}/qrdqn/policy.pth --eps-test 0.2 --buffer-size 1000000 --save-buffer-name expert.hdf5` (note that 1M Atari buffer cannot be saved as `.pkl` format because it is too large and will cause error); +- Train CQL: `python3 atari_cql.py --task {your_task} --load-buffer-name expert.hdf5`. + +We test our CQL implementation on two example tasks (different from author's version, we use v4 instead of v0; one epoch means 10k gradient step): + +| Task | Online QRDQN | Behavioral | GAIL | parameters | +| ---------------------- | ---------- | ---------- | --------------------------------- | ------------------------------------------------------------ | +| PongNoFrameskip-v4 | 20.5 | 6.8 | 19.5 (epoch 20) | `python3 atari_gail.py --task "PongNoFrameskip-v4" --load-buffer-name log/PongNoFrameskip-v4/qrdqn/expert.hdf5 --epoch 20` | +| BreakoutNoFrameskip-v4 | 394.3 | 46.9 | 248.3 (epoch 12) | `python3 atari_gail.py --task "BreakoutNoFrameskip-v4" --load-buffer-name log/BreakoutNoFrameskip-v4/qrdqn/expert.hdf5 --epoch 12` | diff --git a/examples/atari/atari_gail.py b/examples/atari/atari_gail.py new file mode 100644 index 000000000..ee686bf0b --- /dev/null +++ b/examples/atari/atari_gail.py @@ -0,0 +1,172 @@ +import os +import torch +import pickle +import pprint +import datetime +import argparse +import numpy as np +from torch.utils.tensorboard import SummaryWriter +from torch.distributions import Categorical + +from tianshou.utils import TensorboardLogger +from tianshou.env import SubprocVectorEnv +from tianshou.trainer import onpolicy_trainer +from tianshou.utils.net.discrete import Actor, Critic, Discriminator +from tianshou.policy import GAILPolicy +from tianshou.data import Collector, VectorReplayBuffer + +from atari_network import DQN +from atari_wrapper import wrap_deepmind + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--task", type=str, default="PongNoFrameskip-v4") + parser.add_argument("--seed", type=int, default=1626) + parser.add_argument('--buffer-size', type=int, default=10000) + parser.add_argument("--lr", type=float, default=0.00005) + parser.add_argument("--gamma", type=float, default=0.99) + parser.add_argument("--disc-lr", type=float, default=0.0001) + parser.add_argument("--disc-repeat", type=int, default=5) + parser.add_argument("--disc-hidden-sizes", type=int, nargs="*", default=[128]) + parser.add_argument("--epoch", type=int, default=100) + parser.add_argument('--step-per-epoch', type=int, default=100000) + parser.add_argument('--step-per-collect', type=int, default=500) + parser.add_argument('--repeat-per-collect', type=int, default=1) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument('--hidden-sizes', type=int, nargs='*', default=[512]) + parser.add_argument('--training-num', type=int, default=10) + parser.add_argument("--test-num", type=int, default=10) + parser.add_argument('--frames-stack', type=int, default=4) + parser.add_argument("--logdir", type=str, default="log") + parser.add_argument("--render", type=float, default=0.) + parser.add_argument("--resume-path", type=str, default=None) + parser.add_argument("--watch", default=False, action="store_true", + help="watch the play of pre-trained policy only") + parser.add_argument("--log-interval", type=int, default=100) + parser.add_argument( + "--load-buffer-name", type=str, + default="./expert_DQN_PongNoFrameskip-v4.hdf5") + parser.add_argument( + "--device", type=str, + default="cuda" if torch.cuda.is_available() else "cpu") + args = parser.parse_known_args()[0] + return args + + +def make_atari_env(args): + return wrap_deepmind(args.task, frame_stack=args.frames_stack) + + +def make_atari_env_watch(args): + return wrap_deepmind(args.task, frame_stack=args.frames_stack, + episode_life=False, clip_rewards=False) + + +def test_gail(args=get_args()): + # envs + env = make_atari_env(args) + args.state_shape = env.observation_space.shape or env.observation_space.n + args.action_shape = env.action_space.shape or env.action_space.n + # should be N_FRAMES x H x W + print("Observations shape:", args.state_shape) + print("Actions shape:", args.action_shape) + # make environments + train_envs = SubprocVectorEnv([lambda: make_atari_env(args) + for _ in range(args.training_num)]) + test_envs = SubprocVectorEnv([lambda: make_atari_env_watch(args) + for _ in range(args.test_num)]) + # seed + np.random.seed(args.seed) + torch.manual_seed(args.seed) + test_envs.seed(args.seed) + # model + feature_net = DQN(*args.state_shape, args.action_shape, + device=args.device, features_only=True).to(args.device) + actor = Actor(feature_net, args.action_shape, device=args.device, + hidden_sizes=args.hidden_sizes, softmax_output=False).to(args.device) + critic = Critic(feature_net, hidden_sizes=args.hidden_sizes, + device=args.device).to(args.device) + optim = torch.optim.Adam(list(actor.parameters()) + list(critic.parameters()), + lr=args.lr) + disc = Discriminator(feature_net, args.action_shape, hidden_sizes=args.disc_hidden_sizes, + device=args.device).to(args.device) + disc_optim = torch.optim.Adam(disc.parameters(), lr=args.disc_lr) + # buffer + assert os.path.exists(args.load_buffer_name), \ + "Please run atari_qrdqn.py first to get expert's data buffer." + if args.load_buffer_name.endswith('.pkl'): + expert_buffer = pickle.load(open(args.load_buffer_name, "rb")) + elif args.load_buffer_name.endswith('.hdf5'): + expert_buffer = VectorReplayBuffer.load_hdf5(args.load_buffer_name) + else: + print(f"Unknown buffer format: {args.load_buffer_name}") + exit(0) + # define policy + + def dist(logits): + return Categorical(logits=logits) + + policy = GAILPolicy( + actor, critic, optim, dist, expert_buffer, disc, + disc_optim, disc_repeat=args.disc_repeat, + action_space=env.action_space, reward_normalization=True + ).to(args.device) + print("action_type=", policy.action_type, "rew_norm=", policy._rew_norm) + # load a previous policy + if args.resume_path: + policy.load_state_dict(torch.load( + args.resume_path, map_location=args.device)) + print("Loaded agent from: ", args.resume_path) + # buffer + buffer = VectorReplayBuffer( + args.buffer_size, buffer_num=args.training_num, ignore_obs_next=True, + save_only_last_obs=True, stack_num=args.frames_stack) + + # collector + train_collector = Collector(policy, train_envs, buffer, exploration_noise=True) + test_collector = Collector(policy, test_envs, exploration_noise=True) + + # log + log_path = os.path.join( + args.logdir, args.task, 'gail', + f'seed_{args.seed}_{datetime.datetime.now().strftime("%m%d-%H%M%S")}') + writer = SummaryWriter(log_path) + writer.add_text("args", str(args)) + logger = TensorboardLogger(writer) + + def save_fn(policy): + torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth')) + + def stop_fn(mean_rewards): + return False + + # watch agent's performance + def watch(): + print("Setup test envs ...") + policy.eval() + test_envs.seed(args.seed) + print("Testing agent ...") + test_collector.reset() + result = test_collector.collect(n_episode=args.test_num, + render=args.render) + pprint.pprint(result) + rew = result["rews"].mean() + print(f'Mean reward (over {result["n/ep"]} episodes): {rew}') + + if args.watch: + watch() + exit(0) + + result = onpolicy_trainer( + policy, train_collector, test_collector, args.epoch, args.step_per_epoch, + args.repeat_per_collect, args.test_num, args.batch_size, + step_per_collect=args.step_per_collect, + stop_fn=stop_fn, save_fn=save_fn, logger=logger) + + pprint.pprint(result) + watch() + + +if __name__ == "__main__": + test_gail(get_args()) diff --git a/tianshou/policy/__init__.py b/tianshou/policy/__init__.py index ced11aff5..dae9da638 100644 --- a/tianshou/policy/__init__.py +++ b/tianshou/policy/__init__.py @@ -24,6 +24,7 @@ from tianshou.policy.imitation.discrete_bcq import DiscreteBCQPolicy from tianshou.policy.imitation.discrete_cql import DiscreteCQLPolicy from tianshou.policy.imitation.discrete_crr import DiscreteCRRPolicy +from tianshou.policy.imitation.gail import GAILPolicy from tianshou.policy.modelbased.psrl import PSRLPolicy from tianshou.policy.modelbased.icm import ICMPolicy from tianshou.policy.multiagent.mapolicy import MultiAgentPolicyManager @@ -52,6 +53,7 @@ "DiscreteBCQPolicy", "DiscreteCQLPolicy", "DiscreteCRRPolicy", + "GAILPolicy", "PSRLPolicy", "ICMPolicy", "MultiAgentPolicyManager", diff --git a/tianshou/policy/imitation/gail.py b/tianshou/policy/imitation/gail.py new file mode 100644 index 000000000..e663003c6 --- /dev/null +++ b/tianshou/policy/imitation/gail.py @@ -0,0 +1,117 @@ +import torch +import torch.nn.functional as F +import numpy as np +from typing import Any, Dict, List, Optional, Type + +from tianshou.data import Batch, ReplayBuffer, to_torch +from tianshou.policy import PPOPolicy + + +class GAILPolicy(PPOPolicy): + """Implementation of Generative Adversarial Imitation Learning. arXiv:1606.03476. + + :param torch.nn.Module actor: the actor network following the rules in + :class:`~tianshou.policy.BasePolicy`. (s -> logits) + :param torch.nn.Module critic: the critic network. (s -> V(s)) + :param torch.optim.Optimizer optim: the optimizer for actor and critic network. + :param dist_fn: distribution class for computing the action. + :type dist_fn: Type[torch.distributions.Distribution] + :param ReplayBuffer expert_buffer: the replay buffer contains expert experience. + :param Discriminator disc: the discriminator network. + :param torch.optim.Optimizer disc_optim: the optimizer for the discriminator + network. + :param int disc_repeat: the number of discriminator grad steps per model grad + step. Default to 5. + :param float discount_factor: in [0, 1]. Default to 0.99. + :param float eps_clip: :math:`\epsilon` in :math:`L_{CLIP}` in the original + paper. Default to 0.2. + :param float dual_clip: a parameter c mentioned in arXiv:1912.09729 Equ. 5, + where c > 1 is a constant indicating the lower bound. + Default to 5.0 (set None if you do not want to use it). + :param bool value_clip: a parameter mentioned in arXiv:1811.02553 Sec. 4.1. + Default to True. + :param bool advantage_normalization: whether to do per mini-batch advantage + normalization. Default to True. + :param bool recompute_advantage: whether to recompute advantage every update + repeat according to https://arxiv.org/pdf/2006.05990.pdf Sec. 3.5. + Default to False. + :param float vf_coef: weight for value loss. Default to 0.5. + :param float ent_coef: weight for entropy loss. Default to 0.01. + :param float max_grad_norm: clipping gradients in back propagation. Default to + None. + :param float gae_lambda: in [0, 1], param for Generalized Advantage Estimation. + Default to 0.95. + :param bool reward_normalization: normalize estimated values to have std close + to 1, also normalize the advantage to Normal(0, 1). Default to False. + :param int max_batchsize: the maximum size of the batch when computing GAE, + depends on the size of available memory and the memory cost of the model; + should be as large as possible within the memory constraint. Default to 256. + :param bool action_scaling: whether to map actions from range [-1, 1] to range + [action_spaces.low, action_spaces.high]. Default to True. + :param str action_bound_method: method to bound action to range [-1, 1], can be + either "clip" (for simply clipping the action), "tanh" (for applying tanh + squashing) for now, or empty string for no bounding. Default to "clip". + :param Optional[gym.Space] action_space: env's action space, mandatory if you want + to use option "action_scaling" or "action_bound_method". Default to None. + :param lr_scheduler: a learning rate scheduler that adjusts the learning rate in + optimizer in each policy.update(). Default to None (no lr_scheduler). + :param bool deterministic_eval: whether to use deterministic action instead of + stochastic action sampled by the policy. Default to False. + + .. seealso:: + + Please refer to :class:`~tianshou.policy.PPOPolicy` for more detailed + explanation. + """ + + def __init__( + self, + actor: torch.nn.Module, + critic: torch.nn.Module, + optim: torch.optim.Optimizer, + dist_fn: Type[torch.distributions.Distribution], + expert_buffer: ReplayBuffer, + disc: torch.nn.Module, + disc_optim: torch.optim.Optimizer, + disc_repeat: int = 5, + eps_clip: float = 0.2, + dual_clip: Optional[float] = None, + value_clip: bool = False, + advantage_normalization: bool = True, + recompute_advantage: bool = False, + **kwargs: Any, + ) -> None: + super().__init__( + actor, critic, optim, dist_fn, eps_clip, dual_clip, value_clip, + advantage_normalization, recompute_advantage, **kwargs + ) + self.disc = disc + self.disc_optim = disc_optim + self.disc_repeat = disc_repeat + self.expert_buffer = expert_buffer + self.action_dim = actor.output_dim + + def learn( # type: ignore + self, batch: Batch, batch_size: int, repeat: int, **kwargs: Any + ) -> Dict[str, List[float]]: + # update discriminator + losses = [] + for _ in range(self.disc_repeat): + for b in batch.split(batch_size, merge_last=True): + logits_pi = self.disc(b.obs, b.act) + exp_b = to_torch(self.expert_buffer.sample(batch_size)[0], device=b.act.device) + logits_exp = self.disc(exp_b.obs, exp_b.act) + loss_pi = -F.logsigmoid(-logits_pi).mean() + loss_exp = -F.logsigmoid(logits_exp).mean() + loss_disc = loss_pi + loss_exp + self.disc_optim.zero_grad() + loss_disc.backward() + self.disc_optim.step() + losses.append(loss_disc.item()) + # update reward + with torch.no_grad(): + batch.rew = -F.logsigmoid(-self.disc(batch.obs, batch.act)).clamp_(self._eps, None) + # update policy + res = super().learn(batch, batch_size, repeat, **kwargs) + res["loss/disc"] = np.mean(losses) + return res diff --git a/tianshou/utils/net/common.py b/tianshou/utils/net/common.py index 9d03b6120..d7f80b69e 100644 --- a/tianshou/utils/net/common.py +++ b/tianshou/utils/net/common.py @@ -3,6 +3,8 @@ import numpy as np import torch from torch import nn +import torch.nn.functional as F +from typing import Any, Dict, List, Type, Tuple, Union, Optional, Sequence ModuleType = Type[nn.Module] diff --git a/tianshou/utils/net/discrete.py b/tianshou/utils/net/discrete.py index a81e1c645..bf9121c25 100644 --- a/tianshou/utils/net/discrete.py +++ b/tianshou/utils/net/discrete.py @@ -407,7 +407,7 @@ class IntrinsicCuriosityModule(nn.Module): def __init__( self, - feature_net: nn.Module, + feature_net: nn.Module, feature_dim: int, action_dim: int, hidden_sizes: Sequence[int] = (), @@ -447,3 +447,37 @@ def forward( mse_loss = 0.5 * F.mse_loss(phi2_hat, phi2, reduction="none").sum(1) act_hat = self.inverse_model(torch.cat([phi1, phi2], dim=1)) return mse_loss, act_hat + + +class Discriminator(nn.Module): + """Discriminator network used in GAIL policy. + + .. note:: + + Adapted from https://github.com/ku2482/gail-airl-ppo.pytorch/blob/master + /gail_airl_ppo/network/disc.py . + """ + + def __init__( + self, + preprocess_net: nn.Module, + action_shape: Union[int, Sequence[int]], + hidden_sizes: Sequence[int] = (), + preprocess_net_output_dim: Optional[int] = None, + device: Union[str, int, torch.device] = "cpu", + ) -> None: + super().__init__() + self.preprocess = preprocess_net + self.device = device + self.output_dim = 1 + state_dim = getattr(preprocess_net, "output_dim", + preprocess_net_output_dim) + action_dim = int(np.prod(action_shape)) + self.action_dim = action_dim + self.net = MLP(state_dim, action_dim, hidden_sizes, device=self.device) + + def forward( + self, obs: torch.Tensor, act: torch.Tensor, **kwargs: Any + ) -> torch.Tensor: + s, _ = self.preprocess(obs, state=kwargs.get("state", None)) + return self.net(s).gather(-1, act.long().view(-1, 1)) From 00614ec5860f3a46c2d5003cdc5258158aed0e37 Mon Sep 17 00:00:00 2001 From: Yi Su Date: Mon, 21 Feb 2022 10:32:43 +0800 Subject: [PATCH 02/18] fix code format --- examples/atari/atari_gail.py | 126 ++++++++++++++++++++---------- tianshou/policy/imitation/gail.py | 14 ++-- 2 files changed, 92 insertions(+), 48 deletions(-) diff --git a/examples/atari/atari_gail.py b/examples/atari/atari_gail.py index ee686bf0b..40d2d0a9b 100644 --- a/examples/atari/atari_gail.py +++ b/examples/atari/atari_gail.py @@ -1,22 +1,22 @@ +import argparse +import datetime import os -import torch import pickle import pprint -import datetime -import argparse + import numpy as np -from torch.utils.tensorboard import SummaryWriter +import torch +from atari_network import DQN +from atari_wrapper import wrap_deepmind from torch.distributions import Categorical +from torch.utils.tensorboard import SummaryWriter -from tianshou.utils import TensorboardLogger +from tianshou.data import Collector, VectorReplayBuffer from tianshou.env import SubprocVectorEnv +from tianshou.policy import GAILPolicy from tianshou.trainer import onpolicy_trainer +from tianshou.utils import TensorboardLogger from tianshou.utils.net.discrete import Actor, Critic, Discriminator -from tianshou.policy import GAILPolicy -from tianshou.data import Collector, VectorReplayBuffer - -from atari_network import DQN -from atari_wrapper import wrap_deepmind def get_args(): @@ -41,15 +41,19 @@ def get_args(): parser.add_argument("--logdir", type=str, default="log") parser.add_argument("--render", type=float, default=0.) parser.add_argument("--resume-path", type=str, default=None) - parser.add_argument("--watch", default=False, action="store_true", - help="watch the play of pre-trained policy only") + parser.add_argument( + "--watch", + default=False, + action="store_true", + help="watch the play of pre-trained policy only" + ) parser.add_argument("--log-interval", type=int, default=100) parser.add_argument( - "--load-buffer-name", type=str, - default="./expert_DQN_PongNoFrameskip-v4.hdf5") + "--load-buffer-name", type=str, default="./expert_DQN_PongNoFrameskip-v4.hdf5" + ) parser.add_argument( - "--device", type=str, - default="cuda" if torch.cuda.is_available() else "cpu") + "--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu" + ) args = parser.parse_known_args()[0] return args @@ -59,8 +63,12 @@ def make_atari_env(args): def make_atari_env_watch(args): - return wrap_deepmind(args.task, frame_stack=args.frames_stack, - episode_life=False, clip_rewards=False) + return wrap_deepmind( + args.task, + frame_stack=args.frames_stack, + episode_life=False, + clip_rewards=False + ) def test_gail(args=get_args()): @@ -72,25 +80,38 @@ def test_gail(args=get_args()): print("Observations shape:", args.state_shape) print("Actions shape:", args.action_shape) # make environments - train_envs = SubprocVectorEnv([lambda: make_atari_env(args) - for _ in range(args.training_num)]) - test_envs = SubprocVectorEnv([lambda: make_atari_env_watch(args) - for _ in range(args.test_num)]) + train_envs = SubprocVectorEnv( + [lambda: make_atari_env(args) for _ in range(args.training_num)] + ) + test_envs = SubprocVectorEnv( + [lambda: make_atari_env_watch(args) for _ in range(args.test_num)] + ) # seed np.random.seed(args.seed) torch.manual_seed(args.seed) test_envs.seed(args.seed) # model - feature_net = DQN(*args.state_shape, args.action_shape, - device=args.device, features_only=True).to(args.device) - actor = Actor(feature_net, args.action_shape, device=args.device, - hidden_sizes=args.hidden_sizes, softmax_output=False).to(args.device) + feature_net = DQN( + *args.state_shape, args.action_shape, device=args.device, features_only=True + ).to(args.device) + actor = Actor( + feature_net, + args.action_shape, + device=args.device, + hidden_sizes=args.hidden_sizes, + softmax_output=False + ).to(args.device) critic = Critic(feature_net, hidden_sizes=args.hidden_sizes, device=args.device).to(args.device) - optim = torch.optim.Adam(list(actor.parameters()) + list(critic.parameters()), - lr=args.lr) - disc = Discriminator(feature_net, args.action_shape, hidden_sizes=args.disc_hidden_sizes, - device=args.device).to(args.device) + optim = torch.optim.Adam( + list(actor.parameters()) + list(critic.parameters()), lr=args.lr + ) + disc = Discriminator( + feature_net, + args.action_shape, + hidden_sizes=args.disc_hidden_sizes, + device=args.device + ).to(args.device) disc_optim = torch.optim.Adam(disc.parameters(), lr=args.disc_lr) # buffer assert os.path.exists(args.load_buffer_name), \ @@ -108,20 +129,30 @@ def dist(logits): return Categorical(logits=logits) policy = GAILPolicy( - actor, critic, optim, dist, expert_buffer, disc, - disc_optim, disc_repeat=args.disc_repeat, - action_space=env.action_space, reward_normalization=True + actor, + critic, + optim, + dist, + expert_buffer, + disc, + disc_optim, + disc_repeat=args.disc_repeat, + action_space=env.action_space, + reward_normalization=True ).to(args.device) print("action_type=", policy.action_type, "rew_norm=", policy._rew_norm) # load a previous policy if args.resume_path: - policy.load_state_dict(torch.load( - args.resume_path, map_location=args.device)) + policy.load_state_dict(torch.load(args.resume_path, map_location=args.device)) print("Loaded agent from: ", args.resume_path) # buffer buffer = VectorReplayBuffer( - args.buffer_size, buffer_num=args.training_num, ignore_obs_next=True, - save_only_last_obs=True, stack_num=args.frames_stack) + args.buffer_size, + buffer_num=args.training_num, + ignore_obs_next=True, + save_only_last_obs=True, + stack_num=args.frames_stack + ) # collector train_collector = Collector(policy, train_envs, buffer, exploration_noise=True) @@ -130,7 +161,8 @@ def dist(logits): # log log_path = os.path.join( args.logdir, args.task, 'gail', - f'seed_{args.seed}_{datetime.datetime.now().strftime("%m%d-%H%M%S")}') + f'seed_{args.seed}_{datetime.datetime.now().strftime("%m%d-%H%M%S")}' + ) writer = SummaryWriter(log_path) writer.add_text("args", str(args)) logger = TensorboardLogger(writer) @@ -148,8 +180,7 @@ def watch(): test_envs.seed(args.seed) print("Testing agent ...") test_collector.reset() - result = test_collector.collect(n_episode=args.test_num, - render=args.render) + result = test_collector.collect(n_episode=args.test_num, render=args.render) pprint.pprint(result) rew = result["rews"].mean() print(f'Mean reward (over {result["n/ep"]} episodes): {rew}') @@ -159,10 +190,19 @@ def watch(): exit(0) result = onpolicy_trainer( - policy, train_collector, test_collector, args.epoch, args.step_per_epoch, - args.repeat_per_collect, args.test_num, args.batch_size, + policy, + train_collector, + test_collector, + args.epoch, + args.step_per_epoch, + args.repeat_per_collect, + args.test_num, + args.batch_size, step_per_collect=args.step_per_collect, - stop_fn=stop_fn, save_fn=save_fn, logger=logger) + stop_fn=stop_fn, + save_fn=save_fn, + logger=logger + ) pprint.pprint(result) watch() diff --git a/tianshou/policy/imitation/gail.py b/tianshou/policy/imitation/gail.py index e663003c6..a3d2d1757 100644 --- a/tianshou/policy/imitation/gail.py +++ b/tianshou/policy/imitation/gail.py @@ -1,14 +1,15 @@ +from typing import Any, Dict, List, Optional, Type + +import numpy as np import torch import torch.nn.functional as F -import numpy as np -from typing import Any, Dict, List, Optional, Type from tianshou.data import Batch, ReplayBuffer, to_torch from tianshou.policy import PPOPolicy class GAILPolicy(PPOPolicy): - """Implementation of Generative Adversarial Imitation Learning. arXiv:1606.03476. + r"""Implementation of Generative Adversarial Imitation Learning. arXiv:1606.03476. :param torch.nn.Module actor: the actor network following the rules in :class:`~tianshou.policy.BasePolicy`. (s -> logits) @@ -99,7 +100,9 @@ def learn( # type: ignore for _ in range(self.disc_repeat): for b in batch.split(batch_size, merge_last=True): logits_pi = self.disc(b.obs, b.act) - exp_b = to_torch(self.expert_buffer.sample(batch_size)[0], device=b.act.device) + exp_b = to_torch( + self.expert_buffer.sample(batch_size)[0], device=b.act.device + ) logits_exp = self.disc(exp_b.obs, exp_b.act) loss_pi = -F.logsigmoid(-logits_pi).mean() loss_exp = -F.logsigmoid(logits_exp).mean() @@ -110,7 +113,8 @@ def learn( # type: ignore losses.append(loss_disc.item()) # update reward with torch.no_grad(): - batch.rew = -F.logsigmoid(-self.disc(batch.obs, batch.act)).clamp_(self._eps, None) + batch.rew = -F.logsigmoid(-self.disc(batch.obs, batch.act) + ).clamp_(self._eps, None) # update policy res = super().learn(batch, batch_size, repeat, **kwargs) res["loss/disc"] = np.mean(losses) From bf0b33be8f9bf9398581bcd98843dce393fc7a9b Mon Sep 17 00:00:00 2001 From: Yi Su Date: Mon, 21 Feb 2022 11:18:08 +0800 Subject: [PATCH 03/18] update atari_gail.py to latest version --- examples/atari/atari_gail.py | 212 ------------------------ examples/offline/atari_gail.py | 290 +++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+), 212 deletions(-) delete mode 100644 examples/atari/atari_gail.py create mode 100644 examples/offline/atari_gail.py diff --git a/examples/atari/atari_gail.py b/examples/atari/atari_gail.py deleted file mode 100644 index 40d2d0a9b..000000000 --- a/examples/atari/atari_gail.py +++ /dev/null @@ -1,212 +0,0 @@ -import argparse -import datetime -import os -import pickle -import pprint - -import numpy as np -import torch -from atari_network import DQN -from atari_wrapper import wrap_deepmind -from torch.distributions import Categorical -from torch.utils.tensorboard import SummaryWriter - -from tianshou.data import Collector, VectorReplayBuffer -from tianshou.env import SubprocVectorEnv -from tianshou.policy import GAILPolicy -from tianshou.trainer import onpolicy_trainer -from tianshou.utils import TensorboardLogger -from tianshou.utils.net.discrete import Actor, Critic, Discriminator - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--task", type=str, default="PongNoFrameskip-v4") - parser.add_argument("--seed", type=int, default=1626) - parser.add_argument('--buffer-size', type=int, default=10000) - parser.add_argument("--lr", type=float, default=0.00005) - parser.add_argument("--gamma", type=float, default=0.99) - parser.add_argument("--disc-lr", type=float, default=0.0001) - parser.add_argument("--disc-repeat", type=int, default=5) - parser.add_argument("--disc-hidden-sizes", type=int, nargs="*", default=[128]) - parser.add_argument("--epoch", type=int, default=100) - parser.add_argument('--step-per-epoch', type=int, default=100000) - parser.add_argument('--step-per-collect', type=int, default=500) - parser.add_argument('--repeat-per-collect', type=int, default=1) - parser.add_argument("--batch-size", type=int, default=32) - parser.add_argument('--hidden-sizes', type=int, nargs='*', default=[512]) - parser.add_argument('--training-num', type=int, default=10) - parser.add_argument("--test-num", type=int, default=10) - parser.add_argument('--frames-stack', type=int, default=4) - parser.add_argument("--logdir", type=str, default="log") - parser.add_argument("--render", type=float, default=0.) - parser.add_argument("--resume-path", type=str, default=None) - parser.add_argument( - "--watch", - default=False, - action="store_true", - help="watch the play of pre-trained policy only" - ) - parser.add_argument("--log-interval", type=int, default=100) - parser.add_argument( - "--load-buffer-name", type=str, default="./expert_DQN_PongNoFrameskip-v4.hdf5" - ) - parser.add_argument( - "--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu" - ) - args = parser.parse_known_args()[0] - return args - - -def make_atari_env(args): - return wrap_deepmind(args.task, frame_stack=args.frames_stack) - - -def make_atari_env_watch(args): - return wrap_deepmind( - args.task, - frame_stack=args.frames_stack, - episode_life=False, - clip_rewards=False - ) - - -def test_gail(args=get_args()): - # envs - env = make_atari_env(args) - args.state_shape = env.observation_space.shape or env.observation_space.n - args.action_shape = env.action_space.shape or env.action_space.n - # should be N_FRAMES x H x W - print("Observations shape:", args.state_shape) - print("Actions shape:", args.action_shape) - # make environments - train_envs = SubprocVectorEnv( - [lambda: make_atari_env(args) for _ in range(args.training_num)] - ) - test_envs = SubprocVectorEnv( - [lambda: make_atari_env_watch(args) for _ in range(args.test_num)] - ) - # seed - np.random.seed(args.seed) - torch.manual_seed(args.seed) - test_envs.seed(args.seed) - # model - feature_net = DQN( - *args.state_shape, args.action_shape, device=args.device, features_only=True - ).to(args.device) - actor = Actor( - feature_net, - args.action_shape, - device=args.device, - hidden_sizes=args.hidden_sizes, - softmax_output=False - ).to(args.device) - critic = Critic(feature_net, hidden_sizes=args.hidden_sizes, - device=args.device).to(args.device) - optim = torch.optim.Adam( - list(actor.parameters()) + list(critic.parameters()), lr=args.lr - ) - disc = Discriminator( - feature_net, - args.action_shape, - hidden_sizes=args.disc_hidden_sizes, - device=args.device - ).to(args.device) - disc_optim = torch.optim.Adam(disc.parameters(), lr=args.disc_lr) - # buffer - assert os.path.exists(args.load_buffer_name), \ - "Please run atari_qrdqn.py first to get expert's data buffer." - if args.load_buffer_name.endswith('.pkl'): - expert_buffer = pickle.load(open(args.load_buffer_name, "rb")) - elif args.load_buffer_name.endswith('.hdf5'): - expert_buffer = VectorReplayBuffer.load_hdf5(args.load_buffer_name) - else: - print(f"Unknown buffer format: {args.load_buffer_name}") - exit(0) - # define policy - - def dist(logits): - return Categorical(logits=logits) - - policy = GAILPolicy( - actor, - critic, - optim, - dist, - expert_buffer, - disc, - disc_optim, - disc_repeat=args.disc_repeat, - action_space=env.action_space, - reward_normalization=True - ).to(args.device) - print("action_type=", policy.action_type, "rew_norm=", policy._rew_norm) - # load a previous policy - if args.resume_path: - policy.load_state_dict(torch.load(args.resume_path, map_location=args.device)) - print("Loaded agent from: ", args.resume_path) - # buffer - buffer = VectorReplayBuffer( - args.buffer_size, - buffer_num=args.training_num, - ignore_obs_next=True, - save_only_last_obs=True, - stack_num=args.frames_stack - ) - - # collector - train_collector = Collector(policy, train_envs, buffer, exploration_noise=True) - test_collector = Collector(policy, test_envs, exploration_noise=True) - - # log - log_path = os.path.join( - args.logdir, args.task, 'gail', - f'seed_{args.seed}_{datetime.datetime.now().strftime("%m%d-%H%M%S")}' - ) - writer = SummaryWriter(log_path) - writer.add_text("args", str(args)) - logger = TensorboardLogger(writer) - - def save_fn(policy): - torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth')) - - def stop_fn(mean_rewards): - return False - - # watch agent's performance - def watch(): - print("Setup test envs ...") - policy.eval() - test_envs.seed(args.seed) - print("Testing agent ...") - test_collector.reset() - result = test_collector.collect(n_episode=args.test_num, render=args.render) - pprint.pprint(result) - rew = result["rews"].mean() - print(f'Mean reward (over {result["n/ep"]} episodes): {rew}') - - if args.watch: - watch() - exit(0) - - result = onpolicy_trainer( - policy, - train_collector, - test_collector, - args.epoch, - args.step_per_epoch, - args.repeat_per_collect, - args.test_num, - args.batch_size, - step_per_collect=args.step_per_collect, - stop_fn=stop_fn, - save_fn=save_fn, - logger=logger - ) - - pprint.pprint(result) - watch() - - -if __name__ == "__main__": - test_gail(get_args()) diff --git a/examples/offline/atari_gail.py b/examples/offline/atari_gail.py new file mode 100644 index 000000000..6477bfeca --- /dev/null +++ b/examples/offline/atari_gail.py @@ -0,0 +1,290 @@ +import argparse +import os +import pickle +import pprint + +import numpy as np +import torch +from torch.optim.lr_scheduler import LambdaLR +from torch.utils.tensorboard import SummaryWriter + +from examples.atari.atari_network import DQN +from examples.atari.atari_wrapper import wrap_deepmind +from tianshou.data import Collector, VectorReplayBuffer +from tianshou.env import ShmemVectorEnv +from tianshou.policy import GAILPolicy +from tianshou.trainer import onpolicy_trainer +from tianshou.utils import TensorboardLogger, WandbLogger +from tianshou.utils.net.common import ActorCritic +from tianshou.utils.net.discrete import Actor, Critic, Discriminator + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--task', type=str, default='PongNoFrameskip-v4') + parser.add_argument('--seed', type=int, default=4213) + parser.add_argument('--scale-obs', type=int, default=0) + parser.add_argument('--buffer-size', type=int, default=100000) + parser.add_argument('--lr', type=float, default=5e-5) + parser.add_argument('--gamma', type=float, default=0.99) + parser.add_argument('--epoch', type=int, default=100) + parser.add_argument("--disc-lr", type=float, default=0.0001) + parser.add_argument("--disc-repeat", type=int, default=5) + parser.add_argument("--disc-hidden-size", type=int, default=128) + parser.add_argument('--step-per-epoch', type=int, default=100000) + parser.add_argument('--step-per-collect', type=int, default=1000) + parser.add_argument('--repeat-per-collect', type=int, default=4) + parser.add_argument('--batch-size', type=int, default=256) + parser.add_argument('--hidden-size', type=int, default=512) + parser.add_argument('--training-num', type=int, default=10) + parser.add_argument('--test-num', type=int, default=10) + parser.add_argument('--rew-norm', type=int, default=False) + parser.add_argument('--vf-coef', type=float, default=0.5) + parser.add_argument('--ent-coef', type=float, default=0.01) + parser.add_argument('--gae-lambda', type=float, default=0.95) + parser.add_argument('--lr-decay', type=int, default=True) + parser.add_argument('--max-grad-norm', type=float, default=0.5) + parser.add_argument('--eps-clip', type=float, default=0.2) + parser.add_argument('--dual-clip', type=float, default=None) + parser.add_argument('--value-clip', type=int, default=0) + parser.add_argument('--norm-adv', type=int, default=1) + parser.add_argument('--recompute-adv', type=int, default=0) + parser.add_argument('--logdir', type=str, default='log') + parser.add_argument('--render', type=float, default=0.) + parser.add_argument( + '--device', type=str, default='cuda' if torch.cuda.is_available() else 'cpu' + ) + parser.add_argument('--frames-stack', type=int, default=4) + parser.add_argument('--resume-path', type=str, default=None) + parser.add_argument('--resume-id', type=str, default=None) + parser.add_argument( + '--logger', + type=str, + default="tensorboard", + choices=["tensorboard", "wandb"], + ) + parser.add_argument( + '--watch', + default=False, + action='store_true', + help='watch the play of pre-trained policy only' + ) + parser.add_argument('--save-buffer-name', type=str, default=None) + parser.add_argument( + "--load-buffer-name", type=str, default="./expert_DQN_PongNoFrameskip-v4.hdf5" + ) + return parser.parse_args() + + +def make_atari_env(args): + return wrap_deepmind( + args.task, frame_stack=args.frames_stack, scale=args.scale_obs + ) + + +def make_atari_env_watch(args): + return wrap_deepmind( + args.task, + frame_stack=args.frames_stack, + episode_life=False, + clip_rewards=False, + scale=args.scale_obs + ) + + +def test_gail(args=get_args()): + env = make_atari_env(args) + args.state_shape = env.observation_space.shape or env.observation_space.n + args.action_shape = env.action_space.shape or env.action_space.n + # should be N_FRAMES x H x W + print("Observations shape:", args.state_shape) + print("Actions shape:", args.action_shape) + # make environments + train_envs = ShmemVectorEnv( + [lambda: make_atari_env(args) for _ in range(args.training_num)] + ) + test_envs = ShmemVectorEnv( + [lambda: make_atari_env_watch(args) for _ in range(args.test_num)] + ) + # seed + np.random.seed(args.seed) + torch.manual_seed(args.seed) + train_envs.seed(args.seed) + test_envs.seed(args.seed) + # define model + net = DQN( + *args.state_shape, + args.action_shape, + device=args.device, + features_only=True, + output_dim=args.hidden_size + ) + actor = Actor(net, args.action_shape, device=args.device, softmax_output=False) + critic = Critic(net, device=args.device) + optim = torch.optim.Adam(ActorCritic(actor, critic).parameters(), lr=args.lr) + disc = Discriminator( + net, + args.action_shape, + hidden_sizes=[args.disc_hidden_size], + device=args.device + ).to(args.device) + disc_optim = torch.optim.Adam(disc.parameters(), lr=args.disc_lr) + + lr_scheduler = None + if args.lr_decay: + # decay learning rate to 0 linearly + max_update_num = np.ceil( + args.step_per_epoch / args.step_per_collect + ) * args.epoch + + lr_scheduler = LambdaLR( + optim, lr_lambda=lambda epoch: 1 - epoch / max_update_num + ) + + # buffer + assert os.path.exists(args.load_buffer_name), \ + "Please run atari_qrdqn.py first to get expert's data buffer." + if args.load_buffer_name.endswith('.pkl'): + expert_buffer = pickle.load(open(args.load_buffer_name, "rb")) + elif args.load_buffer_name.endswith('.hdf5'): + expert_buffer = VectorReplayBuffer.load_hdf5(args.load_buffer_name) + else: + print(f"Unknown buffer format: {args.load_buffer_name}") + exit(0) + + # define policy + def dist(p): + return torch.distributions.Categorical(logits=p) + + policy = GAILPolicy( + actor, + critic, + optim, + dist, + expert_buffer, + disc, + disc_optim, + disc_repeat=args.disc_repeat, + discount_factor=args.gamma, + gae_lambda=args.gae_lambda, + max_grad_norm=args.max_grad_norm, + vf_coef=args.vf_coef, + ent_coef=args.ent_coef, + reward_normalization=args.rew_norm, + action_scaling=False, + lr_scheduler=lr_scheduler, + action_space=env.action_space, + eps_clip=args.eps_clip, + value_clip=args.value_clip, + dual_clip=args.dual_clip, + advantage_normalization=args.norm_adv, + recompute_advantage=args.recompute_adv + ).to(args.device) + # load a previous policy + if args.resume_path: + policy.load_state_dict(torch.load(args.resume_path, map_location=args.device)) + print("Loaded agent from: ", args.resume_path) + # replay buffer: `save_last_obs` and `stack_num` can be removed together + # when you have enough RAM + buffer = VectorReplayBuffer( + args.buffer_size, + buffer_num=len(train_envs), + ignore_obs_next=True, + save_only_last_obs=True, + stack_num=args.frames_stack + ) + # collector + train_collector = Collector(policy, train_envs, buffer, exploration_noise=True) + test_collector = Collector(policy, test_envs, exploration_noise=True) + # log + log_name = 'gail' + log_path = os.path.join(args.logdir, args.task, log_name) + if args.logger == "tensorboard": + writer = SummaryWriter(log_path) + writer.add_text("args", str(args)) + logger = TensorboardLogger(writer) + else: + logger = WandbLogger( + save_interval=1, + project=args.task, + name=log_name, + run_id=args.resume_id, + config=args, + ) + + def save_fn(policy): + torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth')) + + def stop_fn(mean_rewards): + if env.spec.reward_threshold: + return mean_rewards >= env.spec.reward_threshold + elif 'Pong' in args.task: + return mean_rewards >= 20 + else: + return False + + def save_checkpoint_fn(epoch, env_step, gradient_step): + # see also: https://pytorch.org/tutorials/beginner/saving_loading_models.html + ckpt_path = os.path.join(log_path, 'checkpoint.pth') + torch.save({'model': policy.state_dict()}, ckpt_path) + return ckpt_path + + # watch agent's performance + def watch(): + print("Setup test envs ...") + policy.eval() + test_envs.seed(args.seed) + if args.save_buffer_name: + print(f"Generate buffer with size {args.buffer_size}") + buffer = VectorReplayBuffer( + args.buffer_size, + buffer_num=len(test_envs), + ignore_obs_next=True, + save_only_last_obs=True, + stack_num=args.frames_stack + ) + collector = Collector(policy, test_envs, buffer, exploration_noise=True) + result = collector.collect(n_step=args.buffer_size) + print(f"Save buffer into {args.save_buffer_name}") + # Unfortunately, pickle will cause oom with 1M buffer size + buffer.save_hdf5(args.save_buffer_name) + else: + print("Testing agent ...") + test_collector.reset() + result = test_collector.collect( + n_episode=args.test_num, render=args.render + ) + rew = result["rews"].mean() + print(f'Mean reward (over {result["n/ep"]} episodes): {rew}') + + if args.watch: + watch() + exit(0) + + # test train_collector and start filling replay buffer + train_collector.collect(n_step=args.batch_size * args.training_num) + # trainer + result = onpolicy_trainer( + policy, + train_collector, + test_collector, + args.epoch, + args.step_per_epoch, + args.repeat_per_collect, + args.test_num, + args.batch_size, + step_per_collect=args.step_per_collect, + stop_fn=stop_fn, + save_fn=save_fn, + logger=logger, + test_in_train=False, + resume_from_log=args.resume_id is not None, + save_checkpoint_fn=save_checkpoint_fn, + ) + + pprint.pprint(result) + watch() + + +if __name__ == '__main__': + test_gail(get_args()) From 427f18f099f5b5adbf9faaed2295f6fcd6e95633 Mon Sep 17 00:00:00 2001 From: Yi Su Date: Tue, 22 Feb 2022 02:26:44 +0800 Subject: [PATCH 04/18] update Discriminator net arch --- examples/offline/atari_gail.py | 4 +--- tianshou/utils/net/discrete.py | 12 +++++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/offline/atari_gail.py b/examples/offline/atari_gail.py index 6477bfeca..c451f8b24 100644 --- a/examples/offline/atari_gail.py +++ b/examples/offline/atari_gail.py @@ -29,8 +29,7 @@ def get_args(): parser.add_argument('--gamma', type=float, default=0.99) parser.add_argument('--epoch', type=int, default=100) parser.add_argument("--disc-lr", type=float, default=0.0001) - parser.add_argument("--disc-repeat", type=int, default=5) - parser.add_argument("--disc-hidden-size", type=int, default=128) + parser.add_argument("--disc-repeat", type=int, default=4) parser.add_argument('--step-per-epoch', type=int, default=100000) parser.add_argument('--step-per-collect', type=int, default=1000) parser.add_argument('--repeat-per-collect', type=int, default=4) @@ -125,7 +124,6 @@ def test_gail(args=get_args()): disc = Discriminator( net, args.action_shape, - hidden_sizes=[args.disc_hidden_size], device=args.device ).to(args.device) disc_optim = torch.optim.Adam(disc.parameters(), lr=args.disc_lr) diff --git a/tianshou/utils/net/discrete.py b/tianshou/utils/net/discrete.py index bf9121c25..b89a7a48f 100644 --- a/tianshou/utils/net/discrete.py +++ b/tianshou/utils/net/discrete.py @@ -407,7 +407,7 @@ class IntrinsicCuriosityModule(nn.Module): def __init__( self, - feature_net: nn.Module, + feature_net: nn.Module, feature_dim: int, action_dim: int, hidden_sizes: Sequence[int] = (), @@ -470,14 +470,16 @@ def __init__( self.preprocess = preprocess_net self.device = device self.output_dim = 1 - state_dim = getattr(preprocess_net, "output_dim", - preprocess_net_output_dim) + state_dim = getattr(preprocess_net, "output_dim", preprocess_net_output_dim) action_dim = int(np.prod(action_shape)) self.action_dim = action_dim - self.net = MLP(state_dim, action_dim, hidden_sizes, device=self.device) + self.net = MLP(state_dim + action_dim, 1, hidden_sizes, device=self.device) def forward( self, obs: torch.Tensor, act: torch.Tensor, **kwargs: Any ) -> torch.Tensor: s, _ = self.preprocess(obs, state=kwargs.get("state", None)) - return self.net(s).gather(-1, act.long().view(-1, 1)) + act = to_torch(act, dtype=torch.long, device=self.device) + return self.net( + torch.cat([s, F.one_hot(act, num_classes=self.action_dim)], dim=1) + ) From 173e40bd257afea3f1cb987311a9c0175d797d6a Mon Sep 17 00:00:00 2001 From: Yi Su Date: Tue, 22 Feb 2022 09:23:34 +0800 Subject: [PATCH 05/18] fix a bug about when to compute rewards --- examples/atari/atari_wrapper.py | 22 ++++++++++++++++++++-- examples/offline/atari_gail.py | 11 +++++------ tianshou/policy/imitation/gail.py | 17 +++++++++++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/examples/atari/atari_wrapper.py b/examples/atari/atari_wrapper.py index 4aca61218..86b94b1cc 100644 --- a/examples/atari/atari_wrapper.py +++ b/examples/atari/atari_wrapper.py @@ -181,6 +181,20 @@ def reward(self, reward): return np.sign(reward) +class NoRewardEnv(gym.RewardWrapper): + """sets the reward to 0. + + :param gym.Env env: the environment to wrap. + """ + + def __init__(self, env): + super().__init__(env) + + def reward(self, reward): + """Set reward to 0.""" + return np.zeros_like(reward) + + class FrameStack(gym.Wrapper): """Stack n_frames last frames. @@ -223,7 +237,8 @@ def wrap_deepmind( clip_rewards=True, frame_stack=4, scale=False, - warp_frame=True + warp_frame=True, + no_rewards=False ): """Configure environment for DeepMind-style Atari. The observation is channel-first: (c, h, w) instead of (h, w, c). @@ -234,6 +249,7 @@ def wrap_deepmind( :param int frame_stack: wrap the frame stacking wrapper. :param bool scale: wrap the scaling observation wrapper. :param bool warp_frame: wrap the grayscale + resize observation wrapper. + :param bool no_rewards: wrap the no reward wrapper. :return: the wrapped atari environment. """ assert 'NoFrameskip' in env_id @@ -248,7 +264,9 @@ def wrap_deepmind( env = WarpFrame(env) if scale: env = ScaledFloatFrame(env) - if clip_rewards: + if no_rewards: + env = NoRewardEnv(env) + elif clip_rewards: env = ClipRewardEnv(env) if frame_stack: env = FrameStack(env, frame_stack) diff --git a/examples/offline/atari_gail.py b/examples/offline/atari_gail.py index c451f8b24..9596f969e 100644 --- a/examples/offline/atari_gail.py +++ b/examples/offline/atari_gail.py @@ -77,7 +77,10 @@ def get_args(): def make_atari_env(args): return wrap_deepmind( - args.task, frame_stack=args.frames_stack, scale=args.scale_obs + args.task, + frame_stack=args.frames_stack, + scale=args.scale_obs, + no_rewards=True ) @@ -121,11 +124,7 @@ def test_gail(args=get_args()): actor = Actor(net, args.action_shape, device=args.device, softmax_output=False) critic = Critic(net, device=args.device) optim = torch.optim.Adam(ActorCritic(actor, critic).parameters(), lr=args.lr) - disc = Discriminator( - net, - args.action_shape, - device=args.device - ).to(args.device) + disc = Discriminator(net, args.action_shape, device=args.device).to(args.device) disc_optim = torch.optim.Adam(disc.parameters(), lr=args.disc_lr) lr_scheduler = None diff --git a/tianshou/policy/imitation/gail.py b/tianshou/policy/imitation/gail.py index a3d2d1757..cad2dd18a 100644 --- a/tianshou/policy/imitation/gail.py +++ b/tianshou/policy/imitation/gail.py @@ -92,6 +92,19 @@ def __init__( self.expert_buffer = expert_buffer self.action_dim = actor.output_dim + def process_fn( + self, batch: Batch, buffer: ReplayBuffer, indices: np.ndarray + ) -> Batch: + """Pre-process the data from the provided replay buffer. + + Used in :meth:`update`. Check out :ref:`process_fn` for more information. + """ + # update reward + with torch.no_grad(): + batch.rew = -F.logsigmoid(-self.disc(batch.obs, batch.act) + ).clamp_(self._eps, None) + return super().process_fn(batch, buffer, indices) + def learn( # type: ignore self, batch: Batch, batch_size: int, repeat: int, **kwargs: Any ) -> Dict[str, List[float]]: @@ -111,10 +124,6 @@ def learn( # type: ignore loss_disc.backward() self.disc_optim.step() losses.append(loss_disc.item()) - # update reward - with torch.no_grad(): - batch.rew = -F.logsigmoid(-self.disc(batch.obs, batch.act) - ).clamp_(self._eps, None) # update policy res = super().learn(batch, batch_size, repeat, **kwargs) res["loss/disc"] = np.mean(losses) From 1025a58b4db89127d6d796295ea6e9166c882a8e Mon Sep 17 00:00:00 2001 From: Yi Su Date: Tue, 22 Feb 2022 10:10:08 +0800 Subject: [PATCH 06/18] fix a bug about tensor shape --- tianshou/policy/imitation/gail.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tianshou/policy/imitation/gail.py b/tianshou/policy/imitation/gail.py index cad2dd18a..304d4de7b 100644 --- a/tianshou/policy/imitation/gail.py +++ b/tianshou/policy/imitation/gail.py @@ -4,7 +4,7 @@ import torch import torch.nn.functional as F -from tianshou.data import Batch, ReplayBuffer, to_torch +from tianshou.data import Batch, ReplayBuffer, to_torch, to_numpy from tianshou.policy import PPOPolicy @@ -101,8 +101,10 @@ def process_fn( """ # update reward with torch.no_grad(): - batch.rew = -F.logsigmoid(-self.disc(batch.obs, batch.act) - ).clamp_(self._eps, None) + batch.rew = to_numpy( + -F.logsigmoid(-self.disc(batch.obs, batch.act) + ).clamp_(self._eps, None).flatten() + ) return super().process_fn(batch, buffer, indices) def learn( # type: ignore From ff15c386269702cf44dc24fedd6e3ec57b03032e Mon Sep 17 00:00:00 2001 From: Yi Su Date: Wed, 2 Mar 2022 08:00:55 +0800 Subject: [PATCH 07/18] add gail example for Mujoco tasks --- examples/offline/irl_gail.py | 262 ++++++++++++++++++++++++++++++ tianshou/policy/imitation/gail.py | 42 ++--- tianshou/utils/net/continuous.py | 41 ++++- tianshou/utils/net/discrete.py | 13 +- 4 files changed, 335 insertions(+), 23 deletions(-) create mode 100644 examples/offline/irl_gail.py diff --git a/examples/offline/irl_gail.py b/examples/offline/irl_gail.py new file mode 100644 index 000000000..cd39c4663 --- /dev/null +++ b/examples/offline/irl_gail.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import os +import pprint + +import d4rl +import gym +import numpy as np +import torch +from torch import nn +from torch.distributions import Independent, Normal +from torch.optim.lr_scheduler import LambdaLR +from torch.utils.tensorboard import SummaryWriter + +from examples.atari.atari_wrapper import NoRewardEnv +from tianshou.data import Batch, Collector, ReplayBuffer, VectorReplayBuffer +from tianshou.env import SubprocVectorEnv +from tianshou.policy import GAILPolicy +from tianshou.trainer import onpolicy_trainer +from tianshou.utils import TensorboardLogger +from tianshou.utils.net.common import ActorCritic, Net +from tianshou.utils.net.continuous import ActorProb, Critic, GAILDiscriminator + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--task', type=str, default='HalfCheetah-v2') + parser.add_argument('--seed', type=int, default=0) + parser.add_argument( + '--expert-data-task', type=str, default='halfcheetah-expert-v2' + ) + parser.add_argument('--buffer-size', type=int, default=4096) + parser.add_argument('--hidden-sizes', type=int, nargs='*', default=[64, 64]) + parser.add_argument('--lr', type=float, default=3e-4) + parser.add_argument('--disc-lr', type=float, default=2.5e-5) + parser.add_argument('--gamma', type=float, default=0.99) + parser.add_argument('--epoch', type=int, default=100) + parser.add_argument('--step-per-epoch', type=int, default=30000) + parser.add_argument('--step-per-collect', type=int, default=2048) + parser.add_argument('--repeat-per-collect', type=int, default=10) + parser.add_argument('--disc-repeat', type=int, default=2) + parser.add_argument('--batch-size', type=int, default=64) + parser.add_argument('--training-num', type=int, default=64) + parser.add_argument('--test-num', type=int, default=10) + # ppo special + parser.add_argument('--rew-norm', type=int, default=True) + # In theory, `vf-coef` will not make any difference if using Adam optimizer. + parser.add_argument('--vf-coef', type=float, default=0.25) + parser.add_argument('--ent-coef', type=float, default=0.001) + parser.add_argument('--gae-lambda', type=float, default=0.95) + parser.add_argument('--bound-action-method', type=str, default="clip") + parser.add_argument('--lr-decay', type=int, default=True) + parser.add_argument('--max-grad-norm', type=float, default=0.5) + parser.add_argument('--eps-clip', type=float, default=0.2) + parser.add_argument('--dual-clip', type=float, default=None) + parser.add_argument('--value-clip', type=int, default=0) + parser.add_argument('--norm-adv', type=int, default=0) + parser.add_argument('--recompute-adv', type=int, default=1) + parser.add_argument('--logdir', type=str, default='log') + parser.add_argument('--render', type=float, default=0.) + parser.add_argument( + '--device', type=str, default='cuda' if torch.cuda.is_available() else 'cpu' + ) + parser.add_argument('--resume-path', type=str, default=None) + parser.add_argument( + '--watch', + default=False, + action='store_true', + help='watch the play of pre-trained policy only' + ) + return parser.parse_args() + + +def test_gail(args=get_args()): + env = gym.make(args.task) + args.state_shape = env.observation_space.shape or env.observation_space.n + args.action_shape = env.action_space.shape or env.action_space.n + args.max_action = env.action_space.high[0] + print("Observations shape:", args.state_shape) + print("Actions shape:", args.action_shape) + print("Action range:", np.min(env.action_space.low), np.max(env.action_space.high)) + # train_envs = gym.make(args.task) + train_envs = SubprocVectorEnv( + [lambda: NoRewardEnv(gym.make(args.task)) for _ in range(args.training_num)], + norm_obs=True + ) + # test_envs = gym.make(args.task) + test_envs = SubprocVectorEnv( + [lambda: gym.make(args.task) for _ in range(args.test_num)], + norm_obs=True, + obs_rms=train_envs.obs_rms, + update_obs_rms=False + ) + + # seed + np.random.seed(args.seed) + torch.manual_seed(args.seed) + train_envs.seed(args.seed) + test_envs.seed(args.seed) + # model + net_a = Net( + args.state_shape, + hidden_sizes=args.hidden_sizes, + activation=nn.Tanh, + device=args.device + ) + actor = ActorProb( + net_a, + args.action_shape, + max_action=args.max_action, + unbounded=True, + device=args.device + ).to(args.device) + net_c = Net( + args.state_shape, + hidden_sizes=args.hidden_sizes, + activation=nn.Tanh, + device=args.device + ) + critic = Critic(net_c, device=args.device).to(args.device) + torch.nn.init.constant_(actor.sigma_param, -0.5) + for m in list(actor.modules()) + list(critic.modules()): + if isinstance(m, torch.nn.Linear): + # orthogonal initialization + torch.nn.init.orthogonal_(m.weight, gain=np.sqrt(2)) + torch.nn.init.zeros_(m.bias) + # do last policy layer scaling, this will make initial actions have (close to) + # 0 mean and std, and will help boost performances, + # see https://arxiv.org/abs/2006.05990, Fig.24 for details + for m in actor.mu.modules(): + if isinstance(m, torch.nn.Linear): + torch.nn.init.zeros_(m.bias) + m.weight.data.copy_(0.01 * m.weight.data) + + optim = torch.optim.Adam(ActorCritic(actor, critic).parameters(), lr=args.lr) + # discriminator + disc = GAILDiscriminator( + args.state_shape, + args.action_shape, + hidden_sizes=args.hidden_sizes, + activation=nn.Tanh, + device=args.device + ).to(args.device) + for m in disc.modules(): + if isinstance(m, torch.nn.Linear): + # orthogonal initialization + torch.nn.init.orthogonal_(m.weight, gain=np.sqrt(2)) + torch.nn.init.zeros_(m.bias) + disc_optim = torch.optim.Adam(disc.parameters(), lr=args.disc_lr) + + lr_scheduler = None + if args.lr_decay: + # decay learning rate to 0 linearly + max_update_num = np.ceil( + args.step_per_epoch / args.step_per_collect + ) * args.epoch + + lr_scheduler = LambdaLR( + optim, lr_lambda=lambda epoch: 1 - epoch / max_update_num + ) + + def dist(*logits): + return Independent(Normal(*logits), 1) + + # expert replay buffer + dataset = d4rl.qlearning_dataset(gym.make(args.expert_data_task)) + dataset_size = dataset['rewards'].size + + print("dataset_size", dataset_size) + expert_buffer = ReplayBuffer(dataset_size) + + for i in range(dataset_size): + expert_buffer.add( + Batch( + obs=dataset['observations'][i], + act=dataset['actions'][i], + rew=dataset['rewards'][i], + done=dataset['terminals'][i], + obs_next=dataset['next_observations'][i], + ) + ) + print("dataset loaded") + + policy = GAILPolicy( + actor, + critic, + optim, + dist, + expert_buffer, + disc, + disc_optim, + disc_repeat=args.disc_repeat, + discount_factor=args.gamma, + gae_lambda=args.gae_lambda, + max_grad_norm=args.max_grad_norm, + vf_coef=args.vf_coef, + ent_coef=args.ent_coef, + reward_normalization=args.rew_norm, + action_scaling=True, + action_bound_method=args.bound_action_method, + lr_scheduler=lr_scheduler, + action_space=env.action_space, + eps_clip=args.eps_clip, + value_clip=args.value_clip, + dual_clip=args.dual_clip, + advantage_normalization=args.norm_adv, + recompute_advantage=args.recompute_adv + ) + + # load a previous policy + if args.resume_path: + policy.load_state_dict(torch.load(args.resume_path, map_location=args.device)) + print("Loaded agent from: ", args.resume_path) + + # collector + if args.training_num > 1: + buffer = VectorReplayBuffer(args.buffer_size, len(train_envs)) + else: + buffer = ReplayBuffer(args.buffer_size) + train_collector = Collector(policy, train_envs, buffer, exploration_noise=True) + test_collector = Collector(policy, test_envs) + # log + t0 = datetime.datetime.now().strftime("%m%d_%H%M%S") + log_file = f'seed_{args.seed}_{t0}-{args.task.replace("-", "_")}_gail' + log_path = os.path.join(args.logdir, args.task, 'gail', log_file) + writer = SummaryWriter(log_path) + writer.add_text("args", str(args)) + logger = TensorboardLogger(writer, update_interval=100, train_interval=100) + + def save_fn(policy): + torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth')) + + if not args.watch: + # trainer + result = onpolicy_trainer( + policy, + train_collector, + test_collector, + args.epoch, + args.step_per_epoch, + args.repeat_per_collect, + args.test_num, + args.batch_size, + step_per_collect=args.step_per_collect, + save_fn=save_fn, + logger=logger, + test_in_train=False + ) + pprint.pprint(result) + + # Let's watch its performance! + policy.eval() + test_envs.seed(args.seed) + test_collector.reset() + result = test_collector.collect(n_episode=args.test_num, render=args.render) + print(f'Final reward: {result["rews"].mean()}, length: {result["lens"].mean()}') + + +if __name__ == '__main__': + test_gail() diff --git a/tianshou/policy/imitation/gail.py b/tianshou/policy/imitation/gail.py index 304d4de7b..d19501494 100644 --- a/tianshou/policy/imitation/gail.py +++ b/tianshou/policy/imitation/gail.py @@ -4,7 +4,7 @@ import torch import torch.nn.functional as F -from tianshou.data import Batch, ReplayBuffer, to_torch, to_numpy +from tianshou.data import Batch, ReplayBuffer, to_numpy, to_torch from tianshou.policy import PPOPolicy @@ -74,7 +74,7 @@ def __init__( expert_buffer: ReplayBuffer, disc: torch.nn.Module, disc_optim: torch.optim.Optimizer, - disc_repeat: int = 5, + disc_repeat: int = 2, eps_clip: float = 0.2, dual_clip: Optional[float] = None, value_clip: bool = False, @@ -102,8 +102,7 @@ def process_fn( # update reward with torch.no_grad(): batch.rew = to_numpy( - -F.logsigmoid(-self.disc(batch.obs, batch.act) - ).clamp_(self._eps, None).flatten() + -F.logsigmoid(-self.disc(batch.obs, batch.act)).flatten() ) return super().process_fn(batch, buffer, indices) @@ -112,21 +111,26 @@ def learn( # type: ignore ) -> Dict[str, List[float]]: # update discriminator losses = [] - for _ in range(self.disc_repeat): - for b in batch.split(batch_size, merge_last=True): - logits_pi = self.disc(b.obs, b.act) - exp_b = to_torch( - self.expert_buffer.sample(batch_size)[0], device=b.act.device - ) - logits_exp = self.disc(exp_b.obs, exp_b.act) - loss_pi = -F.logsigmoid(-logits_pi).mean() - loss_exp = -F.logsigmoid(logits_exp).mean() - loss_disc = loss_pi + loss_exp - self.disc_optim.zero_grad() - loss_disc.backward() - self.disc_optim.step() - losses.append(loss_disc.item()) + acc_pis = [] + acc_exps = [] + for b in batch.split(len(batch) // self.disc_repeat, merge_last=True): + logits_pi = self.disc(b.obs, b.act) + exp_b = to_torch( + self.expert_buffer.sample(batch_size)[0], device=b.act.device + ) + logits_exp = self.disc(exp_b.obs, exp_b.act) + loss_pi = -F.logsigmoid(-logits_pi).mean() + loss_exp = -F.logsigmoid(logits_exp).mean() + loss_disc = loss_pi + loss_exp + self.disc_optim.zero_grad() + loss_disc.backward() + self.disc_optim.step() + losses.append(loss_disc.item()) + acc_pis.append((logits_pi < 0).float().mean().item()) + acc_exps.append((logits_exp > 0).float().mean().item()) # update policy res = super().learn(batch, batch_size, repeat, **kwargs) - res["loss/disc"] = np.mean(losses) + res["loss/disc"] = losses + res["stats/acc_pi"] = acc_pis + res["stats/acc_exp"] = acc_exps return res diff --git a/tianshou/utils/net/continuous.py b/tianshou/utils/net/continuous.py index d68f3856f..936711d90 100644 --- a/tianshou/utils/net/continuous.py +++ b/tianshou/utils/net/continuous.py @@ -1,9 +1,10 @@ -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Any, Dict, Optional, Sequence, Tuple, Type, Union import numpy as np import torch from torch import nn +from tianshou.data import to_torch from tianshou.utils.net.common import MLP SIGMA_MIN = -20 @@ -471,3 +472,41 @@ def decode( # decode z with state! return self.max_action * \ torch.tanh(self.decoder(torch.cat([state, latent_z], -1))) + + +class GAILDiscriminator(nn.Module): + """Discriminator network used in GAIL policy. + + .. note:: + + Adapted from https://github.com/ku2482/gail-airl-ppo.pytorch/blob/master + /gail_airl_ppo/network/disc.py . + """ + + def __init__( + self, + state_shape: Union[int, Sequence[int]], + action_shape: Union[int, Sequence[int]], + hidden_sizes: Sequence[int] = (), + activation: Optional[Type[nn.Module]] = nn.Tanh, + device: Union[str, int, torch.device] = "cpu", + ) -> None: + super().__init__() + self.device = device + self.output_dim = 1 + state_dim = int(np.prod(state_shape)) + action_dim = int(np.prod(action_shape)) + self.net = MLP( + state_dim + action_dim, + 1, + hidden_sizes=hidden_sizes, + activation=activation, + device=self.device + ) + + def forward( + self, obs: torch.Tensor, act: torch.Tensor, **kwargs: Any + ) -> torch.Tensor: + obs = to_torch(obs, dtype=torch.float, device=self.device) + act = to_torch(act, dtype=torch.float, device=self.device) + return self.net(torch.cat([obs, act], dim=1)) diff --git a/tianshou/utils/net/discrete.py b/tianshou/utils/net/discrete.py index b89a7a48f..fc55baf00 100644 --- a/tianshou/utils/net/discrete.py +++ b/tianshou/utils/net/discrete.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Sequence, Tuple, Union +from typing import Any, Dict, Optional, Sequence, Tuple, Type, Union import numpy as np import torch @@ -449,7 +449,7 @@ def forward( return mse_loss, act_hat -class Discriminator(nn.Module): +class GAILDiscriminator(nn.Module): """Discriminator network used in GAIL policy. .. note:: @@ -463,6 +463,7 @@ def __init__( preprocess_net: nn.Module, action_shape: Union[int, Sequence[int]], hidden_sizes: Sequence[int] = (), + activation: Optional[Type[nn.Module]] = nn.ReLU, preprocess_net_output_dim: Optional[int] = None, device: Union[str, int, torch.device] = "cpu", ) -> None: @@ -473,7 +474,13 @@ def __init__( state_dim = getattr(preprocess_net, "output_dim", preprocess_net_output_dim) action_dim = int(np.prod(action_shape)) self.action_dim = action_dim - self.net = MLP(state_dim + action_dim, 1, hidden_sizes, device=self.device) + self.net = MLP( + state_dim + action_dim, + 1, + hidden_sizes=hidden_sizes, + activation=activation, + device=self.device + ) def forward( self, obs: torch.Tensor, act: torch.Tensor, **kwargs: Any From db69ddc1dc4e20f2ca986222ab3a71f5cfc4d47e Mon Sep 17 00:00:00 2001 From: Yi Su Date: Thu, 3 Mar 2022 05:40:42 +0800 Subject: [PATCH 08/18] add gail mujoco results --- examples/atari/README.md | 15 - examples/inverse/README.md | 27 ++ examples/{offline => inverse}/irl_gail.py | 0 .../results/gail/HalfCheetah-v2_rew.png | Bin 0 -> 182649 bytes .../inverse/results/gail/Hopper-v2_rew.png | Bin 0 -> 210188 bytes .../inverse/results/gail/Walker2d-v2_rew.png | Bin 0 -> 234675 bytes examples/offline/atari_gail.py | 287 ------------------ tianshou/utils/net/discrete.py | 43 --- 8 files changed, 27 insertions(+), 345 deletions(-) create mode 100644 examples/inverse/README.md rename examples/{offline => inverse}/irl_gail.py (100%) create mode 100644 examples/inverse/results/gail/HalfCheetah-v2_rew.png create mode 100644 examples/inverse/results/gail/Hopper-v2_rew.png create mode 100644 examples/inverse/results/gail/Walker2d-v2_rew.png delete mode 100644 examples/offline/atari_gail.py diff --git a/examples/atari/README.md b/examples/atari/README.md index 820d2f57a..4118fe8d2 100644 --- a/examples/atari/README.md +++ b/examples/atari/README.md @@ -122,18 +122,3 @@ One epoch here is equal to 100,000 env step, 100 epochs stand for 10M. | SeaquestNoFrameskip-v4 | 904 | ![](results/ppo/Seaquest_rew.png) | `python3 atari_ppo.py --task "SeaquestNoFrameskip-v4" --lr 2.5e-5` | | SpaceInvadersNoFrameskip-v4 | 843 | ![](results/ppo/SpaceInvaders_rew.png) | `python3 atari_ppo.py --task "SpaceInvadersNoFrameskip-v4"` | Note that CRR itself does not work well in Atari tasks but adding CQL loss/regularizer helps. - -# GAIL - -To running GAIL algorithm on Atari, you need to do the following things: - -- Train an expert, by using the command listed in the above QRDQN section; -- Generate buffer with noise: `python3 atari_qrdqn.py --task {your_task} --watch --resume-path log/{your_task}/qrdqn/policy.pth --eps-test 0.2 --buffer-size 1000000 --save-buffer-name expert.hdf5` (note that 1M Atari buffer cannot be saved as `.pkl` format because it is too large and will cause error); -- Train CQL: `python3 atari_cql.py --task {your_task} --load-buffer-name expert.hdf5`. - -We test our CQL implementation on two example tasks (different from author's version, we use v4 instead of v0; one epoch means 10k gradient step): - -| Task | Online QRDQN | Behavioral | GAIL | parameters | -| ---------------------- | ---------- | ---------- | --------------------------------- | ------------------------------------------------------------ | -| PongNoFrameskip-v4 | 20.5 | 6.8 | 19.5 (epoch 20) | `python3 atari_gail.py --task "PongNoFrameskip-v4" --load-buffer-name log/PongNoFrameskip-v4/qrdqn/expert.hdf5 --epoch 20` | -| BreakoutNoFrameskip-v4 | 394.3 | 46.9 | 248.3 (epoch 12) | `python3 atari_gail.py --task "BreakoutNoFrameskip-v4" --load-buffer-name log/BreakoutNoFrameskip-v4/qrdqn/expert.hdf5 --epoch 12` | diff --git a/examples/inverse/README.md b/examples/inverse/README.md new file mode 100644 index 000000000..d5aa559aa --- /dev/null +++ b/examples/inverse/README.md @@ -0,0 +1,27 @@ +# Inverse Reinforcement Learning + +In inverse reinforcement learning setting, the agent learns a policy from interaction with an environment without reward and a fixed dataset which is collected with an expert policy. + +## Continuous control + +Once the dataset is collected, it will not be changed during training. We use [d4rl](https://github.com/rail-berkeley/d4rl) datasets to train agent for continuous control. You can refer to [d4rl](https://github.com/rail-berkeley/d4rl) to see how to use d4rl datasets. + +We provide implementation of GAIL algorithm for continuous control. + +### Train + +You can parse d4rl datasets into a `ReplayBuffer` , and set it as the parameter `expert_buffer` of `GAILPolicy`. `irl_gail.py` is an example of inverse RL using the d4rl dataset. + +To train an agent with BCQ algorithm: + +```bash +python irl_gail.py --task HalfCheetah-v2 --expert-data-task halfcheetah-expert-v2 +``` + +## GAIL (single run) + +| task | best reward | reward curve | parameters | +| --------------------------- | ----------- | ------------------------------------- | ------------------------------------------------------------ | +| HalfCheetah-v2 | 4971.56 | ![](results/gail/HalfCheetah-v2_rew.png) | `python3 irl_gail.py --task "HalfCheetah-v2" --expert-data-task "halfcheetah-expert-v2"` | +| Hopper-v2 | 1933.84 | ![](results/gail/Hopper-v2_rew.png) | `python3 irl_gail.py --task "Hopper-v2" --expert-data-task "hopper-expert-v2"` | +| Walker2d-v2 | 2077.99 | ![](results/gail/Walker2d-v2_rew.png) | `python3 irl_gail.py --task "Walker2d-v2" --expert-data-task "walker2d-expert-v2"` | diff --git a/examples/offline/irl_gail.py b/examples/inverse/irl_gail.py similarity index 100% rename from examples/offline/irl_gail.py rename to examples/inverse/irl_gail.py diff --git a/examples/inverse/results/gail/HalfCheetah-v2_rew.png b/examples/inverse/results/gail/HalfCheetah-v2_rew.png new file mode 100644 index 0000000000000000000000000000000000000000..e7f897b50214cd8aac9eddd96b6d08b9bfbb1d01 GIT binary patch literal 182649 zcmeFZbySpH+c!Kj)F3rTDIr4$Dgx3CLyL4MC7>uNB`u9~Du^^vQi6bpbO|WkodP1= z-Mo9$>wfO_J>R!Mi7NbR%EB^u35e-C0mX z^09ACq778m4U-O+f%!<+TeWDwRsG$P(OZkw1f{X`o{8f2 zYiHg@7tTw_#@r}=UB6P6rJdnZ2|w9~YjS1l_q$_(7yNMgspkG_QS~|Ks`BWP#?c??yz1clA%xi0Px{ zx1H6`yLwNoTe{To4+{N!ZbV8mUfT}fsd_EMaVIDT;UvfT$nf)dTDdXhCnatHdm78g&ZP6ctNmiYQ#$`jtR#B2DJF3X_@+dvFt@7Z zM^#=9@m0L{U6D_R%})j1GfeqM$Usl-?)RUEzeY_ngZGe>3g(&DZTPIu^7CU8zZAq$Ex!3(O_P@c zV!Fe%^c2E={IpWu!UwuNwRbodSh0k?DL2Kcwx^~VoVTY`I?r8Cz4j9P33z(U!wl;f z)SuKvUb=%%Cxd7RBrM~&;&nXTvq6a&K-F3Aj=gnYtn7TpltGyOK65dNl`^91^(o@V zwQj=0hW<~@CpF|{tg}0fK2JTH3FDq(j9=paex;j;ipckkNNRL?xUM$Y9MQOE+f-#% zY6(MSr0N#J7BO8CQ zN{BNCV#y&1LIduSe|BfXc=g%-77nS8S|NTOHrWG=Vf+#wy$7_Wkbuv)HrV$uV?I0B zP^`Tn+_-4(_^Xc;4 zw#AeaL4)bc-60DO#We0gdk-Lejrh0l+AlwPNPUxBzNzTeWnL1N&#i^#d4x}%cOBjs z40m`)`;@-3F0PDg{4=pLXJ-@hG}q=AXRI2Uf+po@&CLL3zO&aThFC-*yato)AvayZ ztvD5hYYY}wsga5PCOJME;kOwn5vzfd8#gnF?E?D(LxPqAD}$PXEE}5|uUrgzEM-f7 z_{vaDSPhyJTO3{d%0ATIuV4HqB{k<^{&XgBW<)++Pi(Gg<_F2|*YiDGpip=_$7QsSuSl>Wlp8_xp8{U0>29B>_|cKLUCcUi+^K4xI`^qOH6UoEB_l^ewv#S!#b zlbG+t+R@&*<05i;;q*1RcAHGddmjDV+qnu)Y6q{rXR|CSdCx1GkR_sCsdk*Qr?`5B z#=jGuq?P2B^hHp}zH?c4=<4F=lK;{L`}t+7L6Mx}SZ|KJOVrrO94|ijJ&o^4mF@r7 z_kHQd!t15g-X{aggF^%R-O4@5X$Co-QneEhu0ozE%Gt{|d4^4F8m9UXeRL~y^>iWr z{RHKlv(?io4M@qS_>*qzw;IRAt@ibL^lxK}y3ZsBpZap)R3_UH8_KD_dP zjn}})qx}R+J-Nkrj-xFuD|UjMiJX}HLWm;yd-CdN$LN-5J60#%$uoJ5!ZP4lvBoI z^W!IkQaI6)`3-^rZQAPH;H|$wOg3bsbNxLmumNZ zTap4J#x$m!jexDdUzbRkw#MyXWZFPaL~r|kY1t%Ast~OV%aNF%_or9sMbgX10jl4|jz;+qv_ zq=GkW`&(7_4J`~AIS4)=qp2!6_dF8y#^07nZZ;m29kd=~;WAvzWKL9k%&2sGJmP@u zI=`7&@5JNd$I`ZT!S`B+qvE6XbLetd)W&3qXhk0FeBu7i{Z!|X;B}F!?oV90V!JtV zo9sh(sda_-zVcT-RsWhqYb`8leG-AIL*h)bXM*5V<8plLHuq)g-7}H6@%DG^R-ys5 z{^x24My;z0q(^v~@MNPyllRRoZ&s75IV;skuiYSV2ojjNu(_NVm`EV7ZuiKp^vB+h zjJeg$q2{h;v1>iAXYVK}awqT`87B10>D;tTo)t3NeQTv1WsyrA@!}O9`NHQHN92Ch zGt6Ha>b>e)>xo{eGp#45aY=J#7`^-6FRh+T=0|qQNs#=>0W!-r}b=)LYcs6>=EDej+?Gzmc=$GFW+CM-y1A2%)g<%JG6wG+ZFRH%2G>GJ6)?!|BBvZ*`Uj6(Whh_v&s4Jm!5ynz_@wQ-|8o6CDSyrzvl*MCa2E@2?>yu5!e`ea{-p?06!%}P88zcL`!WRVNzb1QKmyDln7`L3fuHCcpMEXN;!2d3Szs@PJe^$d$DH#8J#_#~YgNUn3 z%E^JB>W^%VjI8WTt?iGD9Ug)UxHh-7?H~~HtLVQ_IW@)&@cbb&4J~^uMFqh})|PA! zA6pw5u{m4Xpq~Q~b`}I5Esg9SBAqQQtn37xMd*L75Cosmm)Yr&KbP1)5uw*oR6$Bw z+ZrKx*>17jq8Gh@L?VT49~%p*Ny+@a9sEy(-qhaSMv$G|$;pY$iHptJ)`XoyKtO=~ z7AHF=Co5RNYWK{_{-HCgl^w&chy3##DI>c_wq`c=X4Y0n^m88?T07W_(9@$|^q)V! z=rnRR`=2*i+5O%Y*dROlH|!j2x7h#lY;dbE`l_IcnX{3Fwv?GA;2C&_D8B%=@Xz)C z>o@=N#{arg>woU#Ck@Y+gSJhCdgWbCc zU;2c+q3`-7e0LO4=;XHgoPhK->FwLMx7N67=F1W+Mq88|9rv9?X@~oaI}_Ju^@UB& zS)#D+&ZXdACV*d>zP-qro{5^}QSL;hBUF7|VXsvYkOggDY-1jGFEU$M+`~`cE zXW)!xzTv3qf}P*M6C(bLod@1Pua!S>y66??N!tzk^XYmw5+fngrUt}bHpP!j$^R^v zUatUIi7Vv9FhsZ`{@?7xMh(dZ4Se_RhMc_otDT)47u9%MrD{J`>g>F{sP^{uI-UZ( z%6pugoJ*_*$NbPhmmL1TkqHzHG#46}mPUVB!1itV!eUl3+j4$oetRs3>({PrP>x97WJs`n>#JNdukTbG_6j+i93GzO7$?DglUXDBC& zo^j|sTm3XN3n++wA$F!!bGoP7($W&KHc{EuohoV67)Y#&LF|$&AH(*#dT+FCymGM8 zw1-~%gP~vh8xdV>Lh1#B^tW%NidUC5!24PRm-85^U3MeYl|}AyLpa?IZ0yz&1?(R0 zPdM$jefT@$5dpi0f$%)p3|J{_Ae;*hpmB^9-XFJ9)6rQfooXNwi>j}e_^79GBUDtcDyccWtA_GH|_Hzzg)(+BmOE656`sp$?g>q zm*(^rqDwKGge#TnHB9Y=+nqvf;3(J(JPtFQZtuAG;1KbS$w%Le=t-Atm#Me6r`{3O!`HWh3Y>%n-`n`yL)@;fDjJUy|c!%Sq2}dbr@7qM{;&W|XX{ z^IV+CTg$N$V-0NUx4uILo_cJRE1W$&7wzqRv);J05^t z!KLSe>#TP3o$tBo8-UC(>(uHmcM7fVDDkAi30M|vXCq`=J&sqI$E<6s7m7)N5R_de zlX$VXTdjJ{UYCL$lDaI|(Nr>K#d3PQAG14TVBk1r$v5KsJuIW5LTB%rrzc5N_jb6z z+$Di2FO0Y9SxQ^r@y!@S!VhGUx*t`TjtRf0-P_>%vvvaZ_~I4oKqO>&x+Cx%Cme=0 zmJ@d4xJ-i2dhEEfyDOd61MRqHQyyaIS(Qy=laP1sj#*VI=9kS};NtNQXNri3NUiKh z`;19IXqqIL=A*>&v{PW-IiN!{X-gt&qU>NMw;D7-XG~ce#WiseO(M$)h1kxzma*93Cgm0*AA&7?gT?SnZGSlF}q8UznLq}Gl+;eSRGJ2V07 zgV?&}FjIR7=^s9>;@X4w>M*2d;KRPo)ot?y>~ZeJz3F6xj=)a0-8LOkny49o@RQNF&w%y33@Qs1Yz(sDq%`i+z9dR7JvfV*lBVc<}sWiVSC5fi#RXWZUhA*Yi`G_X!X%5q?ZjkY`Z zD3Zw2flb^3jx(1*Qbz45OP763Rvygk4>FNug(pWK(X*q>Ne}tAOi1z}TTvOyU;2-z< z5MD~j9b?(GfgVbI*Hxa&A}+=DtD^&$c!Z({58^l;`Fo~e6OiB3dLWZ<05tu_=#v7T zbe?hGuFnK2wl~?%us< z)079V6nS&{mNMjHTJ@YJu9_sfm(2e7Ax&*i^f+jxeEv;jbTrfA;$m;6A{E!EpVGr= z*0vaSAT%S=MIS$QJfnQxO-u*;{YsLdC@oMrf%U5G_9JNuJ;tUht`?1(Ms3k?#4N=k z6~f!?T)iJnl+P10F_Bfb{K+d;OU4wgH54F=)B0?(TUZLJ3>}yIuVCO1;*0q?{27>n z9m7d%=pmbZ%3>{3pM5%t5i~-M+zec%DdIl(85tQ&*|(l;Jw~Ib1r2gwH3%yGYVY9? zP%;A8_kN_i(90j&oE;5(s?N`jjZ2jzQ`NQm^=l62sJK?89UMO8)t()3T8;Aq)pLKm zyEMS*dFIw*#!+*;CR96oy8n$`XtnrN?L-trN?O`)foiPYu578})`?r|-L zP_Rdjd^Bl$r7XHQBnPR_dVuve$CM3I!bBDufkUoRh`l8q91@}^$#W?Mib*pe-T<#K zEqgH69%uX({*hsYoq%{8;0@z0qZmD=h5#ZWp`)R9;UzX-$TVZaI39hCwUJDEB_Whw zxhBZv=kKK8b4l=qPwCG3w{HqBQpbMV4(9*`s@kI^mLAmCJnfGsr=K+KgYlBK(zCOh zse7Ny&dtTw1*{D}44|2vaQep61V>%MKKq;aBt;);0z-^S&=5Za{$gt`!TLzt3mC|s zsxc!WiAPBqSh8SGE-2`$7k?kzOqRR{xf*;{ID@KOJ`AU>Y**}-(c zkN79~9O+8VO?;EikIFad@r=eSOJ4)WsJ~P7u))8j*sQNgJ)YXD8zyM@1sK{N08nmW zkS7T`aDIjp+%<(u08ruYUHdnIT()MNIV5c|mv_1jptWnY=Vz5W3q6uR<`Y!t_p(gu zreh2|I6QgBEV#_R)wr*)Uwro=FHZq&m={Ms<>rB??q0rl#@|GOj32!|LIWSm%uoIe zVJKt?h_4KnKq2z8&L<|F@ZwJuowVfQWXU#VZ94~XUYQ|#paZuNVxde}vc z>TldEyiwc>B1PR@&iHq$?}618A~^(qp%1{C%F47RU6xr}Iy;qyT|W%izz`*mb143}o4^9ET}UGuGqTql zq~Tu?)(QvS>tNE7E58P)Uz~tlrbTg&jP1Z(O`i0}D@vC66VI{mW@8_PM*vOIVJn}F z9OhvRGN*?`NF}wr|1&m!2phyeQ0P0SjsvvEY1jyl<21T6oH3yfHg94C0`a0Rm5hO5`Z48iVFyOKbPCl!P|XAzhJC+bm!<=g>FH z(usC67iAaPNi=qS1{}<65c8W``(>Z8sbU=yffFwRLhvk0+HJ`mNhGrK(o9}QC*GoJ z^DC$Qf(jZPo^HoKnkTCOiHjd;A#9O&Y-LEpcf>UA-vRQE@;v!Y+gt7GRDQZQ_5vM1(KY+)G9MX4HJUjSKgidv27}R>qRkgP(taERU@&c#GCFt*oEZzb~ z)(reFYXG%vVB8ktN$eT*xJ?5=Z;pDFV4q!fkiO>FT^$|s(Pwo~oJ%nvx3gickbJf` zW^FsDZPG$v;P&w3lL3enT}F%(EKlq{(QMCkaEW;WK>gFMD+&A{?QsFd*R>oCxs%=Z zHoHvz!ige3VC7*U9Y>5Pv}@eTf!Z8M9QoB^Q|^RVS@*Cnu$RwX%mKM2;Ee3<%W`!~ zOX4ew>_*wI4EG34xS2ngX(UnRro);@{lOiYVi3JdaeDNl)WiF{9*;eCd$-N~6n3A2 zK$_ZV{Hd|XsKL>f3MB^B$_bZzY?`W^t2+k(n614i0ht|~9!E`0%{FwthHt|D6)wWp zpdlemV&4oQ)irQDC|w&`dpMWSA;LY-NqTZ#i$<^{aIIp~2vglEX$(9f(Swwgisk$* zkd~^wNFV%u~YQe;$yz}Ye^!JeqYZ1LXd`G!c4{4^{}p7w$f`?^5IO!%pz;QPIg91)# z_euDhTU$BB1CpF3wLuitehuptvvTyickcwJL;Y8(9G9}f^KAf!;y0SY_$DuTG=~nS zK;e2WyGvICXg$hKy3S8t(%=TB9ogy`A2th!(plEw-(a8i3lW3 zb?to#X?7(Wh?X*;cmx62StV_{RS;jRC#F&$@-`FWB@BDpldmWA41~+G{p!j*;*sd! zI;x|kg=0T8-E*NUh^-W4Gm}(rv_t;}x!u0)U51U^kvC0}kZ6FYXDS3i`WG+RCwmB- z0nc^VRQ2F_c*-q+hPg(cd)_Ked-xof7V*Udi(k5>Da>OC)>nds;2zc`q`I( z=&PTXd6*Tb%aT%1v;maXUQF$xQrqnb(qio;YeWWBk6c$i$Tl`(DH5QvU%5E$&8>{(@Q!@g;>k^N76mEdCgKKidImj84B^^m-?POP*UVF6I zn_)~!NE0R*lPVbyzsS8a$y1dU;(v*oF_l+PM9$ep9OEVkCHOw$T36tyXM5KczZiH5 z@-%4Vo#g_BS9gN=s%m0R-G`!bwIt!P>f_JPvCfKVrkO#GTYn1#MqF4j{1=x02slv} z;rzs2TIIGGrZ&LBdAtO%qTI7*<%j7E7Zht+Bboho>aJ=MmV2n z7asNcb)~_UJN-3*qg5d3xz)yA+r0Iot%IAg zBL6hM&p~*zk;EgyABNEHmXIy)&Jp_SjK^&^?=l>u{RMw2>#82NpnP%2=~hKbN}U^S zJ_FIh#VcOJARuio#yz@PrDUSwx(S2!WaekPTS<-)gh5h2;}AUIF0lY%tZ^~Uy6~x* z=Td);aU&MZrg#h?jbJl~l+h)Pv_2JIb|5&^K=X`mygWbKuLWpee`ChDIu+u}XDE>* zq3URgnu(NW)KN$@CsgaCucv(3n*cLEts-X#qr;BjY>f1 z<1ZZlG5qD>!QJ4tG_kKJl*B)P7U!Vv!A_S zxepwf^>gCU7{TBM0B4v$DkS{!s)Qtv7ujS{_x968Qkv;DkW*oKA$pXJE3i=g&Imv@ z0cjsH!3rrm882samFVF6ix{hkip$?No;+K|L<-}I{@L8jUIgZ51|%N&)XDDKb?^WL z*bN^!f*}MA>&DCNI_&WUStVC9cwmT0(o2++-NZyXMW&Y|d|Fw#IJ8wj2Z<&z~XJ@z5s4J^C=65^|r@d0Ez6A)oa zW5kx`pgu_4{uc>;Ifia?XyBu8fv*bZ2)23m!z-48A)x-Fjs_4)M6}Qubb|H1+vb;| z9vP~*9ujlU7WXC^r`YS4ER?*T{!@NJr^(wB_>?)hxEie6@(MtiYP8hadeGq=PEtjB zRn>i~ibW<8K{)%A;zSA^gaE@igu*M(I19L84Qgp=X=9h5@$vfwT8$=;pn)L!sig0D z>L38BXYC@=q=c3lckc%IbB~r-Qax=>pnnTQ>t`PO0{;42kddUI#zp8PV!z946P zdmLBC?oOQ?a}uMV#Pjgb6uK~wEIU6@O_zzgPQ~4AGoR#ibWByGTlm1+^wtb8DV*L! zwJPsl15+)i_8c3_zTY2M>i7uCsvF$5ZaJ=K9iD7={QxmkG^nGx1}J>nSsfd~WVAO~ zC1&K-NS(f**DsUR3g_!5M!M{;S+*{pJ`wt4z-^QIIVgm!qFLe7hiQMRcrX8)Ydb@4 zT-UCnON-hcjpMGXSDj8~VN=<=M6u~rG^3NEl)gRE%L5TY_VI*z+>M;h8xKBq8vN0} zL101b4@4jz+zxF=0HO)g05$-f-sLh7&C19K3dc_|@K^>A_iA}Kd=q4`T0!iGF73_n z?S4w}hjCL+u^t&6`{I(c$LxVXl*{1lavlzzdG>zl%XN6K50EpXC z1FkIH9hl1UziO|8MPn%;OW)1j?a+v!w}a;~F?h>2d^`n8L_mzTj+v(-u6v{e4rs_a z=SFi?T$0Acn$lw!BoW^NKNC8+0UTibaV!q>K_fC2vN#c&*S-;R zHQ;^eqz)(_QIU}MbU*i3;YOxfeez)d@F4~yz}m*YgN(M+alfv;IUbMZ@#ts5b?W!( zKJ*p9Jbk`@j4m_x_NvvMe^MeSaGnkVCp}6lOa0W>v`{k1L5JwH-zLRAes&rjDq6|40HP9u-eM)YAIa4&7ZmtF4?*-yNlX7F`6kbsgKI0(m4v#gDGOM0F& zLCmwDYQtraxGaY*;n2=?uhM!Ph_jVV!AM};>_ZOPo~OGV0)o%QSIrne?U%2|j1@qv zK5pj_Y$K3%uK@hMl zq#09^_AHwxFK+GhDIb4+1^DYyGsyFFM(Www;rwY$Uj)*AH1aZ+aqJWU6#m*!{Lk^5 zY|332qIDNIx`ZAg`(occpmW6i5yZj4!R;|?moHy-2p=kylq8^p4};=$2OO*Pn@8m# zrw3@6hyo{;!;uP_80w=nq_zKSFQH9XI;;?4d7LCB1~D-)^Wg`x%@OE?Ke{R{ zE&KfTS7}*yEl?7i{-p5n3xtzkHZlRHOz% z6SGY9ZelX~BMchL=YNsX0A~|vfSd*@ausBz3h&!|#!^#NeVF;8dy&%qf?rV)XG?2q zBtW6}{RwH}PpX?}=>MEqM0$A<-)@HZmYjej&TDWEt-wzp(OBv7YAWzywnvLuZA=Lk z68nGkFw_NHMhghN-%+5h^dDsjpgnIH(BQ=lYkc?@2ZV{RG4O);9=l87{SgnLGodNO zAoUjpd3x!OX;F0B)Hnrba`K??qk7GW#43r5r0JqC1|(m zfg=L`YLj*DV3TY6#H@c#z5L7W|JS4am)-x1E&oWjRa_m&tPQ=rFxQpLj!w|L>ioe7 z)KGQ`Z7hIsj>Fzv8BxE0hc{tH%_#-An0yVtY?diGu?cwRZ<~fZqcx2mB*iaR%q3G(oDOltyt3SAHgF7JUhgu$pj0URhM^+jjk zpdTM@{m7KYx#Emu_n=;)ihqt$$K3I(ZHs1o1(4nC`Oc&SHD5afs*Voo{@^AnE7|E* zF~C2m(LJP?2-Dy?(E9dRX&F>W)s^O{!8UHA&QAwC19ra|NGK3G;46x!1i^&y_=to1 z5H=cKCJ$wW{Eg|lrJ(NQC8yL1khGwr&t`kT!VQ1SFvhe1ZKQ>)3a9X4vZ;(0dIsjQ zbtMpXcMCjqF_ySeQJqBgJ6)HO{ z9PmW`Zd`7tw->kqt5?N`%3?Y%e!nf-jgJYtu+htmr2c}i0Vc|qfkvDQ0Y_K6Ae7R8 zen4Y@pid?efdagQn39Tvox>1>4%mtxm{E}WH>^Ai4|KtOmJgR?>+%70D@STVuo5ii z#YeiaN5qAo5>}tj{)yp%I#{HrwBuJp2~R!Vg|s@2!-A z3&$h{B{4*}A)KGSPM%g)@VPM(q2d)I3Z1^@@5KQ2K)nc$_x{QY z92#f>deM>^A8Zdzopn1IqVRg%$HL!~L*d6??|+7(IbaV{L?Kf^ebUQ@RI-j3s(-}a zOU#V{I9#=Z3zy}KyRUm zdsmS2Sr0|q@*kEtLoWCzr_Kg##=QiQ(o zv;0Ll5DmV$3xwdcHYp^gFc=gF3#M=SC0ICoqqiv{JmoUr@KlB}ia(fcDi8vN|Kc#{ zM3#bZ2P3Cmf+Ip7u{fyLb=CjoFkt!;e!QVFI7bm(a~U9tQ21}&0qTB=A|7~p$*o<6 z2by5fPkGG*ixMjO!@S%9b*n%WfxK?qwD?6iIBg)WHsH0s7)dA+1pFEhyWf%+BN}dx z*@5OTyZdi}1v31T!w|F#r>J5=12CP*9l;S_fI-10!e1N)Ojj}KCUpgiRz}pgfG7fP z{Nf#;?x!f~0!b?I%b|rVhd zswj$P65Kor4!zj+rU`co9gfFynRrT$>iPkFr#7G<(IFg!2-)B#b10(lk28~a5$ z5Dg%&)Zn$#DCTZVpx8j{en}!QAHNj)TAh6IZ-E89);~Gy1!VZQV!!dD0Z05C=+zrP zj9(lEOb2@P+kzNDF*lM@jc8Ho*M z!V_5H;^GM2JUX5N>1cZNgc_(#%EhqhF|x9Dj9$e>X{BcvV&ZTX{ebZ!iUZJ; zgzh%7+^7eQClSO9GThGL?|ryTI-@{pQU%>z*V5KT;+`^QnjSX$9FOg#xDU2=`AsOC zB~K$5l?&{oDVZ_bb1xqB`_Z)NNP`IBeSNz$qL2v?K*>smCII5=V94g>br*o+_giWL zut9W$_-hAfDy`jaB#U|0gi;Hrfo5w4!CuMY3@|iUA$EHi*Pt}g;#slIV6;eQl?mjZZ=kEK=tA_azIs*}I&Th|xXy!I!m(+? zMUPDX8LLw5!mW4(kx~(8r9*gvFk^EaL&pmxk7mn7kyd(eP5-E|mdF$eV?}30e_k^{ zuiKqpf)|67%XIVwwJY8SAPS{6@XQZK$%D|ZSfl&%7AT|fq>Bk})M0d(C!NWTtc-j# z>PeF>G9S3H*pnWx0$K`fyTx&Mbd=R$2y~GLRHVew#hvV3_suU9b1vu#y+!GS-87V5p&Jyn|mq#WlpI~#e~7BVp7nr z!994XdJxRDSi>$^f(fLR9b+11NY70+C7HVGfHO)CIAP)`BOur`7jlhKA0=@-?Nx=CTKv2u2CoYMAGlHlckuc_m9_ax!lydv^Nj-g! zL!(0=ROsU6tI2BDww$atH$aPKSef@gu4YK|;#g^cLL8?wm|8mhj&6O!rxm@Qshm7H zbsND-fO0$h5yNTTf6XoomxSqH=*7SVXrPvy8WHS07}-wS?I@>*eLyELZt~>dc@I*6 zPs<5*(wK%IKBkY3@ApG3gV@(p&pO{9l?Svm{L{5vy6y^KIZE^SG$oOPKz9sTmr(BL zwjfY~`>eq80gNohZ8QMxF)NE6zKx2CY6g9K*T8(-hOP_fZf*f>ye#PXipM4<$H&#L zL1!$AobCQK_UfI;(%eoHFeZkc5xJ>dB;|2-XaY(=$_f*do&0YIW(@xME&$NILPc$s z*6Ei}<^z!;N#!Si;sEL6AHL3w>un5DGg+6l5`(>bz)?IcR5!f{>j1HQ5B^Oh={gQ@ zP~i^n4)K&YunRtt&^}@t1z-}J1byd_-3s8h9hmaIeIP%{BIwr$>gv=z<1;gc0*h~7 zJqF!dOg1lLVobtmpEeRR-Q=(uFALnRCLj82AiFw@QP_m7V3GE=;aLQ0Eq$|7wT>&gPmL4r~)h z2{ElxelRKzs3QrB+>$s_6znDFg^;E+@*bM!SLHlPb{AH7cpkLZr|swo>V`q;tyWJM#K!OQCr>}b8dHiXFY(B;xfEOBKUyG{4UewM;sXBC5pGag2fwC+#NH{ z!^@7E=pgAXYyeZfCeQH=JWoI2O6{yojD=2ht0+?F*+(qIyy%IZQ}QN$9}aq&q(Eo5 za2$Fl^TQYN9;Kj)7Z528a!2}?+~6D{Gg-ID=l3aZLtc;olFx@{d zk-~z=y_(P0Fme|96>p5?_q{tCVTo>y?{?5enC75953S#B+J6&~OhnD{WLx*j|W8U-xo$vCXM=$RA%>Fk|zCPt~FbB)|pw6q#i6jIf z1*WDhhs{K(D@A}#WmX&AG_uDRLQQY1NFq5fV_^&&M?v@KRgxhM9)rvc9+(#;QM8vF z;8v%rPiZ#8z?oc>_qu-TVs5P|D{LAT#Em7>v2^iz#oXhTQ7fVrp@qhcvBC=d!$sJ2 z&*f)kuJxq3L#bP{6xnQNdcj#QBI1K>TSk^`T4=bdzVLN?viQb%jLchB(p+ ztt(U_6f#IAw64_Oz6x)YMn=F8xmOa|$Y`tS%;H9Rt+w>$KJs+mu}Zr3B*u+xwX;U9 zcSyUIhpmMtQX^$2JS1al`9=Fg28Cmk<9r9<`>an3R?o<-h-raDRRUD-bR9=`K?_>I+Um>$?CGnQ^1Bo|xNTGpiZ)c$|Dp zrssm6io&dbe_FZc*-eQV4=N97<%Z$Owcv{yM$nTHxADOIZg$@&0EOM5`5Rz-`Of~@ zM8{JC9BR9E4r5>c2nN}~>)XJRw4vPqx`}+5dfSovD>vTFsCY^+_v+o2YPpRx5- z&$qpCuDlFkuh9aHKmzEo`_t18T67R%Jn1DAFni!@HokY>B@TQI^4CAjUZX7VtD8^f z-@zO204>X7%)tSk>$WdMO3}!V)PPUso#xpb#^P;!L3*E6qX9{-w=)ex?MA(}TNx9F zrKO|IBqi%%N4m->6|-I4T^oyP>NPiJ(gc&_&k8>fAIDdI{%B+Og`XB$icX5>ikT9? zI?x(;{{tq!K?6ZnqHVC)d+DKkJtNQ-#^ihp-R;O@u92jAN#AAhs=AC1Lhz_QGr{VH z*vW9Q|ex~_u}C4d^^uM66f!jqQLBm)SGEJiqg zj+%TsKPwHT;!~1&aYfIXMw|#_pKr(tV9ubwW#!Brg@-T=0?g1!lhp>-Kpy&FAIgeK zxe8eI>l({TI5e>6ncdryFyGFL<63b=52S&TwN=lYyjjPk5>}oLd#JoFw() zYOx<4MJsyp5j|+hYdIRIpLj4dC`ouIAif*4ZphkuDBm>J?=P{F5sJTLACk+O3Q-X^w@1(AY~ySO|8 z4vJUFNb@N1IvRjV%lasaF};ysyU(*#v7vmJ_lZy%r#qiW&yUBmwcyBfs`;y4yFz^>tBs2Ul zD^R&uaXjx91a1nPd#uWH+M$39Fr#}1fC8f=he7Zyd8SD5^R8;}If)>K}flqJB>&E4YMN`BkWu?%+rl zA*MOrLe~o=cU8TLqCGlKp>$Vev*Uuc)782Bjho~5IXY$QSiycKmWXtGP<8+sachQd zlc<5SlW?R8lc@ud+7u4dyAf0zlnwW3-=kFud?(M#VZhXjmFK4J=6Ge)Ue`QNpI&!e z%MT>Hvp%f-?p$kaz3uXba?8Xyzo*bWmM4=Q;`hWn?E06AO2#C&0>Z*vLX*HZT;#bp zXKJ`@m>;m2QN>qe%!Y?B?0urzx0YM)=P@);=W`~vdPh1>H-)aQTJ7Pzmq^J0uZ_R) zU@>YbJPG=moxI zW=SkYzGr}0c1OYS{8W;G-QsHQmH6p(<+jdOjwI_Zi`y+c4cau#+pSkMPxsV!-t~^% zJ`Qraqw{7W!Z_U^u{wxe<#1P4O^0RSrI2aa?21EK)TF@O8?{GuwXYMsC9gVtBR+Gq zy9dY)C!eREdIQK_+XY{nnj(NEJN-*JE_zDwVP&{K0}>N?K!*YoP|l05hEHXeLAXf- zzLJO$l=uQ0*Ti&^M`8Kc$U9cGDuW)|LEi)x{b({a@~Ez|3TXzjiHREC@vfrFy1f_f zb84qsk!GN88C~_v!4zPRS~;bWTTO9oo~{Wdb^hAzrQk>sAgi7sI{h&_O!b2U8v($#+$dG8~kT zJg^bBZO{RzGfwYG6gmoMJ%>YHPNAit3d79f4qn+MuWaXYGVR^wUE)(Un;p!S@rv3J z&Ar}JwX<&lC*Lgx$8KNf3Kjk)cIiQez0yMIcJ0Y=={A?^+O@cZ{fGhOiMgRn??A`z z)&!PWhsRptbtGD+jjTm;@8_!(BX;bs3$Id|eLQ3GT-DXR&LS!PCVuPgnE+OkQgr+K zlKs+ziu#W>>KXh*P(BcaIDRxRC4(uVt+K}TPI(ZbB5)_%xkbNVU4qj8mJqOqdM7%* zmosQcwn+=I5}O8U!01L{i*fztlnx}#*dd5i&(3xsWO~4(;PKuHGZq8W(bzx+OU6R5 zDm@R+Sxregdqk+B>+zVlru}Hh%<@+$QfncXcf}(QK6W>gq`%Fy4fD)`O}^AgRjuZfS;GrVEI_cdm{Ep8g2&_ z->A&A1=O57J0kVFC%>Y`VAXb-=S{n|8a117tTSE~c)!NI+nb?%8An|P?x8ylgE5YR5^@Rnp2_)n zr_=J+%LP_UN))Ng#1`4)+FD-l7knW2|FD>drvIiE?5X&B`RZGs3(CFrSNS5-(0l3$ zpD%d@z7$RB=cVT`HY_C2IF&o=i5GkDp5cT=uOtx#4rft4sWrHH^&``1wHPZa_th^q z<*bJm*WEkr=U!)d{aUa{%G3pB-6Ccm`q;9zbRXx;qp`9_>~fC z(lX#uXmF|M=Pn2AIQ?Qp6xAJc&&Kusk#Tc1bmr!I0Ca_VrAHgu0f7=J5r}$sQF+ zclpC*yNs3c^Mio%qqy3^Z&s}rfY(-?p~f6Xd+mvuR3kXIG+?xP6hHHS=>VYBPbgVe z`2t$+rTJ`0uHOVsmUPX{O5{2^KlSs{jj213v`*{JtStFy@HGkvofAu|my^OJvNATJ zO}(+!wbGLZ?yA}Uz*W=JnN5zb+$WCwxF2y=lCM#99-QaG$er~0oZal1Wya~VQdwWd zsz+{_FHsBgSqJ_f-CidrPukPPJh$1`dWXb~y6`$u~`t_ShW@!Wf{n0zm_dM78 zKA8?t^WFnKj$uoO2%=>(Zi#aD-?&m6dY&dr& zy_pZj!rlX=k3}>Q+kp0|ho#2>gLK`o!0LYW<$ps8wh);{RUSkNtXM=EO_wEe7{GM@ zE1vu(#3Th@-9yi*tmD4U;iK0{yHoQFKQb` zVHg^Lp-UNtZd4QmgrU1rT98r+>F(~35JWna5a~_{MOsoC>5y)|J-_F9zw@5+_xVe% zVJ}MpM82zM}Dz_t?UIYV@>Qk=Z<{tm|~VP2Z?mBC=eF-3%|F_O4OlQR7)L zXh2|m({^X5V5uBEW#E44jCvqoSIse73w{Z?n>oC7T}9kfPG#2Tfy_Nn{&uIahTOa~ zcSTQeVRkyVr2Ms2k0zSiFQs@fuN=ROzppqP;uQ1@d}a4%VZqKi#A+$jSuoW93iUFg z3Pb}!QT{l;>uWw>UlRC@`eX95Q6OZ;+3r~h2rZMe{`3!mk9d7FD#seu@=3tA{`)8m zDS|;f;m4D@Ol>6xLXe72HM_`uv1=v2CU(vOHzST@ljilT^qN7tM8DPPTUT@7q?;5t zM2$AG3YYNjz2vWcKmYElmhVl1pTa4jSD5dLUv%4`!fTxOZ4dAcL@22j*#%vzUfI?7 zd_&U_d#3oCD}6n+e`iD$Pevxx@j9pmY*0C7Ckq6Z1yBZoE>wfk*(hYG4}Z&21pZy! z(cR;C0v4AqBj%La&k1h%>aM{a*AvvDXl{NQu~eYGh5G0MbD7$ot--eH%gx?n`WJrY zt$&`?$>|##_?c+xaI{?`71ot*uEvj_j_x&2gBRy^-{<6uHs#F&8u8`s@DVyZnUB05 zp1XZ1>NaWOq~uK;_3A8a>FXmZI1iDP_Uwpo({o>@{ zC(b6gCGlTc!R>uBBNGF#K&jT<*Pmj~ttz^h-G&|fnp%^qtoGj@UZ`G%jS0IFhVa=D zq8xZ`d2nGHTchK9t<*2aMC%V6npbx`E5VLA;=i905&wOAP~ewj(}4NcN>(h0t7kGM zp>)CbFzWA~-7f46q&eAAly}hf`gEm2f%+Ee zBM!W7MzpV6OWOs@LAr=~m+_F}SR=&g6RW&J?;0{A7~W$oo%5cBF0xGnsx(`4(&AqEEJ5UHY?f-o!=<>hNuu_0Kh{c8flpBQIt*A;)Wfuc$2DASeBy zt$db+Dlg0POC!fJ%fN#Ba>dPEo4_|KF?Onl50^jwwbM#1=MVUJ;ApGbqW|dWoQSt> zrVB{E#8+Yj1fvRKa~h+*Bh)}>Ds`X4l_?LLtyeqRd+ge%6R4o-l+_Qia9OW*U+i&e zqfQ{l^Fbe$&!}&qJ|c(e3!{BK&4?>lJ&i(GJDuHUc!m)svx8ri)0R-t_F2B~-48G9$JNO=Rv22=$*b#mwx)4$A$;o!!XoDV#iZ+9YffbkigF%(27ov?rkftH z2~|PC?6`VV3gUkQZ2$|xX)H)o!1_8}sPDHSxsuJ^5F*0yz)XEU| z2W?r(kOe7){Y?HJ5>KMn)>@{=eof4r0QA6n z9{&n2&UN@%Gg`P^2)lC5J3Oi@INsw#oYo1SQONHPKQl8i2dM9YiEN{uZ<#;Sld;7i zGu-EGJ+s>Ba*M|80OLa&t=pGHiaHA|W*WL{%XRt3pV}maZJv3I`YzKYME~^Mw%>VC z-?-t%-e9Z2w)1UnJb@fB#r2sKsM(sbG?T>}knOa+9Oir`x7jt~)7N&;-ydFC>IT|z26D83mF2UvPf;?b_DJ#%sACpbf|P=&xqg`e`0Ram@Ig;%@C{ut3qDqX( z-EzoRgU>AYY6>6*D*S30G=x4|&fsN++|uTlxwn9ywl2SI^+x60d}7nkNDi+%(a}?j z3C2mqo~w%US#H~?ShmW;M3}EnV0_#fw z>qCtq3iypW(n7xsnSY^<^c=e3g8&Lt)Q|o^8s>n6&nVJkr%RK9E}aMvh}j^>iWnA) zkq9h!X$o)h@&|K9sc)Y9Tot6OxauPa#Y8|7cG zVHfPIVPBhX+DeXTe_S}+Kb)T@qc_=Q$ET?d>~RNO#e2 z@`|bC7k;--jAhFe>#d=@dr|LW(C_osGZPRQmjcRn_qqJmUY6)x>vY$2RwNh?8&|mr zPhE=?S*(ZBn}I2eW+-PzKwJU|5JWANQ8d*K)w57$HJD5sUIXl*7o|ZDDdSvhJKh|X z28o;U=OBko0d7G=2*L$UEEWKUSZP&U5B@}7h)=udWaH=ZWy2g9wxN@RFJZm~UT;0j zH|S=%%`eC($9X&~P@OX5_s-qELfh0tw&r=_x2yPY-u4r{EBw8b>bby!F-S#)$GY^3 z=VvT!-ujlc)fwKY8?1`nm_Li*x}V@xwWIcO^YNNYUcDW+wQoSIV8WA})wUwQ`FY&=_tX2TgVn?ugVPnW_{)qFLW;D(T6#D8vf>TXUQ!; zi23*jMVcUr(s0Fo2|;l9RkK?`UTdOr8V63yi?T)HdSe+1r)oDeP(dDoD+1G0;gHOp zK-og*8Ko4w5CY@%adeXGI^anJYDmYJ(F?CTf;?duU;uEmFOyF@cg+k zD*LMOuj6yo*3yR)F4!7Nf9CKN?FgKK#n%4aQaVkA#g;{mR2Hu6S*{)>QctEIrHXgRVTQlx`D3iI+dHU;(@MYp~Wm8Lw zGMJdr4hxXNLvEygvot>8YjRzmd)wMwqJP@*eojdK_3tl)!W@a;wmTBN+W(>QG4K~q zv-|$T@R6AU6RCq$`iCl2mjvNcllruO@mKZpnl@_g(qmVa(**Li>az0B8y~_>YM*aD z>Yu9D1^8ZH6tECg>RVB1T5FTV@F75!Gb z$9s|EAi@TAIG&fDi&N;sEMLGo~* zlu6|`csw2?g3Mp<=vP#g=iWkF%pbtRqGDWv#da@hE`*}p`0KU$R{=8G$UwOSM;o_>ako!vf8<@0<_e5moraDs zHhw)878XAG!`Q!f``5!Y&}Ss=VCmaJr=qxdZQxPKRY#pw*sCn!i<}`Qc_SgZ=t2-; zKEm;wgZ)`rsJpm)`6PXw#f(@>9IB<{4<+Vkb?0|RwBy911>O?8#zZGUjz|7bO8Th| zVe;=A!B-qBeFBw3Q-8zrM44X}=bUH{eQ4cYa!r63|MLVpqf;vZf{8m3-N@K32fk}tQ zE)AH`MxLR%V8@C-AseB@pXf!gCL3j$M2@*<0#0JFnO&ajLU+R2Plw_CHZCVLl1pjK ziH*)epKP-N-|TiJ@qD=}%mzwal=7p&V z0duh3;EkM*N&r*YbjW2A2Yew0zt*W7laD~o0IMNv*-97@@y{T>^zOAjkowD+x}zeF zp}d+D|D~Ub40j*P)%qnU!os+*$_Vjf2QiF-{%W5llx4E*IdDC|p2_u0)LxsWA<9g} z8Hb|R;K2E6q^F~O(RPmHsyks$;*Tdb8R*qWWtt?{>C-A~?bc(!=)j*}i+CcnLpjQX2@R>2H4T8AUH7$#ns znuD~C6Et{($Vu-wp>oLkiL@WFyteNl$NRQc3Fkb!KQS}@r{9~h#aCF=&x*<(5u=@j z@k;bu4L$A&F_|gT!7o(w4E(p8fyPg>DS8-=1`C%5#(6k0Oo+(F1E)r<$0Ap}Pyu{a zLXy7{fjmW7$!llxrxg4?I7$hGk>l8~J_2;_bHfG;QUIkMn3gY>Dvcd7@jV`qbpLda zJ;}H_`d!;9cQc~~bGvdn_j4L|t6|OKf0B#C#xIqazdhx!e9KRah?j@Fh9TWs7dmw+ zeyy-ohq&ZecRKFeMQ*hvWj%do^3PX!yivH0+{L zST8u7lgVY;_2XwNw8A#xGV2}l?6O%oXB$PRV~W2%l$a{->h&E;MN1)NUZ#Z;6>MLpk&r*3iE{?hSD<6&q7TjqGI-KL|fa22cn8eGb&0;=S z<*5|XF~=!>zb#B#LvaI+Jo}-)4nY49!(l+`sq-usvgy11FVbBXoNC4(=NQY2oDke209M+Ke#<4_wwLR z%9~6pQxmzc+0)f+-{7HIleT;Pla)6M<6?_N?RG|!>*w7YH4)qAirc1g?vMKM;Czjw zG9GvWe;z8g3oXM~G=9GIj5u)mZt(rRJ~y{now%~z6tyIo_r&W5+KdJt?- z)GgesYH*7P{nU#}dErp1k|2#E-%-7@#i897$8d-jEjpb^z^H{qiR=lX^F4zBw**ya zI2y`Md;tiE6tV%BA!qK5B%>lDuvb2e#YGzWIt`q&*HnU;u{il3{7D{M=6fiN--oF^ zN52>l{;0N;iI1Gj_UN8l_Ssmu@O*6WvR2Qrx!~;lOX%du{tZ5zVlPIkUIK0Ci|-CU zfNP$_1yfxDvhJT#BlJZTDNXl8HJn#w%+mY_&UgiE|2>SK+*9fkz9m=^yYM}XK(gQp zK4_x!zPtY9Jf8CkN}M*|(P~Wq{c+#>HJf8~>_`0bs-zt@^htdDj-y>{I$89qpYl(& zarcAqCP?D%u(Z97vEOx)+kb(c^;Y<|N(;02vaLxlGXKGEs(Tw4DWDrAnesM@MWhE& zH<2)&APKC23*WH!$08cE>lW=Q@m(S{4R14{lJ05hc!Wwb1m`WUiE;_|7yJ-6UWjt5{7(7zTJcn)G%I)a z?SrK4|MB2U+FC4-XY$B{||R|%93n4;we-*KL> zMGosnt||AiLTs2FUX6~(NrpadM0UhPM0h&D`*Gl_3q4bD1(c*rpPYFp`fsz4JxxP_ zS3#7|h%{3_neI5^pOwV>Le4V?G~}JwrZg%QKgRo@=Ok%;I%~eQB*ksz!6{_Ek9fv$UoGRFY>KLTf@T3RWe&oY9zQ1=V|0?XkrtAjiJ)L%k; zF}lN{7LuQn3iPguG_U$1P9bd9S~Pjokwv1k#?lsu_$26i9{+=z3r4{d)uo+x1)lFV z_4ioy53Ua$dNmx$8HR0U)V`Dm|CTnDi?NYj9kEUIS_-<%yf}N3K^%fJ%bNaMT0}M% zfbio_bF{ZVWU z^S$hiPX%+jUJM&$(dW<0`n+CdQGR?wkF`H!;<#E^?$hqB?zX2HfI}RR2PJ8sWU-X; z$6<>D)_L!y%*5!}V3<8(-TwBe;?`%!zz(<|U5=XD$V|EtOW%(lQMI4HZ=LReqv*OD z_;)Zv5{S`+cvx<&0(43CVk;Hr70SwwhSGik$1RUcAo6+i?B!Ml4qF;>?YBVo^#m(3 zWA*W;_ED{mj3xMPB;fwtE@BqDpEVrtva}SKSVp#&|J*hZN6wv1p4Jmh=tsuN zgqn^p9G9tD@O~wDN3VD<0?g-0&T7V6tEgCP`r;j$oW~S8t+N@`7fGg)Op5!t`H09Qmn89$CNQ$oCriA!}LWp{JSI+3SC_ zl|xQp0s8Xr5w9hE+z1wBC>aWeg-=fU(RLF}4^?O-Hfdcs&D8iG-;L=&OL2>hHn zW7;N@6xN z^8x|hzJA}05`~C~E!JdcPon0A!AG>;!j=K%(@Y09H;)&>c!dr^u&H5v77ztWf;ZSY_(#O}9u=`?uyT__}4<~82lD>5PY`x!g%`>T?2_FYzpI<6@b zNwge7|4{-JFsWeHr=Hv2HrjQ4&81~`ZO=`!=oJ)m%f^T!zlU@)YTsA9B4Lf+9-%jb zsc*GZ?(dsS?Jq}oPc?>eXFD9{%@siaaF@kpa^h9m0fN7^1@yWMqNc1wD%w zp5N*f%#V#tTUoSZ3#y5c2^qUf!e|imc@yzCb<+(8 z+%u<|>~XK9>>`o@1$*3=RwG4j{K??wzNf(WudeXA1@Epe zj+T;jjRR7+4e+>i$_=UWEMA>}{u)!z!By$wb-na}(M$ddmf!G8g%}~1t!|M6Fo!w- z%th?@9z(kXMv+GY9_rftd?PQ;bo+OijcS-UBphcB6}w7;J5$=A#jl1SPWIQJzm&Sq zrA-n*$GmuTB~>~v>Os?}VRzAyVP=wgNeb*B2HAqo^ot>zZ2IeOd0siPTJG)2En zEAoX8e<8k>rngRHQs=JMXlYH)Q`WLP*we}%Kxrg>zp!YjLYbfkO~r35#pLe`QQgDM zL-f3fz=1cn`QH0A@z{NGrc+3be~EohlOYia!aZUxq8mFlnzm;Bm)5U%wKVSsI8uE0 zoG86Pq8wiP@bqEflYXDC5sAQ!Aj&$`kHja73*0|gD?$ORiQj4A>tF=1h8hKHqPOAH zn0S<^@?O-u?*kJH3uY#Se>)pqn%OdF8;L*8kjK_^seV0y_xD#O?Gb<4@-k4G3k5Uh zgkIB z5Bo4t{o3i=UgZrtoGP{Z(XP>&g(n54#ASPd_A*O=@dxXXc1fH+F-?Oeje1M z4_bSg2E_I`_)H$BU1^0gnVLLOIh^efa(=pzQ5W-8?uk5U`;aEXrKBDa)r($-8R?f# z8Vs}!TPmIIDSqe*_Rl3hZl)#J6^`Js1@PJwn%F|g;*NUzf>3YYyZh&C-lAD>U+aa| z=RZF_BL&@mcoBGT(Ei0E<)>F?cYj~t6Xcf7pw;`CH&A_8bZp25tz!U4uvA#| z0eUx(NWf%fW_AEVa#Fi{Kyi=!*BKbu*a2G39@HpTC4-nIh3>vAv`Q5m_vtn)eh4BS z51m=;Sl+_enG2?d9}s!=JMhYfV)W_HFOu|TMCWltpCgQv)^g&`ddn8Q?PBwfl$sac z7oD|}{>^g2Osgh2DwF{T5cnBR!wibHuUAtYSn2?avr!pu{Yg|Da zpMj*yG}~7)A#!>VA~31kY-#3Z6ju2Vk2kr(l=JF^7VRShaTK!0KsD&fkCxH(f1w`i z5)&y3XJxiGDWn9p}wvSYd~V>FXK@II=vpDZH|ypKfp zi=RsSwaar}7}qanT1~tCiMpPl5;hu=(oaZu9+6%7^asqqUj%{`{-Bw# z6D3=jUv&-|MNGlN83;EA(s$a01>v%2>VHfR31f0?6vdggno~TKs9qke;rh~|{$Gq9 z>#u=Hyhmgl9O;&|)_lUP3f+$XFgN7d(RRk`P?pIVp~es^Z%_x*v!ljuWGU<60AV05 zdT&xvN8J&teC8q(+i`?0T7X~0skGbCk9* z|1gTxm;>6Ed}Kq4c9;vSF$MnMhK>nG+5NIl_U7HWSTaY#zFK?qW)h~LYf7iqN+JJ2 zr2>+_zY6GFazgXVDwx%1p;S3@FBU@}p(Koew&-@ZsD2`YM2%5?IbCHj+~8!jWY{m| zU%y$Hk*%EZbU0TUf2!HzF=$D}-kPi+0vZjEK*J;*jes532U!n~XNwQ-3pxASj1_)e z56;030n-@C=RCK|`0VGKIT*$BIs8!#bEb>!0iDmj1C0l26CpYf!37~s#GzHF2Vr_i z#fllSiO60J6WvT_JduRHS7`t0jdy-~oj{~&&gGx$m5Oe+QlmwG34I&ZF6zeweQxKg zU}zNanM?D@0=<~OximX+D#)d*28t&uhL;}LS^Q^~at`nYL@7DRkbn~(N{QZkZkYuH6wO>-i zV2bPm=Dj@IWvC%97j&qO{s7~R!7=v7demc(kjD0i=1#L95D!PI`HHSy%sn8EOf#80 zHjVb^LA-!cW&Bo)Lts(fU*NOyKLDRKO29w8cg^(^A!BMiTg&}3Q#jQvDH?4VXru)G zX>!9aVL=UX6~C@@T311}G`BA~v@iEWEp_5cGW-GE;28GJ|FBHQBd)K#eeJ&TSqzX( zHM(#%gDJ3P+TBmp(Yt}*11^wX5x55Z;FY$sPjl?atiWL9>;XAa7;EgeKq&FMz+6iy zWB}^1YLAkTL_lzf)ea@IpoQYf5&l~^fXzb$L+{ZG%J_mdi3mNR2bWrQJ{SWgJ{pet zhA%@o2NRz}+r2aq#0pGaOXNA|(AEB2^-j9ZYL@jnZKjqaLG>eEO{hN!CNW!d)!+lB z1}y0_QS|cz{fNJ_C!F=hdT41fvUZJ^gHZ9*a&^TEp2Iin=m0@+(MY}pr6f&{J4Vl1 z>o>YN{T;Y6(azmC^brXBe`z!ICw6yqABQT#)$i<4m^sBa30w(wVx0Fk*tO3jnVyr) zc~erUG|6eCZpV)9hr}thyr7}B9iVBSIDRFC?1yrQ`QZ(-N-aXl?-3gA z)31}iVk|iAH1-a)MJnJ3mpoQdBm&dc1;6OmC=fFM*@z?2BpCATT)ny+L%;}A=b&9( z7Z{Fh3VNUd7eUm8-{f_=!yeB3JvzcSD6|t>WUrUr4=Q2_X5Tvixy*y|kZSV!eISmJ zJT71}3I%c%hu>8`Rt#EV#PwoWWxAxme+vFh`zdk&M-N`DikF0HNrOM5w9$}X4?!%B z+$8crU@4JH!Lf0moSgr~4T3+;-yU^NddcS_^a=vFJp-^+r#D{0Stg2V(W{U1)wQ z5Os3^vM35$otbd_d*NbOlNBaJ00IYrt~H6c2NBQtk?$l$PC-C(G>-UJUG(&SU`b?! zAx|n_8W1!>2g50oAM(+lG8*q-f{v@<2TeIDnL=tB5=3Szfif}S$VjL{s|UuHHa~_< zi~VHA4g)<^4a&G6*eno&3Fh}?Nn-ybnOoccZ8uEDKExR3CF9k zchE&v7PB&4ZzhJUkRf|X)Za9>8=RBc);_mnf#>Lit~>oc9DuK5^e3_#(XX>BODmbU zGkrouw6nVMHPaDyN@b_phne#5?h1d)w-xueJic9tuA)eVRM34dX7Sr^*k@}4eBAMy zWec{c3+^nB7gs+zs#fs(7YKa!7td5(LKoW3pGPmW9Xi)8f5r6k8M=;+^LhjxvBS#^T~^K*RvF(C_tgqyIc|*W z_b4-_)+gLP?j;cF2CUS7Yf!~RWHUhueS{a&;y#tQ%Ehh>1u+)MFMQSZQj5mgY^01G zd9F>0j);E=ap#Sm#(!i(bDAie)=yg8U{400a~@Ofj3Fp2j5lb+B%TtByssj5<1F6i zzl={N^%3Z3NjPI)Ty)(4Pf35UX3nn&ntrQAjlKU#19=TdGf+oz00M{7Z%Arlp~UK6 z)tF&@)PUpe`=BUKT2cUge>+^1VoZ_lN=V8_cR@+hdKCOB^pupxzeOP3&UOWW-i_4;Ai5XM2Wh3Ws91<nk$l+fTp+b3e*_Pq=1Z}eI>qi}6 z)&%bk&{z4f_+KQNDu)@#F=tm7@3;a4t|TrazyydcFmfPGP9U*%lNbqwpfk|)RKDyo z?G8T{4bCT{O8g;B2vZ|RR|v={{UII141q!4KRA7C){Y8bRgez`o+fLNhC}SB9zq4m zAM}RXvs2?U*n2sXK?(O&OEM0EYNStshcD6t1sSEjKE%0*?Gj@DK(7CcR!3PH5l;$z zp91MRny_@R#o@jk{%s4DHmBEQezU*7Abz131vgEV>Fe&-3lk$qi%S3E3d3nMQhREY zu2bSZ<9YCnOf7azQ;;C0AJU`w+W~CJCo(FFF;DS$=|{w64&7bZZzTM@5OG_zr@^^DiEWgZGI3w#>N8yJ-**KEziT}EgvmXw%Hz!Ln;9*&%a%&F-m^tR^#R_9))xkq zgc*b2`lB-w9{~0>)*=b5&Ae-H^h~txjT;M;^P{QXjxi413~j5y`CP|PmKImpS!W4B za4a)K3eFCB4`F0~yD4l92A3R+sWKGQHm`%}8Y*&{423B`YMuV0+q}o2oY0RG(ReWloj&H;G<%-sQMzRQ#S1p zv$Hu>nNh9_2(jK%)v!Va7(gJgW{6LN3(_P0qx6?dk6DF6aoCdoUm~*W+i?bgN3Y`8 zxIB&4O8!+&-2Ra54)gH%>t8hMc0Lct2z_vzV+?vc+D9yF!$4RpCl{>lerDEAw+nIt zAiPyZTk`g#aQC}N%zNtEhz=swwbmItakqwphV6?}$^K!GD7+VXx+S;GKrQ%;9fnko zc3R_~4=a~{!$$W|_duNrc6<0;$Q^4V?s-IAj83r%NMou@=tt#PFArE7sT&bMekG4} zTzRosM4U@-%Q%(B2Kn!Hpc6X;k3z}k9#Ie!858#=G)ziuo^_&IQBPThxEL?X&dHZ_|XP1;Rt#(ewSAnfO>-1gsQ z+MTn`3;3)eXhQyg8Get-@-lmW5EC3BqXM%?80Ns_{{s+gNe_ODkqCDB^-Pvn7&TNV zt-b!;3Yz9ifM{U+G|C!Dr7CCvnPy~BC^1Bo40G{1Kz?bF=71w|04k@_wn=Z9!DGuT z+Y1ZWL1nW}E6`&AT9QCv6O9kF_Z6YEj1T?4|KhLy)OLpQJRKDcja zf;`{B?J2hxOpB+Ppi^78Uk}BTEmcR6h>kaGp}XGNpTA3)Cx!2KJJkc19}oUJCVw0q z9pp6sW=pHfIUNN%!!22wLd#R~eGv>02`_j3Z`Fch7Tz=?vGa7!;WvSEH2xH`KCyMY zy`pc6zIPtQ?)KdKkM8C;JG^&^JKS0-(Ki=t!BAt^C*9nSzo?d!FCXF5fdx=ZW^3>r8jjcf>H5V z8S}{;e(K-`c88;I8y5&O18Y?K!Y-&}tR8=40|B&}*;V&;-Z#?;DNTSYJ zX=HG@Ua2=!GoSR72G@S5r&O^<8&CWH+Cg=6DG-dI=Il%4hk(f=^seP5ZN}l{2Owyc zh@#*Q#?2N)({wNm4sG>e)HMcj@H>EKhC~V2q9nB4Z%!t2^cR9lrm%SHh;1Lls}z$uN7qY$ z$mlbn>hTPx@L-xCMn8e6C8BkH+a+}juUHgY7mXdL^=K=|_oA87eH{hF8B z#}EW})eh;w?K!((mM{&vHk~ra!L(C;+kD#4&SO}-ezW=5L6jDcyW&4q`nmBMaVaTs zD(&Fmbq_J_Wob9HOVL%{J^xOnk!VWepqW31V_tioI^4INRJ3 zz~GFcDSnvVmGND>yu@p%9!oL?P_1&Pv?Ym*&Nu9C667NO4JJNC`*?|F4(JoL6C)D< z;+xS$HP&pDWBhzdPEL+esKH|w;=_X$HnOp^2LR#44it^spUBK-`ICx@pY1eN z@8_@2(}13RjW%M3$Tme~>koQza$o_1IK@jjs&r%7EK$tJD%Yqaz$ZwZRrEQ!|lSO3e}z~sWvI$JDAsirZkOl_<*9yX}q>n>$f zG%+UgkUM>`KeJzS7pcanwF6;^ML}J7Q?Tul5q=2eu@vWtwcsIvh9YH<)VXUyS^P4|9_s%-WD=N^=SSt?Sm^U8(&X@k) zObReB8Lr@`dLnN>+tlO05@)^w6_>yBhmW+H8x%$fG*w=1l}K?@?uU?CAa^7D(?Jb` z#F8#=3fgiHrAaW5OzI$26+s8uNU>=>zsn3>g18$fUAhi$I{TOKZcTi~216c*Q0Nl2 z`_&9Qdj2rPfnMD#cYYUA9HVfc{`&Ao-h(7-=E4TOdIts+kn}f!r#WKux`NqhcrZ4e zmg7;Wrs&VlapB9IQZeEF2-IJYkQ#y|4P9o*4h<81ry5E_g3e50N|a%HBr6VRE>R&= z044ZZ)WEoG&845h@m~&+s8UzhYfz(8jlPYuxn=r}-ZsL21{aVt{~Hq{bt`bmjvFZt zj+ohxu8{7}c1{kuxG@wNDwMtUJ~#NoUJqCXBVNBgR95r8b9((r<5%>buWu3Ox3|)d zvJS?{67RDM-UfNKi|$*OzSxmTvbXfZ7*gGvl_b1_0PCbK|uRqA8{aU7gwsd<= zj(itpeV#~d@-&HA2EHtS9+66dLG@b*F%VY}E`9MLA!Q-&Ws$nJI@q*ilNM}{5Oyg4 z-=d8*)M<+$G2M^-I3$8IZ_=9hQD@%1GJz{EXY0#v#aAI%s&i5gw7 zV6Ek#9u?xV_LpCMDNthS4uwpT02+Wi!f85`SQ#MHj41n4XNb5cL0|j(Uv*LDEZ^V6 zr`R5z&bFU_%560S?n;_!?ORK><|l>t?4N3RxSNYf5!w8Z5dzrQ!ekQ*@hnNP2b(8mnZnnb>(4V6F&)VTttO zDOu*$-lN^HMbDS$Vtf_0*pVEp(UVmWwcyK1rMURaeYvJErNGW*2Do?0L z=p=)H$&q9p5wbSiJ!<^yOkA*S8-wTQ>oljMhwHrVUDro1iZ4H7ziWF+G48;-b7h4M z3Le5tJ}G&N9{wspLQ5GIJ>S9HTRluh6(;M+^}>EC2zJ75_qrFdmO|Od9uCMir$n7~ z{`-|V0}C*O19KVHNAicg1_w7+UuqmM@+rU(l6l!!n@YiXAg|pwNGGLG#SCodvyyd^ z?Y?PoJa;?grmengXPVrrfTJ}miQ%l3=1i}os}A{x`@P2;WK+hGaBcKp^G)c%?xpsl ziRP6de*l2QHPzTkDh4{LAE)`sR2wQi4`S!zBM-GcH={LiNV16Y_ZuRt|O`ctoBDW>Qnt}## z%1c$=W`0Uqiz2FKmd%-5R=Z^v`|@d-?kQT@zCT~pQU%%UuWVv1%=gN==&=BOl-M}) zw|L1|un+M#`Oku;qfKN{8Mg8#aZm!e2ScK#0V@Ase-;yvmhc05&bu>SDj{dAu^xe- zQ0aFi@}J$k?$wLUz@HV=sz|cEYAYhS7-V<#FDar^Rf62L`9*4XQuuYz2pMvfM6Im? z2Jy><#&cNT6IAe7!>Mo-fhz6$FCbMzm5UZHKN^8YAeH{Jy3;{*5Dwc%f43_uE~#SY zHS2%xpJYFLG5Yk-&}SovsLR{$=I65)*eQXcvJ(G}tNWK#X2maLys^D|mfpoSt2%j))|H$Mj&I;mWx8Dn~%ViK-)tgo3KI^On&=M3M z`ZnObcrAEe5hj}gisMct1KZJi2+>)b59rUsA|Bj^X7}EA3Z!D-y$gKPF|~=6GTKk} z&8X3}{(s7F6dSv|W>7_)21Nn+qA=``FgCS&chS8oWp%0;KHHj~cR6(TLq2B;K7m7? zp$bl)IWpML{9om$k5F0di>+Z$cYXpLKO8Q){Gf)9$QD7N{`RXYH*b(0POa3Q&a2Og zl_(cIPNo9Pk=J4itgIJxYIQP*O z-s@9=@`ve+v57_Ho#7lQzAx`YJ7dF{{T*5=iA5HC+YOfSImXC+8dO8NaTeD}eN`f4 za^(`Ft_^xg&mO+qboKQw`N3_vgNWg1cl(_isJ|CtG&G1l6lZXe%wGQvWD7SseK?B4 z{QVY8Kz_IPj#A{hZ8$*sp2BbbzY-~rmMQwg&IVpJ6`u+!c?6X3b}F0m1-fhmWxttv z)GSo;2u~CgXno_iU?5EQeyk{dLcFmh@VRQpu!XYRakM%KXBAaAw>sRDU((G2e9P2Z zYoh&Ekl1w7~ z43GvSoE$y)I)tJyqC-2_v1{&Uia~FW;X|W;mv|0-ha6&D`}s~Cqi;mcoC+2}VdA2` zSe9z#7K>KY=e6!B;g&+sZQk9nPg$B6Ptod-D3 zBDr{fWhU!H#79}5{UZ$zpYQC>XX9XwibJ-$#!KVVaM>uS^T#f${b;GYx>?UzZmGGZ zp%FI5FSNq|(T)o-4&6M_X^5LP^)$mYNGWe&6uK%{z2XFQQHwXX<`M2%bXhLL&ck&= z(#{}@dpm^{TU4lU*-@>YR?ekE!}0dx=d(@{Vo*3qhcX>qYdwDr%9_D?YM_ED26*Lh z*BqTXYH=9l=k>+mWJbXf=GByfzFQwR` z3CQi(mZf1i-{a|x2Y)np^8RF%1HtMFNTejJP^J~Sy$815;0?ejPDL5rYu9>J3Jte- zEr-a68cG{B&?b9+UX=mb+He<^0_zmdKjl)Bun|?e{?*toDQ1$#dHQJ z$R(u<0!(2*-pHm9+jr69 zXo)f$&VhE6@0IrlWn(vq9Lir!Gc5iGMno?Kq8LG`)~7X?J)3;Q%f@C(D&aFZ5C3v2 zt8bJ-S^s>03J}C`V3E+8LJ`qYO#dJAU;Oawvf&!-8zi%l`o#llJN)5>u4x`&x#93o zk)0)GlmX<%n9#2Rg+m;!mLD8V)Jfi6=tkL<&6lnj>pgfuIQ9UP&KP~J$edv@`M2k! z)OR8={_@QsLH3(rL*i!}u`kWSJY&Rl>lm!J1+eE>1;{$g8Vd)??EBo_(fd)F;Z?g! z1-G{OS_Xy8@fsp3-mM3=f@J6j4)Rha$QFu7 zIx*y=vB5)a4n#!gJMHy5Q^XP_xg7AEmL-J&x_tO_NNEg(+-gM;WBthu) zQ8|ebPL=m42%oTri0F@mzGicmVq?}tWBNq7!u{Y`lZDUquu>wGo$L&r=Fvy-nXVsk ze7fd09Y?Q9XI-3cjt!YG9#V~F_~Q~2hY5;I-Hqxi!*p5H+pXEpJ3$+QFeMIehLlNb za?#Il@*akCAa}C?0LD#}Tg2UOKq9gqAU} zYEGK@S6$f$WY;B{JZ+4n3t7fz<8u&sDE~<1+PF`wiIY^A9};~SAN}%}hm0Uwjw^*) z?-)TwoL!8V^A>K!2CqPfHIcaPhMT%Ot;e+cJE9cJywql)-PY}pkj0BrjQNUk-^Xmt zZy3(H@Eg1VGFL?*b7tU;3|p%Bd2xX^l5_a~&1-?z6((04h?B3>{?!?B?s_wD_0Y9h z2Iu(@=cr!aTA1jmwE2``e&$Dt&^O;J2SEz21R8uj*O46`^lbhi3!{HK_5>=+q@oVW zcuvt;V)F^_Ukq#>V4<2^cuD2XiEe*2YoY06f@$fg+FfZPW^pu<$P(P$u#))-hs@<(}2jjNq@jVG>aS0j&+Rsah$#J5?2Pk zXrGsiGuJvnX&R~)|JuZX5;}0Zf-ylcfl0O;F;agX%Y+Y}v=7-1_et-0Q+>mZewA-O zA6u|^PToA*h87)39`}+)wku*C39_T&lN(d+gz^o_{|{q-85MQ+egWe!3^4Qz-3%d} z0@5%vNQ$I%Nr@mK-QA^#(nyK|A|N1A(jXlQQc8-_ozI#3zQ4c!oBvwR^NzKA;jHU( z&K3LGd!M?+rYKe>YnFaPM9}*FXZJ3dML92!Z9}t!1 z^c#HnF-~2gDx&EUDuCVRoQL{eXQ5C`x;LbJ!i(jq9&}u!Mq9NA;OOP!-fNaDE6v=z%SQvijHs}V+{iNYTSYC z9j=!_{cn><^K&);4>+Qeer!WnhwX`1Wodx5CKg%i_~O4<$2%{D6|Q8A>+Ai?(+WGY zF{pPsj=Cf#rDxkmO6fUS_1c)%F`K6O14<+gii~8kR+BZ4vJN(1E?uqi1pjti>}V8z zu@iN+0tO}zH1Okb`g43z0pu8jjr=Wijt7v-9L4KBQcNyPQJKTE2)7C?~e^gUadg((S(KLtZ zb4Yvm<^yQ7LqWrx_+4}H(zk{>FaQsk6u`i~qAIRcYxP38l9i4$G$r#>T;Q z@nVMIfbtS~`&!_p>ZUSIPoH# zTKu8zRe6AO*tVtx80J{NqGmkv@%zbA%c}dm-y7~1E2y4d&klEPFH*f3T@On==7@^r zE*$m%Zg2t113H!w`!jhzm+p}#{u#AH(NSJb08v>YjRu29bX1UV=7avOF z^ScM@rP&-us!%;+)NY;3o>)tU#s7)|gd1EQB&yt;l1ZAQf zoBg#4+=pz&w0wDVH0(Ep5&PQWpJs!@^X_u;0AEXD#mp^Qk#+B7NUY)4>{?Fj+G8I+ zi1}4%iws3Nhn1x?<>k$FzZB{{Wqbo*ZMCb(QwynyoVSpvM>aS&QypJ55z!ASZfXcJ zV3cAb@3V)sr(Icv9BMzcH15YWo|qYGbHLSaImls~eG4S5{4 z4r;yJu5Ps%@nMxJEIcXG=*b#9=}eDHSA=??+ZT_+&hffpaG;r2k2B5k2h}4+Y9>M)a1)+@0L6V@$3m?3~QRU!@Rw&s;As?a-8fMjmHSq9o z)!fuSDW(NH-5C5<0Z`*dX0khu^fv5k{@G3uywgkZ&fE1LQcU7{82Bles5-jnNlFA5 ziPg=kB(7ybLWBCnW%+_K?pQG(?xwW+=rX?s`mcv$>9Qf&`B<+nmfjfGIc^e@BF=WT zZ`$1Let*UIqa5$p%{v|$t(+6vo1-&g!`!II06lZ49YW@AshrNty+Y{9 zPpQOOU2*^5o^ck#sE(7F5lz@$;;vsPf>GD290$|nEc%x2*;GKFb5=u4-+QXj|^P-5@e<0+@FXo<#TEuH|C|7cmDW4xcKj()=R60 zzG{MC@$Fdm*IXU{Ex?+7Cs$_Z%y;PA#STMtZ}GHW^;hPoxW*cP;mB6%xHzC%GCM%Z z=>a;}&x#R16i~%0|7-E>c=Rv4hH33Z_3n0efmA8pNksu6Wc%w@fz5$`zZK|DV=h1N z13ATgQOw2R{B|G~Tu{q!2rcS&!T4CTdic3UTzwW0kpJJNAXoGh!~v$D7>ug7Po$Z( zMs&%}DmMfY0tL~l3wqlbpXr@PLO_eSc ze`xv1r6x=~dAA`mtRxk5lt4p5iwS^l5a7bgowIcdyfv4Y;%O&hroRs^1c4NGz#%JS zn8K=cn|S;FGACyJJb{#v7A_D1?+?c0C!C(CmC{36+d?pVYva-}S95)djs0~}C)3E7 zFz0fZL(>M8cpA_61#_OJZ75VN5*)4?x)S@>M+^$S2zuRLOin>|#5p&~rydnD( zxno5sR$j0~Qs!5H5c!kKMM-S?-Im|+{H6KEr|*GBe^W!Qr=BpS5Uqv?tZl_8$dVBj z#zA6>?$VLFz7m*RzR?{DD5^QUwm`sR?10sI=Ea|R;K-7uj0g&`Vt0}^?kK(d;@ z0LReF_Lg7TxPze;KSquMtYztdt@w1YLE!b$b(H2(6wmDdj3n^!ab>gGhvda6=W&Z{kKydD{+m+vzn#Q zY-6D`ZQ_gH-o0 zo(%1nb{u}4Ynk)y(rFQ*_{p-g_yOyWhQg+15uU@(-{q7d`+=Wn%mnz>gh8VlRkSw< zuqQbB5DrGA`u!b-XjnQyn@9hY>rbF_FdAx7EVFOJdKbZnarK@f>Ua4wIOa`QNc;6h z`KZtXK5`vnOj?aJ-4|OYorQ7t<6tXjA=t{AcZo?qTEvSJX~t5(fpvmjUTsv1}^1Jp1!c^YCv-y`wcZ0~niedjL02!Y= z^0w~%MS)s+(^*Rsgn20&OnjNsF+x|{1#4UkVdP#QcsHzyz>8P!bB+!@co~Bxzz5eF z`;D1zt-tTPYw9aCVXwLC`Q$R3De;lsoexZMj9!wQT;?Q=T3sv_ts95B(nmCz5z_gp zp^lf=_@g%;6MRhnW8|YL*{EO{A>6<{k*6~K;K|RtW+uVUX^uU^VzncU3ji4vggAeq z%2e$gqh!JYfHmR{EBC_$@bHWySZ(7@Jhxs{{7#!QFyiuGBjI5E`wm7TLp8;vSsT?_ z2C(($`#)oWbTA`CKEw4xs|Wxaw6s#$_7)`VpTK$G=qo~NwUDp|2_H3&jA*%M{f8SK zAV*54JZ@IFb~ta=y{S(@ea8wy8ZRg2h_~|Q){)vo$tDBhGuKnWfk?;yghAjJLt#rA z-hoa0v}-!O(!cFQ6$>7z$iXt_&iZ$EsBiLC*MRca{c~fi(J%Z5TT3)Qs;ffB^aP#b z9jVVSSy61Z4Kd$O0*%!?F9SN-=n^sBdtTgeY~Hk>e|56{Kk*hdvi;MkMYmfLlVF4}^1qr6d@2_GwB(aXR`U`Q!dwX{~v(R$`% zqZfn5rwpdW$O;1$81xaY$+PgWP#c4>NKR2OU=>MyHddZv!_)s)pC8V|5GhhUSAYrvPS&r^7xs;mt{Ld{|ls}BJrHfZNF%)1R%;yf~q!@OuoTas&x@i7H!#|_$guo1@i!PEu;I@cA>R1y!C1Z`B(N#|+Ne2A4; zekyQ^kY6lfIH}P60?nfC{%5Gap_CN*WXWaoQ`Yr!PoX6DEiKfk%r5-mni%TK6y?Kb z(^*xTiVkMy|CeCW0+G>hI0)u3One4x3G>^QuwtJN=7_0z7T%FQeE*de18{0MVhV$W zRs?7ey7-u91m0(jb-X%H#)=L0ojX!gF%XCPwTDze7=acGs>|~U@_@t&M)zQdM+ZLs zbo@@poonK9`*CQCTrmFdy24KkuodX=zo^T)`u-0Iy>OBg5RaL6PK9k6NEVrOu#hUQ zFgG$c6UHPpTVD9Ml@$~=H$f79mjAbfhm_+9xs!mnkv1-1gzX3O1?@z%Jaqtv6(jl9 zNvPWhadM}ZeTbliF9;g)RzXjIwVyxo;OGnT=FeaW8EbV9tZ%}3Id1b+F;ko85#jFj zx#*6urlld>cGOuhdIn8Jx9qz`lJ6W^4qrISu+WmVOCtOdMg%6YlKr{&tdqjp-y*5rq zvB1ELcIVXRW*}1Pj;zRkYo0~{%AdgX0eEdx)q8d;X6+VokuNaZB|>-Tk0JKLCbT$4 zoJ5NM?%2A5q(wE8F{(6sDSj8x2>b?w7|kDKU$~9?T*L^1_#RyQ&9?`BMUEwXw&(E} zyz7=$tijprKm%~_UfI$CnYVJx-Z2p$YZqi0ejEWC;bG+H>6b2b-dC{bb#@K8u>GVI z`PCOY`^kH5m^bWw$)uox!)<8#q9AT;_xxuZFE$z3a+)#-M1XbTqX@)fr9nUt6142y zR_$}n!B7cm-ym5&3FjPFe9vg&kJop$tJD%Yqaz$ZwZRrEQ!|lSO3e}z~sWvI$JDAsirZkOl_<*9yX}q>n>$f zG%+UgkUM>`KeJzS7pcanwF6;^ML}J7Q?Tul5q=2eu@vWtwcsIvh9YH<)VXUyS^P4|9_s%-WD=N^=SSt?Sm^U8(&X@k) zObReB8Lr@`dLnN>+tlO05@)^w6_>yBhmW+H8x%$fG*w=1l}K?@?uU?CAa^7D(?Jb` z#F8#=3fgiHrAaW5OzI$26+s8uNU>=>zsn3>g18$fUAhi$I{TOKZcTi~216c*Q0Nl2 z`_&9Qdj2rPfnMD#cYYUA9HVfc{`&Ao-h(7-=E4TOdIts+kn}f!r#WKux`NqhcrZ4e zmg7;Wrs&VlapB9IQZeEF2-IJYkQ#y|4P9o*4h<81ry5E_g3e50N|a%HBr6VRE>R&= z044ZZ)WEoG&845h@m~&+s8UzhYfz(8jlPYuxn=r}-ZsL21{aVt{~Hq{bt`bmjvFZt zj+ohxu8{7}c1{kuxG@wNDwMtUJ~#NoUJqCXBVNBgR95r8b9((r<5%>buWu3Ox3|)d zvJS?{67RDM-UfNKi|$*OzSxmTvbXfZ7*gGvl_b1_0PCbK|uRqA8{aU7gwsd<= zj(itpeV#~d@-&HA2EHtS9+66dLG@b*F%VY}E`9MLA!Q-&Ws$nJI@q*ilNM}{5Oyg4 z-=d8*)M<+$G2M^-I3$8IZ_=9hQD@%1GJz{EXY0#v#aAI%s&i5gw7 zV6Ek#9u?xV_LpCMDNthS4uwpT02+Wi!f85`SQ#MHj41n4XNb5cL0|j(Uv*LDEZ^V6 zr`R5z&bFU_%560S?n;_!?ORK><|l>t?4N3RxSNYf5!w8Z5dzrQ!ekQ*@hnNP2b(8mnZnnb>(4V6F&)VTttO zDOu*$-lN^HMbDS$Vtf_0*pVEp(UVmWwcyK1rMURaeYvJErNGW*2Do?0L z=p=)H$&q9p5wbSiJ!<^yOkA*S8-wTQ>oljMhwHrVUDro1iZ4H7ziWF+G48;-b7h4M z3Le5tJ}G&N9{wspLQ5GIJ>S9HTRluh6(;M+^}>EC2zJ75_qrFdmO|Od9uCMir$n7~ z{`-|V0}C*O19KVHNAicg1_w7+UuqmM@+rU(l6l!!n@YiXAg|pwNGGLG#SCodvyyd^ z?Y?PoJa;?grmengXPVrrfTJ}miQ%l3=1i}os}A{x`@P2;WK+hGaBcKp^G)c%?xpsl ziRP6de*l2QHPzTkDh4{LAE)`sR2wQi4`S!zBM-GcH={LiNV16Y_ZuRt|O`ctoBDW>Qnt}## z%1c$=W`0Uqiz2FKmd%-5R=Z^v`|@d-?kQT@zCT~pQU%%UuWVv1%=gN==&=BOl-M}) zw|L1|un+M#`Oku;qfKN{8Mg8#aZm!e2ScK#0V@Ase-;yvmhc05&bu>SDj{dAu^xe- zQ0aFi@}J$k?$wLUz@HV=sz|cEYAYhS7-V<#FDar^Rf62L`9*4XQuuYz2pMvfM6Im? z2Jy><#&cNT6IAe7!>Mo-fhz6$FCbMzm5UZHKN^8YAeH{Jy3;{*5Dwc%f43_uE~#SY zHS2%xpJYFLG5Yk-&}SovsLR{$=I65)*eQXcvJ(G}tNWK#X2maLys^D|mfpoSt2%j))|H$Mj&I;mWx8Dn~%ViK-)tgo3KI^On&=M3M z`ZnObcrAEe5hj}gisMct1KZJi2+>)b59rUsA|Bj^X7}EA3Z!D-y$gKPF|~=6GTKk} z&8X3}{(s7F6dSv|W>7_)21Nn+qA=``FgCS&chS8oWp%0;KHHj~cR6(TLq2B;K7m7? zp$bl)IWpML{9om$k5F0di>+Z$cYXpLKO8Q){Gf)9$QD7N{`RXYH*b(0POa3Q&a2Og zl_(cIPNo9Pk=J4itgIJxYIQP*O z-s@9=@`ve+v57_Ho#7lQzAx`YJ7dF{{T*5=iA5HC+YOfSImXC+8dO8NaTeD}eN`f4 za^(`Ft_^xg&mO+qboKQw`N3_vgNWg1cl(_isJ|CtG&G1l6lZXe%wGQvWD7SseK?B4 z{QVY8Kz_IPj#A{hZ8$*sp2BbbzY-~rmMQwg&IVpJ6`u+!c?6X3b}F0m1-fhmWxttv z)GSo;2u~CgXno_iU?5EQeyk{dLcFmh@VRQpu!XYRakM%KXBAaAw>sRDU((G2e9P2Z zYoh&Ekl1w7~ z43GvSoE$y)I)tJyqC-2_v1{&Uia~FW;X|W;mv|0-ha6&D`}s~Cqi;mcoC+2}VdA2` zSe9z#7K>KY=e6!B;g&+sZQk9nPg$B6Ptod-D3 zBDr{fWhU!H#79}5{UZ$zpYQC>XX9XwibJ-$#!KVVaM>uS^T#f${b;GYx>?UzZmGGZ zp%FI5FSNq|(T)o-4&6M_X^5LP^)$mYNGWe&6uK%{z2XFQQHwXX<`M2%bXhLL&ck&= z(#{}@dpm^{TU4lU*-@>YR?ekE!}0dx=d(@{Vo*3qhcX>qYdwDr%9_D?YM_ED26*Lh z*BqTXYH=9l=k>+mWJbXf=GByfzFQwR` z3CQi(mZf1i-{a|x2Y)np^8RF%1HtMFNTejJP^J~Sy$815;0?ejPDL5rYu9>J3Jte- zEr-a68cG{B&?b9+UX=mb+He<^0_zmdKjl)Bun|?e{?*toDQ1$#dHQJ z$R(u<0!(2*-pHm9+jr69 zXo)f$&VhE6@0IrlWn(vq9Lir!Gc5iGMno?Kq8LG`)~7X?J)3;Q%f@C(D&aFZ5C3v2 zt8bJ-S^s>03J}C`V3E+8LJ`qYO#dJAU;Oawvf&!-8zi%l`o#llJN)5>u4x`&x#93o zk)0)GlmX<%n9#2Rg+m;!mLD8V)Jfi6=tkL<&6lnj>pgfuIQ9UP&KP~J$edv@`M2k! z)OR8={_@QsLH3(rL*i!}u`kWSJY&Rl>lm!J1+eE>1;{$g8Vd)??EBo_(fd)F;Z?g! z1-G{OS_Xy8@fsp3-mM3=f@J6j4)Rha$QFu7 zIx*y=vB5)a4n#!gJMHy5Q^XP_xg7AEmL-J&x_tO_NNEg(+-gM;WBthu) zQ8|ebPL=m42%oTri0F@mzGicmVq?}tWBNq7!u{Y`lZDUquu>wGo$L&r=Fvy-nXVsk ze7fd09Y?Q9XI-3cjt!YG9#V~F_~Q~2hY5;I-Hqxi!*p5H+pXEpJ3$+QFeMIehLlNb za?#Il@*akCAa}C?0LD#}Tg2UOKq9gqAU} zYEGK@S6$f$WY;B{JZ+4n3t7fz<8u&sDE~<1+PF`wiIY^A9};~SAN}%}hm0Uwjw^*) z?-)TwoL!8V^A>K!2CqPfHIcaPhMT%Ot;e+cJE9cJywql)-PY}pkj0BrjQNUk-^Xmt zZy3(H@Eg1VGFL?*b7tU;3|p%Bd2xX^l5_a~&1-?z6((04h?B3>{?!?B?s_wD_0Y9h z2Iu(@=cr!aTA1jmwE2``e&$Dt&^O;J2SEz21R8uj*O46`^lbhi3!{HK_5>=+q@oVW zcuvt;V)F^_Ukq#>V4<2^cuD2XiEe*2YoY06f@$fg+FfZPW^pu<$P(P$u#))-hs@<(}2jjNq@jVG>aS0j&+Rsah$#J5?2Pk zXrGsiGuJvnX&R~)|JuZX5;}0Zf-ylcfl0O;F;agX%Y+Y}v=7-1_et-0Q+>mZewA-O zA6u|^PToA*h87)39`}+)wku*C39_T&lN(d+gz^o_{|{q-85MQ+egWe!3^4Qz-3%d} z0@5%vNQ$I%Nr@mK-QA^#(nyK|A|N1A(jXlQQc8-_ozI#3zQ4c!oBvwR^NzKA;jHU( z&K3LGd!M?+rYKe>YnFaPM9}*FXZJ3dML92!Z9}t!1 z^c#HnF-~2gDx&EUDuCVRoQL{eXQ5C`x;LbJ!i(jq9&}u!Mq9NA;OOP!-fNaDE6v=z%SQvijHs}V+{iNYTSYC z9j=!_{cn><^K&);4>+Qeer!WnhwX`1Wodx5CKg%i_~O4<$2%{D6|Q8A>+Ai?(+WGY zF{pPsj=Cf#rDxkmO6fUS_1c)%F`K6O14<+gii~8kR+BZ4vJN(1E?uqi1pjti>}V8z zu@iN+0tO}zH1Okb`g43z0pu8jjr=Wijt7v-9L4KBQcNyPQJKTE2)7C?~e^gUadg((S(KLtZ zb4Yvm<^yQ7LqWrx_+4}H(zk{>FaQsk6u`i~qAIRcYxP38l9i4$G$r#>T;Q z@nVMIfbtS~`&!_p>ZUSIPoH# zTKu8zRe6AO*tVtx80J{NqGmkv@%zbA%c}dm-y7~1E2y4d&klEPFH*f3T@On==7@^r zE*$m%Zg2t113H!w`!jhzm+p}#{u#AH(NSJb08v>YjRu29bX1UV=7avOF z^ScM@rP&-us!%;+)NY;3o>)tU#s7)|gd1EQB&yt;l1ZAQf zoBg#4+=pz&w0wDVH0(Ep5&PQWpJs!@^X_u;0AEXD#mp^Qk#+B7NUY)4>{?Fj+G8I+ zi1}4%iws3Nhn1x?<>k$FzZB{{Wqbo*ZMCb(QwynyoVSpvM>aS&QypJ55z!ASZfXcJ zV3cAb@3V)sr(Icv9BMzcH15YWo|qYGbHLSaImls~eG4S5{4 z4r;yJu5Ps%@nMxJEIcXG=*b#9=}eDHSA=??+ZT_+&hffpaG;r2k2B5k2h}4+Y9>M)a1)+@0L6V@$3m?3~QRU!@Rw&s;As?a-8fMjmHSq9o z)!fuSDW(NH-5C5<0Z`*dX0khu^fv5k{@G3uywgkZ&fE1LQcU7{82Bles5-jnNlFA5 ziPg=kB(7ybLWBCnW%+_K?pQG(?xwW+=rX?s`mcv$>9Qf&`B<+nmfjfGIc^e@BF=WT zZ`$1Let*UIqa5$p%{v|$t(+6vo1-&g!`!II06lZ49YW@AshrNty+Y{9 zPpQOOU2*^5o^ck#sE(7F5lz@$;;vsPf>GD290$|nEc%x2*;GKFb5=u4-+QXj|^P-5@e<0+@FXo<#TEuH|C|7cmDW4xcKj()=R60 zzG{MC@$Fdm*IXU{Ex?+7Cs$_Z%y;PA#STMtZ}GHW^;hPoxW*cP;mB6%xHzC%GCM%Z z=>a;}&x#R16i~%0|7-E>c=Rv4hH33Z_3n0efmA8pNksu6Wc%w@fz5$`zZK|DV=h1N z13ATgQOw2R{B|G~Tu{q!2rcS&!T4CTdic3UTzwW0kpJJNAXoGh!~v$D7>ug7Po$Z( zMs&%}DmMfY0tL~l3wqlbpXr@PLO_eSc ze`xv1r6x=~dAA`mtRxk5lt4p5iwS^l5a7bgowIcdyfv4Y;%O&hroRs^1c4NGz#%JS zn8K=cn|S;FGACyJJb{#v7A_D1?+?c0C!C(CmC{36+d?pVYva-}S95)djs0~}C)3E7 zFz0fZL(>M8cpA_61#_OJZ75VN5*)4?x)S@>M+^$S2zuRLOin>|#5p&~rydnD( zxno5sR$j0~Qs!5H5c!kKMM-S?-Im|+{H6KEr|*GBe^W!Qr=BpS5Uqv?tZl_8$dVBj z#zA6>?$VLFz7m*RzR?{DD5^QUwm`sR?10sI=Ea|R;K-7uj0g&`Vt0}^?kK(d;@ z0LReF_Lg7TxPze;KSquMtYztdt@w1YLE!b$b(H2(6wmDdj3n^!ab>gGhvda6=W&Z{kKydD{+m+vzn#Q zY-6D`ZQ_gH-o0 zo(%1nb{u}4Ynk)y(rFQ*_{p-g_yOyWhQg+15uU@(-{q7d`+=Wn%mnz>gh8VlRkSw< zuqQbB5DrGA`u!b-XjnQyn@9hY>rbF_FdAx7EVFOJdKbZnarK@f>Ua4wIOa`QNc;6h z`KZtXK5`vnOj?aJ-4|OYorQ7t<6tXjA=t{AcZo?qTEvSJX~t5(fpvmjUTsv1}^1Jp1!c^YCv-y`wcZ0~niedjL02!Y= z^0w~%MS)s+(^*Rsgn20&OnjNsF+x|{1#4UkVdP#QcsHzyz>8P!bB+!@co~Bxzz5eF z`;D1zt-tTPYw9aCVXwLC`Q$R3De;lsoexZMj9!wQT;?Q=T3sv_ts95B(nmCz5z_gp zp^lf=_@g%;6MRhnW8|YL*{EO{A>6<{k*6~K;K|RtW+uVUX^uU^VzncU3ji4vggAeq z%2e$gqh!JYfHmR{EBC_$@bHWySZ(7@Jhxs{{7#!QFyiuGBjI5E`wm7TLp8;vSsT?_ z2C(($`#)oWbTA`CKEw4xs|Wxaw6s#$_7)`VpTK$G=qo~NwUDp|2_H3&jA*%M{f8SK zAV*54JZ@IFb~ta=y{S(@ea8wy8ZRg2h_~|Q){)vo$tDBhGuKnWfk?;yghAjJLt#rA z-hoa0v}-!O(!cFQ6$>7z$iXt_&iZ$EsBiLC*MRca{c~fi(J%Z5TT3)Qs;ffB^aP#b z9jVVSSy61Z4Kd$O0*%!?F9SN-=n^sBdtTgeY~Hk>e|56{Kk*hdvi;MkMYmfLlVF4}^1qr6d@2_GwB(aXR`U`Q!dwX{~v(R$`% zqZfn5rwpdW$O;1$81xaY$+PgWP#c4>NKR2OU=>MyHddZv!_)s)pC8V|5GhhUSAYrvPS&r^7xs;mt{Ld{|ls}BJrHfZNF%)1R%;yf~q!@OuoTas&x@i7H!#|_$guo1@i!PEu;I@cA>R1y!C1Z`B(N#|+Ne2A4; zekyQ^kY6lfIH}P60?nfC{%5Gap_CN*WXWaoQ`Yr!PoX6DEiKfk%r5-mni%TK6y?Kb z(^*xTiVkMy|CeCW0+G>hI0)u3One4x3G>^QuwtJN=7_0z7T%FQeE*de18{0MVhV$W zRs?7ey7-u91m0(jb-X%H#)=L0ojX!gF%XCPwTDze7=acGs>|~U@_@t&M)zQdM+ZLs zbo@@poonK9`*CQCTrmFdy24KkuodX=zo^T)`u-0Iy>OBg5RaL6PK9k6NEVrOu#hUQ zFgG$c6UHPpTVD9Ml@$~=H$f79mjAbfhm_+9xs!mnkv1-1gzX3O1?@z%Jaqtv6(jl9 zNvPWhadM}ZeTbliF9;g)RzXjIwVyxo;OGnT=FeaW8EbV9tZ%}3Id1b+F;ko85#jFj zx#*6urlld>cGOuhdIn8Jx9qz`lJ6W^4qrISu+WmVOCtOdMg%6YlKr{&tdqjp-y*5rq zvB1ELcIVXRW*}1Pj;zRkYo0~{%AdgX0eEdx)q8d;X6+VokuNaZB|>-Tk0JKLCbT$4 zoJ5NM?%2A5q(wE8F{(6sDSj8x2>b?w7|kDKU$~9?T*L^1_#RyQ&9?`BMUEwXw&(E} zyz7=$tijprKm%~_UfI$CnYVJx-Z2p$YZqi0ejEWC;bG+H>6b2b-dC{bb#@K8u>GVI z`PCOY`^kH5m^bWw$)uox!)<8#q9AT;_xxuZFE$z3a+)#-M1XbTqX@)fr9nUt6142y zR_$}n!B7cm-ym5&3FjPFe9vg&kJop