From 60008e3905167bfbc5bfa13078229cb2c8799225 Mon Sep 17 00:00:00 2001 From: imoneoi Date: Mon, 18 May 2020 16:08:34 +0800 Subject: [PATCH 1/7] Added DiagGaussian to fix log_probg --- tianshou/policy/modelfree/sac.py | 4 ++-- tianshou/policy/utils.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tianshou/policy/utils.py diff --git a/tianshou/policy/modelfree/sac.py b/tianshou/policy/modelfree/sac.py index d1357a51c..6e1a8f516 100644 --- a/tianshou/policy/modelfree/sac.py +++ b/tianshou/policy/modelfree/sac.py @@ -6,6 +6,7 @@ from tianshou.data import Batch from tianshou.policy import DDPGPolicy +from tianshou.policy.utils import DiagGaussian class SACPolicy(DDPGPolicy): @@ -94,13 +95,12 @@ def forward(self, batch: Batch, obs = getattr(batch, input) logits, h = self.actor(obs, state=state, info=batch.info) assert isinstance(logits, tuple) - dist = torch.distributions.Normal(*logits) + dist = DiagGaussian(*logits) x = dist.rsample() y = torch.tanh(x) act = y * self._action_scale + self._action_bias log_prob = dist.log_prob(x) - torch.log( self._action_scale * (1 - y.pow(2)) + self.__eps) - log_prob = torch.unsqueeze(torch.sum(log_prob, 1), 1) act = act.clamp(self._range[0], self._range[1]) return Batch( logits=logits, act=act, state=h, dist=dist, log_prob=log_prob) diff --git a/tianshou/policy/utils.py b/tianshou/policy/utils.py new file mode 100644 index 000000000..56aa03510 --- /dev/null +++ b/tianshou/policy/utils.py @@ -0,0 +1,13 @@ +import torch + + +class DiagGaussian(torch.distributions.Normal): + """Diagonal Gaussian Distribution + + """ + + def log_prob(self, actions): + return super().log_prob(actions).sum(-1, keepdim=True) + + def entropy(self): + return super().entropy().sum(-1) From b3d89fbfb76578baccc01f629770accaefc0ede3 Mon Sep 17 00:00:00 2001 From: imoneoi Date: Mon, 18 May 2020 16:09:02 +0800 Subject: [PATCH 2/7] Disable PPO dual_clip --- test/continuous/test_ppo.py | 8 +++++--- tianshou/policy/modelfree/ppo.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/continuous/test_ppo.py b/test/continuous/test_ppo.py index 8fd50421d..a1c9d2b38 100644 --- a/test/continuous/test_ppo.py +++ b/test/continuous/test_ppo.py @@ -8,6 +8,7 @@ from tianshou.env import VectorEnv from tianshou.policy import PPOPolicy +from tianshou.policy.utils import DiagGaussian from tianshou.trainer import onpolicy_trainer from tianshou.data import Collector, ReplayBuffer @@ -44,7 +45,7 @@ def get_args(): parser.add_argument('--max-grad-norm', type=float, default=0.5) parser.add_argument('--gae-lambda', type=float, default=0.95) parser.add_argument('--rew-norm', type=bool, default=True) - parser.add_argument('--dual-clip', type=float, default=5.) + # parser.add_argument('--dual-clip', type=float, default=5.) parser.add_argument('--value-clip', type=bool, default=True) args = parser.parse_known_args()[0] return args @@ -85,7 +86,7 @@ def test_ppo(args=get_args()): torch.nn.init.zeros_(m.bias) optim = torch.optim.Adam(list( actor.parameters()) + list(critic.parameters()), lr=args.lr) - dist = torch.distributions.Normal + dist = DiagGaussian policy = PPOPolicy( actor, critic, optim, dist, args.gamma, max_grad_norm=args.max_grad_norm, @@ -93,7 +94,8 @@ def test_ppo(args=get_args()): vf_coef=args.vf_coef, ent_coef=args.ent_coef, reward_normalization=args.rew_norm, - dual_clip=args.dual_clip, + # dual_clip=args.dual_clip, + # dual clip cause monotonically increasing log_std :) value_clip=args.value_clip, # action_range=[env.action_space.low[0], env.action_space.high[0]],) # if clip the action, ppo would not converge :) diff --git a/tianshou/policy/modelfree/ppo.py b/tianshou/policy/modelfree/ppo.py index 85cb7e166..3f9b5e989 100644 --- a/tianshou/policy/modelfree/ppo.py +++ b/tianshou/policy/modelfree/ppo.py @@ -53,7 +53,7 @@ def __init__(self, ent_coef: float = .01, action_range: Optional[Tuple[float, float]] = None, gae_lambda: float = 0.95, - dual_clip: float = 5., + dual_clip: float = None, value_clip: bool = True, reward_normalization: bool = True, **kwargs) -> None: From 9b1e161d5f87f5d5f5a6b867aef03274256a04e3 Mon Sep 17 00:00:00 2001 From: imoneoi Date: Tue, 4 Aug 2020 11:02:40 +0800 Subject: [PATCH 3/7] added BipedalWalkerHardcore-v3 example --- examples/bipedal_hardcore_sac.py | 159 +++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 examples/bipedal_hardcore_sac.py diff --git a/examples/bipedal_hardcore_sac.py b/examples/bipedal_hardcore_sac.py new file mode 100644 index 000000000..afe9e79e9 --- /dev/null +++ b/examples/bipedal_hardcore_sac.py @@ -0,0 +1,159 @@ +import os +import gym +import torch +import pprint +import argparse +import numpy as np +from torch.utils.tensorboard import SummaryWriter + +from tianshou.env import SubprocVectorEnv +from tianshou.trainer import offpolicy_trainer +from tianshou.data import Collector, ReplayBuffer +from tianshou.policy import SACPolicy +from tianshou.utils.net.common import Net +from tianshou.utils.net.continuous import ActorProb, Critic + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--task', type=str, default="BipedalWalkerHardcore-v3") + parser.add_argument('--seed', type=int, default=0) + parser.add_argument('--buffer-size', type=int, default=1000000) + parser.add_argument('--actor-lr', type=float, default=3e-4) + parser.add_argument('--critic-lr', type=float, default=1e-3) + parser.add_argument('--gamma', type=float, default=0.99) + parser.add_argument('--tau', type=float, default=0.005) + parser.add_argument('--alpha', type=float, default=0.1) + parser.add_argument('--epoch', type=int, default=1000) + parser.add_argument('--step-per-epoch', type=int, default=2400) + parser.add_argument('--collect-per-step', type=int, default=10) + parser.add_argument('--batch-size', type=int, default=128) + parser.add_argument('--layer-num', type=int, default=1) + parser.add_argument('--training-num', type=int, default=8) + parser.add_argument('--test-num', type=int, default=8) + parser.add_argument('--logdir', type=str, default='log') + parser.add_argument('--render', type=float, default=0.) + parser.add_argument('--rew-norm', type=int, default=0) + parser.add_argument('--ignore-done', type=int, default=0) + parser.add_argument('--n-step', type=int, default=4) + parser.add_argument( + '--device', type=str, + default='cuda' if torch.cuda.is_available() else 'cpu') + args = parser.parse_known_args()[0] + return args + + +def test_sac_bipedal(args=get_args()): + torch.set_num_threads(1) # we just need only one thread for NN + + # env wrapper for reward scale, action repeat and action noise + class EnvWrapper(object): + def __init__(self, env, + action_repeat=3, + reward_scale=5, + act_noise=0.3): + self._env = env + self.action_repeat = action_repeat + self.reward_scale = reward_scale + self.act_noise = act_noise + + def __getattr__(self, name): + return getattr(self._env, name) + + def step(self, action): + # add action noise + action += self.act_noise * (-2 * np.random.random(4) + 1) + + r = 0.0 + for _ in range(self.action_repeat): + obs_, reward_, done_, info_ = self._env.step(action) + + # remove done reward penalty + if done_: + break + + r = r + reward_ + + # scale reward + return obs_, self.reward_scale * r, done_, info_ + + def MakeEnv(): + return EnvWrapper(gym.make(args.task)) + + def IsStop(reward): + return reward >= 300 * 5 + + env = MakeEnv() + args.state_shape = env.observation_space.shape or env.observation_space.n + args.action_shape = env.action_space.shape or env.action_space.n + args.max_action = env.action_space.high[0] + + train_envs = SubprocVectorEnv( + [lambda: MakeEnv() for _ in range(args.training_num)]) + # test_envs = gym.make(args.task) + test_envs = SubprocVectorEnv( + [lambda: MakeEnv() for _ in range(args.test_num)]) + + # seed + np.random.seed(args.seed) + torch.manual_seed(args.seed) + train_envs.seed(args.seed) + test_envs.seed(args.seed) + + # model + net_a = Net(args.layer_num, args.state_shape, device=args.device) + actor = ActorProb( + net_a, args.action_shape, + args.max_action, args.device + ).to(args.device) + actor_optim = torch.optim.Adam(actor.parameters(), lr=args.actor_lr) + + net_c1 = Net(args.layer_num, args.state_shape, + args.action_shape, concat=True, device=args.device) + critic1 = Critic(net_c1, args.device).to(args.device) + critic1_optim = torch.optim.Adam(critic1.parameters(), lr=args.critic_lr) + + net_c2 = Net(args.layer_num, args.state_shape, + args.action_shape, concat=True, device=args.device) + critic2 = Critic(net_c2, args.device).to(args.device) + critic2_optim = torch.optim.Adam(critic2.parameters(), lr=args.critic_lr) + + policy = SACPolicy( + actor, actor_optim, critic1, critic1_optim, critic2, critic2_optim, + args.tau, args.gamma, args.alpha, + [env.action_space.low[0], env.action_space.high[0]], + reward_normalization=args.rew_norm, + ignore_done=args.ignore_done, + estimation_step=args.n_step) + + # collector + train_collector = Collector( + policy, train_envs, ReplayBuffer(args.buffer_size)) + test_collector = Collector(policy, test_envs) + # train_collector.collect(n_step=args.buffer_size) + # log + log_path = os.path.join(args.logdir, args.task, 'sac') + writer = SummaryWriter(log_path) + + def save_fn(policy): + torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth')) + + # trainer + result = offpolicy_trainer( + policy, train_collector, test_collector, args.epoch, + args.step_per_epoch, args.collect_per_step, args.test_num, + args.batch_size, stop_fn=IsStop, save_fn=save_fn, writer=writer) + + test_collector.close() + if __name__ == '__main__': + pprint.pprint(result) + # Let's watch its performance! + env = MakeEnv() + collector = Collector(policy, env) + result = collector.collect(n_episode=16, render=args.render) + print(f'Final reward: {result["rew"]}, length: {result["len"]}') + collector.close() + + +if __name__ == '__main__': + test_sac_bipedal() From 01fab7a85970b81b8f6eb5d6d8ed5acb6919c2fe Mon Sep 17 00:00:00 2001 From: imoneoi Date: Tue, 4 Aug 2020 11:05:28 +0800 Subject: [PATCH 4/7] remove local utils --- tianshou/policy/utils.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tianshou/policy/utils.py diff --git a/tianshou/policy/utils.py b/tianshou/policy/utils.py deleted file mode 100644 index 56aa03510..000000000 --- a/tianshou/policy/utils.py +++ /dev/null @@ -1,13 +0,0 @@ -import torch - - -class DiagGaussian(torch.distributions.Normal): - """Diagonal Gaussian Distribution - - """ - - def log_prob(self, actions): - return super().log_prob(actions).sum(-1, keepdim=True) - - def entropy(self): - return super().entropy().sum(-1) From 09a3824c2bc0df25e2b0417ac9f8a451efccceaf Mon Sep 17 00:00:00 2001 From: Trinkle23897 <463003665@qq.com> Date: Wed, 5 Aug 2020 10:12:11 +0800 Subject: [PATCH 5/7] minor fix --- examples/bipedal_hardcore_sac.py | 56 ++++++++++++++------------------ 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/examples/bipedal_hardcore_sac.py b/examples/bipedal_hardcore_sac.py index afe9e79e9..988b20bf5 100644 --- a/examples/bipedal_hardcore_sac.py +++ b/examples/bipedal_hardcore_sac.py @@ -42,41 +42,35 @@ def get_args(): args = parser.parse_known_args()[0] return args +# env wrapper for reward scale, action repeat and action noise +class EnvWrapper(object): + def __init__(self, env, action_repeat=3, + reward_scale=5, act_noise=0.3): + self._env = env + self.action_repeat = action_repeat + self.reward_scale = reward_scale + self.act_noise = act_noise + + def __getattr__(self, name): + return getattr(self._env, name) + + def step(self, action): + # add action noise + action += self.act_noise * (-2 * np.random.random(4) + 1) + r = 0.0 + for _ in range(self.action_repeat): + obs_, reward_, done_, info_ = self._env.step(action) + # remove done reward penalty + if done_: + break + r = r + reward_ + # scale reward + return obs_, self.reward_scale * r, done_, info_ + def test_sac_bipedal(args=get_args()): torch.set_num_threads(1) # we just need only one thread for NN - # env wrapper for reward scale, action repeat and action noise - class EnvWrapper(object): - def __init__(self, env, - action_repeat=3, - reward_scale=5, - act_noise=0.3): - self._env = env - self.action_repeat = action_repeat - self.reward_scale = reward_scale - self.act_noise = act_noise - - def __getattr__(self, name): - return getattr(self._env, name) - - def step(self, action): - # add action noise - action += self.act_noise * (-2 * np.random.random(4) + 1) - - r = 0.0 - for _ in range(self.action_repeat): - obs_, reward_, done_, info_ = self._env.step(action) - - # remove done reward penalty - if done_: - break - - r = r + reward_ - - # scale reward - return obs_, self.reward_scale * r, done_, info_ - def MakeEnv(): return EnvWrapper(gym.make(args.task)) From 48ec1ef4bfd8f214605109e3c048ed1da047d9e0 Mon Sep 17 00:00:00 2001 From: Trinkle23897 <463003665@qq.com> Date: Wed, 5 Aug 2020 10:14:27 +0800 Subject: [PATCH 6/7] fix pep8 --- examples/bipedal_hardcore_sac.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/bipedal_hardcore_sac.py b/examples/bipedal_hardcore_sac.py index 988b20bf5..1a1871cc1 100644 --- a/examples/bipedal_hardcore_sac.py +++ b/examples/bipedal_hardcore_sac.py @@ -42,8 +42,9 @@ def get_args(): args = parser.parse_known_args()[0] return args -# env wrapper for reward scale, action repeat and action noise + class EnvWrapper(object): + """Env wrapper for reward scale, action repeat and action noise""" def __init__(self, env, action_repeat=3, reward_scale=5, act_noise=0.3): self._env = env From db384583a0d0b6d1bfcb765525a61ad4603e17a9 Mon Sep 17 00:00:00 2001 From: Trinkle23897 <463003665@qq.com> Date: Wed, 5 Aug 2020 10:18:32 +0800 Subject: [PATCH 7/7] fix envwrapper --- examples/bipedal_hardcore_sac.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/examples/bipedal_hardcore_sac.py b/examples/bipedal_hardcore_sac.py index 1a1871cc1..3d9c435f7 100644 --- a/examples/bipedal_hardcore_sac.py +++ b/examples/bipedal_hardcore_sac.py @@ -45,9 +45,9 @@ def get_args(): class EnvWrapper(object): """Env wrapper for reward scale, action repeat and action noise""" - def __init__(self, env, action_repeat=3, + def __init__(self, task, action_repeat=3, reward_scale=5, act_noise=0.3): - self._env = env + self._env = gym.make(task) self.action_repeat = action_repeat self.reward_scale = reward_scale self.act_noise = act_noise @@ -72,22 +72,19 @@ def step(self, action): def test_sac_bipedal(args=get_args()): torch.set_num_threads(1) # we just need only one thread for NN - def MakeEnv(): - return EnvWrapper(gym.make(args.task)) - def IsStop(reward): return reward >= 300 * 5 - env = MakeEnv() + env = EnvWrapper(args.task) args.state_shape = env.observation_space.shape or env.observation_space.n args.action_shape = env.action_space.shape or env.action_space.n args.max_action = env.action_space.high[0] train_envs = SubprocVectorEnv( - [lambda: MakeEnv() for _ in range(args.training_num)]) + [lambda: EnvWrapper(args.task) for _ in range(args.training_num)]) # test_envs = gym.make(args.task) test_envs = SubprocVectorEnv( - [lambda: MakeEnv() for _ in range(args.test_num)]) + [lambda: EnvWrapper(args.task) for _ in range(args.test_num)]) # seed np.random.seed(args.seed) @@ -143,7 +140,7 @@ def save_fn(policy): if __name__ == '__main__': pprint.pprint(result) # Let's watch its performance! - env = MakeEnv() + env = EnvWrapper(args.task) collector = Collector(policy, env) result = collector.collect(n_episode=16, render=args.render) print(f'Final reward: {result["rew"]}, length: {result["len"]}')