From 7558497395fe093dd333d4dac7be57afb60be640 Mon Sep 17 00:00:00 2001 From: jvasso <49073175+jvasso@users.noreply.github.com> Date: Sat, 6 Jul 2024 16:20:46 +0200 Subject: [PATCH 1/2] Added support for minibatch in PPO process_fn --- tianshou/policy/modelfree/ppo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tianshou/policy/modelfree/ppo.py b/tianshou/policy/modelfree/ppo.py index 196cd72e4..87ab5014c 100644 --- a/tianshou/policy/modelfree/ppo.py +++ b/tianshou/policy/modelfree/ppo.py @@ -137,8 +137,11 @@ def process_fn( self._buffer, self._indices = buffer, indices batch = self._compute_returns(batch, buffer, indices) batch.act = to_torch_as(batch.act, batch.v_s) + logp_old = [] with torch.no_grad(): - batch.logp_old = self(batch).dist.log_prob(batch.act) + for minibatch in batch.split(self._batch, shuffle=False, merge_last=True): + logp_old.append(self(minibatch).dist.log_prob(minibatch.act)) + batch.logp_old = torch.cat(logp_old, dim=0).flatten() batch: LogpOldProtocol return batch From 918c122268fdbacc3df09adee08db6162373215a Mon Sep 17 00:00:00 2001 From: Michael Panchenko <35432522+MischaPanch@users.noreply.github.com> Date: Sat, 20 Jul 2024 09:34:59 +0200 Subject: [PATCH 2/2] Typo --- tianshou/policy/modelfree/ppo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tianshou/policy/modelfree/ppo.py b/tianshou/policy/modelfree/ppo.py index 87ab5014c..1933c7d54 100644 --- a/tianshou/policy/modelfree/ppo.py +++ b/tianshou/policy/modelfree/ppo.py @@ -139,7 +139,7 @@ def process_fn( batch.act = to_torch_as(batch.act, batch.v_s) logp_old = [] with torch.no_grad(): - for minibatch in batch.split(self._batch, shuffle=False, merge_last=True): + for minibatch in batch.split(self.max_batchsize, shuffle=False, merge_last=True): logp_old.append(self(minibatch).dist.log_prob(minibatch.act)) batch.logp_old = torch.cat(logp_old, dim=0).flatten() batch: LogpOldProtocol