From 188a02082857ce299e565e36ea82129e71c3db22 Mon Sep 17 00:00:00 2001 From: Prabhat Nagarajan Date: Fri, 28 Nov 2025 14:31:19 -0700 Subject: [PATCH 1/3] Adds errata for rainbow --- examples/atari/reproduction/rainbow/README.md | 6 +++++- examples/atari/reproduction/rainbow/train_rainbow.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/atari/reproduction/rainbow/README.md b/examples/atari/reproduction/rainbow/README.md index d23d04022..2453fbd98 100644 --- a/examples/atari/reproduction/rainbow/README.md +++ b/examples/atari/reproduction/rainbow/README.md @@ -33,7 +33,11 @@ To view the full list of options, either view the code or run the example with t ## Results -These results reflect PFRL commit hash: `a0ad6a7`. + +These results reflect PFRL commit hash: `a0ad6a7`. +**Errata: In the original Rainbow paper, the target network was updated every 8000 timesteps, not 32000. The file remains the same as it is the file generating the results.** +- Source: [Original Rainbow paper - Table 1](https://arxiv.org/pdf/1710.02298) +- Code: [DQN Zoo Reproduction](https://github.com/google-deepmind/dqn_zoo/blob/master/dqn_zoo/rainbow/run_atari.py#L74) | Results Summary || | ------------- |:-------------:| diff --git a/examples/atari/reproduction/rainbow/train_rainbow.py b/examples/atari/reproduction/rainbow/train_rainbow.py index 065ed8b24..24cbe1299 100644 --- a/examples/atari/reproduction/rainbow/train_rainbow.py +++ b/examples/atari/reproduction/rainbow/train_rainbow.py @@ -152,7 +152,7 @@ def phi(x): explorer=explorer, minibatch_size=32, replay_start_size=args.replay_start_size, - target_update_interval=32000, + target_update_interval=32000, # errata: 8000 in paper is in units of steps, here in units of updates update_interval=update_interval, batch_accumulator="mean", phi=phi, From 65b80e999ded64643f8508672cd6506d15208df7 Mon Sep 17 00:00:00 2001 From: Prabhat Nagarajan Date: Fri, 28 Nov 2025 14:35:00 -0700 Subject: [PATCH 2/3] improves clarity --- examples/atari/reproduction/rainbow/train_rainbow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/atari/reproduction/rainbow/train_rainbow.py b/examples/atari/reproduction/rainbow/train_rainbow.py index 24cbe1299..30d9f7f18 100644 --- a/examples/atari/reproduction/rainbow/train_rainbow.py +++ b/examples/atari/reproduction/rainbow/train_rainbow.py @@ -152,7 +152,7 @@ def phi(x): explorer=explorer, minibatch_size=32, replay_start_size=args.replay_start_size, - target_update_interval=32000, # errata: 8000 in paper is in units of steps, here in units of updates + target_update_interval=32000, # errata: Should be 8000 steps to match the original paper. update_interval=update_interval, batch_accumulator="mean", phi=phi, From f1b8ea3be5babb392f31f43e26338d0c31050662 Mon Sep 17 00:00:00 2001 From: Prabhat Nagarajan Date: Fri, 28 Nov 2025 14:46:26 -0700 Subject: [PATCH 3/3] makes things lint compatible --- examples/atari/reproduction/rainbow/train_rainbow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/atari/reproduction/rainbow/train_rainbow.py b/examples/atari/reproduction/rainbow/train_rainbow.py index 30d9f7f18..031bb32ce 100644 --- a/examples/atari/reproduction/rainbow/train_rainbow.py +++ b/examples/atari/reproduction/rainbow/train_rainbow.py @@ -152,7 +152,8 @@ def phi(x): explorer=explorer, minibatch_size=32, replay_start_size=args.replay_start_size, - target_update_interval=32000, # errata: Should be 8000 steps to match the original paper. + # errata: target_update_interval should be 8000 to match original paper. + target_update_interval=32000, update_interval=update_interval, batch_accumulator="mean", phi=phi,