[RLlib] Issue 15973: Trainer.with_updates(validate_config=...) behaves confusingly.
diff --git a/rllib/agents/dqn/apex.py b/rllib/agents/dqn/apex.py
index e9476229e1c9f..65e486048e2df 100644
--- a/rllib/agents/dqn/apex.py
+++ b/rllib/agents/dqn/apex.py
@@ -21,7 +21,6 @@
 from ray.rllib.agents.dqn.dqn import calculate_rr_weights, \
     DEFAULT_CONFIG as DQN_CONFIG, DQNTrainer, validate_config
 from ray.rllib.agents.dqn.learner_thread import LearnerThread
-from ray.rllib.agents.trainer import Trainer
 from ray.rllib.evaluation.worker_set import WorkerSet
 from ray.rllib.execution.common import (STEPS_TRAINED_COUNTER,
                                         _get_global_vars, _get_shared_metrics)
@@ -77,7 +76,6 @@ class OverrideDefaultResourceRequest:
     @override(Trainable)
     def default_resource_request(cls, config):
         cf = dict(cls._default_config, **config)
-        Trainer._validate_config(cf)
 
         eval_config = cf["evaluation_config"]
 
diff --git a/rllib/agents/impala/impala.py b/rllib/agents/impala/impala.py
index f2bc9512fbe94..b12ed5e9b88b1 100644
--- a/rllib/agents/impala/impala.py
+++ b/rllib/agents/impala/impala.py
@@ -2,7 +2,7 @@
 
 import ray
 from ray.rllib.agents.impala.vtrace_tf_policy import VTraceTFPolicy
-from ray.rllib.agents.trainer import Trainer, with_common_config
+from ray.rllib.agents.trainer import with_common_config
 from ray.rllib.agents.trainer_template import build_trainer
 from ray.rllib.execution.learner_thread import LearnerThread
 from ray.rllib.execution.multi_gpu_learner import TFMultiGPULearner
@@ -103,7 +103,6 @@ class OverrideDefaultResourceRequest:
     @override(Trainable)
     def default_resource_request(cls, config):
         cf = dict(cls._default_config, **config)
-        Trainer._validate_config(cf)
 
         eval_config = cf["evaluation_config"]
 
diff --git a/rllib/agents/qmix/qmix.py b/rllib/agents/qmix/qmix.py
index c2584378ded03..dc8802f42da55 100644
--- a/rllib/agents/qmix/qmix.py
+++ b/rllib/agents/qmix/qmix.py
@@ -95,6 +95,8 @@
         "lstm_cell_size": 64,
         "max_seq_len": 999999,
     },
+    # Only torch supported so far.
+    "framework": "torch",
 })
 # __sphinx_doc_end__
 # yapf: enable
diff --git a/rllib/agents/qmix/tests/test_qmix.py b/rllib/agents/qmix/tests/test_qmix.py
index 5c214a5d859e3..c5dad9d983df3 100644
--- a/rllib/agents/qmix/tests/test_qmix.py
+++ b/rllib/agents/qmix/tests/test_qmix.py
@@ -54,6 +54,14 @@ def step(self, action_dict):
 
 
 class TestQMix(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        ray.init()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        ray.shutdown()
+
     def test_avail_actions_qmix(self):
         grouping = {
             "group_1": ["agent_1"],  # trivial grouping for testing
@@ -65,7 +73,6 @@ def test_avail_actions_qmix(self):
             lambda config: AvailActionsTestEnv(config).with_agent_groups(
                 grouping, obs_space=obs_space, act_space=act_space))
 
-        ray.init()
         agent = QMixTrainer(
             env="action_mask_test",
             config={
@@ -75,7 +82,7 @@ def test_avail_actions_qmix(self):
                 },
                 "framework": "torch",
             })
-        for _ in range(5):
+        for _ in range(4):
             agent.train()  # OK if it doesn't trip the action assertion error
         assert agent.train()["episode_reward_mean"] == 21.0
         agent.stop()
diff --git a/rllib/agents/trainer.py b/rllib/agents/trainer.py
index d17b50f4cf67d..2abfdec87f18f 100644
--- a/rllib/agents/trainer.py
+++ b/rllib/agents/trainer.py
@@ -553,7 +553,6 @@ def default_resource_request(
             cls, config: PartialTrainerConfigDict) -> \
             Union[Resources, PlacementGroupFactory]:
         cf = dict(cls._default_config, **config)
-        Trainer._validate_config(cf)
 
         eval_config = cf["evaluation_config"]
 
diff --git a/rllib/agents/trainer_template.py b/rllib/agents/trainer_template.py
index 2d28703a2d6c3..73a3c86e4dac0 100644
--- a/rllib/agents/trainer_template.py
+++ b/rllib/agents/trainer_template.py
@@ -12,8 +12,8 @@
 from ray.rllib.policy import Policy
 from ray.rllib.utils import add_mixins
 from ray.rllib.utils.annotations import override, DeveloperAPI
-from ray.rllib.utils.typing import EnvConfigDict, EnvType, ResultDict, \
-    TrainerConfigDict
+from ray.rllib.utils.typing import EnvConfigDict, EnvType, \
+    PartialTrainerConfigDict, ResultDict, TrainerConfigDict
 
 logger = logging.getLogger(__name__)
 
@@ -124,9 +124,6 @@ def __init__(self, config=None, env=None, logger_creator=None):
 
         def _init(self, config: TrainerConfigDict,
                   env_creator: Callable[[EnvConfigDict], EnvType]):
-            # Validate config via custom validation function.
-            if validate_config:
-                validate_config(config)
 
             # No `get_policy_class` function.
             if get_policy_class is None:
@@ -211,6 +208,16 @@ def fn(env, env_context, task_fn):
 
             return res
 
+        @staticmethod
+        @override(Trainer)
+        def _validate_config(config: PartialTrainerConfigDict,
+                             trainer_obj_or_none: Optional["Trainer"] = None):
+            # Call super (Trainer) validation method first.
+            Trainer._validate_config(config, trainer_obj_or_none)
+            # Then call user defined one, if any.
+            if validate_config is not None:
+                validate_config(config)
+
         @override(Trainer)
         def _before_evaluate(self):
             if before_evaluate_fn:
diff --git a/rllib/examples/two_step_game.py b/rllib/examples/two_step_game.py
index b014d5e0351b3..737e7389437fb 100644
--- a/rllib/examples/two_step_game.py
+++ b/rllib/examples/two_step_game.py
@@ -55,6 +55,8 @@
 if __name__ == "__main__":
     args = parser.parse_args()
 
+    ray.init(num_cpus=args.num_cpus or None)
+
     grouping = {
         "group_1": [0, 1],
     }
@@ -123,7 +125,6 @@
             },
             # Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0.
             "num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")),
-            "framework": args.framework,
         }
         group = True
     else:
@@ -134,8 +135,6 @@
         }
         group = False
 
-    ray.init(num_cpus=args.num_cpus or None)
-
     stop = {
         "episode_reward_mean": args.stop_reward,
         "timesteps_total": args.stop_timesteps,
@@ -146,7 +145,7 @@
         "env": "grouped_twostep" if group else TwoStepGame,
     })
 
-    results = tune.run(args.run, stop=stop, config=config, verbose=1)
+    results = tune.run(args.run, stop=stop, config=config, verbose=2)
 
     if args.as_test:
         check_learning_achieved(results, args.stop_reward)

<!-- Thank you for your contribution! Please review https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before opening a pull request. -->

Issue 15973: Trainer.with_updates(validate config=...) behaves confusingly.

- Trainer._validate_config is called multiple times during initialization: 1) Trainer.setup(), 2) Trainer.default_resource_request (overridden by IMPALA and APEX), 3) trainer_template._init() (<- yet another time during setup, which is unnecessary).
- This PR removes all unnecessary calls to this method and makes sure that - when overridden by the user via `Trainer.with_updates(validate_config=...)`, only the user-defined method is used from here on.

<!-- Please add a reviewer to the assignee section when you create a PR. If you don't have the access to it, we will shortly find a reviewer and assign them to your PR. -->

Issue #15973 

## Why are these changes needed?


<!-- Please give a short summary of the change and the problem this solves. -->

Closes #15973 
## Related issue number

<!-- For example: "Closes #1234" -->

## Checks

- [x] I've run `scripts/format.sh` to lint the changes in this PR.
- [x] I've included any doc changes needed for https://docs.ray.io/en/master/.
- [x] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
   - [x] Unit tests
   - [ ] Release tests
   - [ ] This PR is not tested :(
