AutoResetWrapper integration with gym.make()
diff --git a/gym/envs/registration.py b/gym/envs/registration.py
index f53ac621205..70f0b9345e5 100644
--- a/gym/envs/registration.py
+++ b/gym/envs/registration.py
@@ -90,6 +90,7 @@ class EnvSpec:
         nondeterministic: Whether this environment is non-deterministic even after seeding
         max_episode_steps: The maximum number of steps that an episode can consist of
         order_enforce: Whether to wrap the environment in an orderEnforcing wrapper
+        autoreset: Whether the environment should automatically reset when it reaches the done state
         kwargs: The kwargs to pass to the environment class
 
     """
@@ -100,6 +101,7 @@ class EnvSpec:
     nondeterministic: bool = field(default=False)
     max_episode_steps: Optional[int] = field(default=None)
     order_enforce: bool = field(default=True)
+    autoreset: bool = field(default=False)
     kwargs: dict = field(default_factory=dict)
     namespace: Optional[str] = field(init=False)
     name: str = field(init=False)
@@ -132,6 +134,10 @@ def make(self, **kwargs) -> Env:
         _kwargs = self.kwargs.copy()
         _kwargs.update(kwargs)
 
+        if "autoreset" in _kwargs:
+            self.autoreset = _kwargs["autoreset"]
+            del _kwargs["autoreset"]
+
         if callable(self.entry_point):
             env = self.entry_point(**_kwargs)
         else:
@@ -142,15 +148,23 @@ def make(self, **kwargs) -> Env:
         spec = copy.deepcopy(self)
         spec.kwargs = _kwargs
         env.unwrapped.spec = spec
+
         if self.order_enforce:
             from gym.wrappers.order_enforcing import OrderEnforcing
 
             env = OrderEnforcing(env)
+
         assert env.spec is not None, "expected spec to be set to the unwrapped env."
         if env.spec.max_episode_steps is not None:
             from gym.wrappers.time_limit import TimeLimit
 
             env = TimeLimit(env, max_episode_steps=env.spec.max_episode_steps)
+
+        if self.autoreset:
+            from gym.wrappers.autoreset import AutoResetWrapper
+
+            env = AutoResetWrapper(env)
+
         return env
 
 
diff --git a/tests/wrappers/test_autoreset.py b/tests/wrappers/test_autoreset.py
index f003a7280df..827f08f4b9a 100644
--- a/tests/wrappers/test_autoreset.py
+++ b/tests/wrappers/test_autoreset.py
@@ -1,10 +1,13 @@
+import types
 from typing import Optional
+from unittest.mock import MagicMock
 
 import numpy as np
 import pytest
 
 import gym
 from gym.wrappers import AutoResetWrapper
+from tests.envs.spec_list import spec_list
 
 
 class DummyResetEnv(gym.Env):
@@ -62,6 +65,49 @@ def test_autoreset_reset_info():
     assert isinstance(info, dict)
 
 
+@pytest.mark.parametrize("spec", spec_list)
+def test_make_autoreset_true(spec):
+    """
+    Note: This test assumes that the outermost wrapper is AutoResetWrapper
+    so if that is being changed in the future, this test will break and need
+    to be updated.
+    Note: This test assumes that all first-party environments will terminate in a finite
+    amount of time with random actions, which is true as of the time of adding this test.
+    """
+    env = None
+    with pytest.warns(None) as warnings:
+        env = spec.make(autoreset=True)
+
+    ob_space = env.observation_space
+    obs = env.reset(seed=0)
+    env.action_space.seed(0)
+
+    env.unwrapped.reset = MagicMock(side_effect=env.unwrapped.reset)
+
+    done = False
+    while not done:
+        obs, reward, done, info = env.step(env.action_space.sample())
+
+    assert isinstance(env, AutoResetWrapper)
+    assert env.unwrapped.reset.called
+
+
+@pytest.mark.parametrize("spec", spec_list)
+def test_make_autoreset_false(spec):
+    env = None
+    with pytest.warns(None) as warnings:
+        env = spec.make(autoreset=False)
+    assert not isinstance(env, AutoResetWrapper)
+
+
+@pytest.mark.parametrize("spec", spec_list)
+def test_make_autoreset_default_false(spec):
+    env = None
+    with pytest.warns(None) as warnings:
+        env = spec.make()
+    assert not isinstance(env, AutoResetWrapper)
+
+
 def test_autoreset_autoreset():
     env = DummyResetEnv()
     env = AutoResetWrapper(env)

# Description

This is a pull request to integrate the autoreset wrapper into gym.make() with it being disabled by default. The keyword argument is autoreset. If autoreset is set to False or a value is not provided for autoreset in gym.make, then the env will not be wrapped with the autoreset wrapper. 

# Notes for Code Reviewers

Something to note that might be relevant for the code review is that currently the only test is for cartpole  because it is guaranteed to terminate in a relatively short number of time-steps. I'm open to suggestions for how to improve test coverage.

Also it might be worth checking if you agree with the way I drop the autoreset key word from _kwargs in registration.py before passing _kwargs to the entrypoint function. I'm not sure if there was a better way, but it seemed correct because we are not adding auto-reset as a keyword argument to the actual environment constructors(so far at least).

Fixes https://github.com/openai/gym/issues/2564

## Type of change

Please delete options that are not relevant.

- [x] This change requires a documentation update

# Checklist:

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `pre-commit run --all-files` (see `CONTRIBUTING.md` instructions to set it up)
- [ ] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my feature works
- [x] New and existing unit tests pass locally with my changes

