[Serve] fix flaky `test_autoscaling_policy`
diff --git a/python/ray/serve/_private/client.py b/python/ray/serve/_private/client.py
index 7c75f4da74c29..470dc6fd363c3 100644
--- a/python/ray/serve/_private/client.py
+++ b/python/ray/serve/_private/client.py
@@ -94,16 +94,22 @@ def __del__(self):
     def __reduce__(self):
         raise RayServeException(("Ray Serve client cannot be serialized."))
 
+    def shutdown_cached_handles(self):
+        """Shuts down all cached handles.
+
+        Remove the reference to the cached handles so that they can be
+        garbage collected.
+        """
+        for cache_key in list(self.handle_cache):
+            del self.handle_cache[cache_key]
+
     def shutdown(self, timeout_s: float = 30.0) -> None:
         """Completely shut down the connected Serve instance.
 
         Shuts down all processes and deletes all state associated with the
         instance.
         """
-
-        # Shut down handles
-        for k in list(self.handle_cache):
-            del self.handle_cache[k]
+        self.shutdown_cached_handles()
 
         if ray.is_initialized() and not self._shutdown:
             try:
diff --git a/python/ray/serve/tests/conftest.py b/python/ray/serve/tests/conftest.py
index 142f7606bb846..69f53fcacf209 100644
--- a/python/ray/serve/tests/conftest.py
+++ b/python/ray/serve/tests/conftest.py
@@ -83,7 +83,7 @@ def serve_instance(_shared_serve_instance):
     # Clear all state between tests to avoid naming collisions.
     _shared_serve_instance.delete_deployments(serve.list_deployments().keys())
     # Clear the ServeHandle cache between tests to avoid them piling up.
-    _shared_serve_instance.handle_cache.clear()
+    _shared_serve_instance.shutdown_cached_handles()
 
 
 def check_ray_stop():

<!-- Thank you for your contribution! Please review https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before opening a pull request. -->

<!-- Please add a reviewer to the assignee section when you create a PR. If you don't have the access to it, we will shortly find a reviewer and assign them to your PR. -->

## Why are these changes needed?

Calling `.clear()` will free up the memory and can cause issue in raylet. Refactored `shutdown_cached_handles()` method for iterating through all cached handles and remove their reference to safely garbage collect the handles (which is what's done when calling `serve.shutdown()`). 

## Related issue number

Fix https://buildkite.com/ray-project/oss-ci-build-branch/builds/5018#01896734-ab26-4231-aa48-f2d9e11c436b/2955-4683

## Checks

- [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR.
- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
- [ ] I've included any doc changes needed for https://docs.ray.io/en/master/.
    - [ ] I've added any new APIs to the API Reference. For example, if I added a 
           method in Tune, I've added it in `doc/source/tune/api/` under the 
           corresponding `.rst` file.
- [ ] I've made sure the tests are passing. Note that there might be a few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
   - [ ] Unit tests
   - [ ] Release tests
   - [ ] This PR is not tested :(
