[serve] autoscaling release test deflake and debugging
diff --git a/release/serve_tests/workloads/locust_utils.py b/release/serve_tests/workloads/locust_utils.py
index 01fe525c11604..e0fb7cdf9f8a1 100644
--- a/release/serve_tests/workloads/locust_utils.py
+++ b/release/serve_tests/workloads/locust_utils.py
@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from itertools import chain
 import json
 import logging
@@ -7,6 +7,7 @@
 from typing import Any, Dict, List
 
 import ray
+from ray.serve._private.utils import generate_request_id
 
 
 logger = logging.getLogger(__file__)
@@ -72,14 +73,16 @@ class EndpointUser(FastHttpUser):
 
             @task
             def test(self):
-                headers = {"Authorization": f"Bearer {token}"} if token else None
+                request_id = generate_request_id()
+                headers = (
+                    {"Authorization": f"Bearer {token}", "X-Request-ID": request_id}
+                    if token
+                    else None
+                )
                 with self.client.get(
                     "", headers=headers, json=data, catch_response=True
                 ) as r:
-                    if r.status_code == 200:
-                        r.request_meta["context"]["request_id"] = r.headers[
-                            "x-request-id"
-                        ]
+                    r.request_meta["context"]["request_id"] = request_id
 
             @events.request.add_listener
             def on_request(
@@ -92,6 +95,7 @@ def on_request(
             ):
                 if exception:
                     request_id = context["request_id"]
+                    response.encoding = "utf-8"
                     err = FailedRequest(
                         request_id=request_id,
                         status_code=response.status_code,
@@ -267,12 +271,13 @@ def run_locust_load_test(config: LocustLoadTestConfig) -> LocustTestResults:
 
     # Collect results and metrics
     stats: LocustTestResults = ray.get(master_ref)
-    errors = sorted(chain(*ray.get(worker_refs)), key=lambda e: e.start_time)
+    errors = sorted(chain(*ray.get(worker_refs)), key=lambda e: e.start_time_s)
 
     # If there were any requests that failed, raise error.
     if stats.num_failures > 0:
+        errors_json = [asdict(err) for err in errors]
         raise RuntimeError(
-            f"There were failed requests: {json.dumps(errors, indent=4)}"
+            f"There were failed requests: {json.dumps(errors_json, indent=4)}"
         )
 
     return stats
diff --git a/release/serve_tests/workloads/resnet_50.py b/release/serve_tests/workloads/resnet_50.py
index 9e8fe5f190231..080c5c8b30131 100644
--- a/release/serve_tests/workloads/resnet_50.py
+++ b/release/serve_tests/workloads/resnet_50.py
@@ -33,8 +33,16 @@ def __init__(self):
 
     async def __call__(self, request: starlette.requests.Request) -> str:
         uri = (await request.json())["uri"]
+
         try:
             image_bytes = requests.get(uri).content
+        except (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.ChunkedEncodingError,
+        ):
+            return
+
+        try:
             image = Image.open(BytesIO(image_bytes)).convert("RGB")
         except PIL.UnidentifiedImageError:
             return

[serve] autoscaling release test deflake and debugging

Deflake release test. From the logs sometimes the http request to get the image fails with connection reset error. For those few occasions we can just return.
Also seeing some 504 errors but need more visibility, so this also adds better tracking for failed requests.


Signed-off-by: Cindy Zhang <cindyzyx9@gmail.com>
