Remove duplicated inference unit tests
diff --git a/tests/unit/inference/test_inference.py b/tests/unit/inference/test_inference.py
index c9073e66b67b..775a20cc9fba 100644
--- a/tests/unit/inference/test_inference.py
+++ b/tests/unit/inference/test_inference.py
@@ -90,37 +90,6 @@ def verify_models():
         pytest.fail(f"Model(s) do not have an assigned task: {_missing_task_models}")
 
 
-# Fixture to add skips for certain configurations
-@pytest.fixture()
-def invalid_test(model_w_task, dtype, enable_cuda_graph, enable_triton):
-    model, task = model_w_task
-    msg = ""
-    if enable_cuda_graph and (torch_info["cuda_version"] == "0.0"):
-        msg = "CUDA not detected, cannot use CUDA Graph"
-    elif enable_cuda_graph and pkg_version.parse(torch.__version__) < pkg_version.parse("1.10"):
-        msg = "CUDA Graph is only available in torch versions >= 1.10"
-    elif "gpt-j-6b" in model:
-        if dtype != torch.half:
-            msg = f"Not enough GPU memory to run {model} with dtype {dtype}"
-        elif enable_cuda_graph:
-            msg = f"Not enough GPU memory to run {model} with CUDA Graph enabled"
-    elif "gpt-neox-20b" in model:  # TODO: remove this when neox issues resolved
-        msg = "Skipping gpt-neox-20b for now"
-    elif ("gpt-neox-20b" in model) and (dtype != torch.half):
-        msg = f"Not enough GPU memory to run {model} with dtype {dtype}"
-    elif ("bloom" in model) and (dtype != torch.half):
-        msg = f"Bloom models only support half precision, cannot use dtype {dtype}"
-    elif ("bert" not in model.lower()) and enable_cuda_graph:
-        msg = "Non bert/roberta models do no support CUDA Graph"
-    elif enable_triton and not (dtype in [torch.half]):
-        msg = "Triton is for fp16"
-    elif enable_triton and not deepspeed.HAS_TRITON:
-        msg = "triton needs to be installed for the test"
-    elif ("bert" not in model.lower()) and enable_triton:
-        msg = "Triton kernels do not support Non bert/roberta models yet"
-    return msg
-
-
 """ Fixtures for inference config """
 
 
@@ -257,6 +226,36 @@ def verify_injection(module):
     verify_injection(model)
 
 
+# Verify that test is valid
+def validate_test(model_w_task, dtype, enable_cuda_graph, enable_triton):
+    model, task = model_w_task
+    msg = ""
+    if enable_cuda_graph and (torch_info["cuda_version"] == "0.0"):
+        msg = "CUDA not detected, cannot use CUDA Graph"
+    elif enable_cuda_graph and pkg_version.parse(torch.__version__) < pkg_version.parse("1.10"):
+        msg = "CUDA Graph is only available in torch versions >= 1.10"
+    elif "gpt-j-6b" in model:
+        if dtype != torch.half:
+            msg = f"Not enough GPU memory to run {model} with dtype {dtype}"
+        elif enable_cuda_graph:
+            msg = f"Not enough GPU memory to run {model} with CUDA Graph enabled"
+    elif "gpt-neox-20b" in model:  # TODO: remove this when neox issues resolved
+        msg = "Skipping gpt-neox-20b for now"
+    elif ("gpt-neox-20b" in model) and (dtype != torch.half):
+        msg = f"Not enough GPU memory to run {model} with dtype {dtype}"
+    elif ("bloom" in model) and (dtype != torch.half):
+        msg = f"Bloom models only support half precision, cannot use dtype {dtype}"
+    elif ("bert" not in model.lower()) and enable_cuda_graph:
+        msg = "Non bert/roberta models do no support CUDA Graph"
+    elif enable_triton and not (dtype in [torch.half]):
+        msg = "Triton is for fp16"
+    elif enable_triton and not deepspeed.HAS_TRITON:
+        msg = "triton needs to be installed for the test"
+    elif ("bert" not in model.lower()) and enable_triton:
+        msg = "Triton kernels do not support Non bert/roberta models yet"
+    return msg
+
+
 @pytest.mark.inference
 class TestModelTask(DistributedTest):
     world_size = 1
@@ -270,11 +269,11 @@ def test(
         query,
         inf_kwargs,
         assert_fn,
-        invalid_test,
         perf_meas=True,
     ):
-        if invalid_test:
-            pytest.skip(invalid_test)
+        invalid_test_msg = validate_test(model_w_task, dtype, enable_cuda_graph, enable_triton)
+        if invalid_test_msg:
+            pytest.skip(invalid_test_msg)
 
         model, task = model_w_task
         local_rank = int(os.getenv("LOCAL_RANK", "0"))
@@ -353,10 +352,10 @@ def test(
         query,
         inf_kwargs,
         assert_fn,
-        invalid_test,
     ):
-        if invalid_test:
-            pytest.skip(invalid_test)
+        invalid_test_msg = validate_test(model_w_task, dtype, enable_cuda_graph=False, enable_triton=False)
+        if invalid_test_msg:
+            pytest.skip(invalid_test_msg)
 
         model, task = model_w_task
         local_rank = int(os.getenv("LOCAL_RANK", "0"))
@@ -434,7 +433,6 @@ def test(
     ids=["t5", "roberta"],
 )
 @pytest.mark.parametrize("dtype", [torch.float], ids=["fp32"])
-@pytest.mark.parametrize("enable_cuda_graph", [False], ids=["noCG"])
 class TestInjectionPolicy(DistributedTest):
     world_size = [1, 2]
 
@@ -445,12 +443,11 @@ def test(
         query,
         inf_kwargs,
         assert_fn,
-        invalid_test,
         dtype,
-        enable_cuda_graph,
     ):
-        if invalid_test:
-            pytest.skip(invalid_test)
+        invalid_test_msg = validate_test(model_w_task, dtype, enable_cuda_graph=False, enable_triton=False)
+        if invalid_test_msg:
+            pytest.skip(invalid_test_msg)
 
         model, task = model_w_task
         local_rank = int(os.getenv("LOCAL_RANK", "0"))
@@ -485,7 +482,6 @@ def test(
     ],
 )
 @pytest.mark.parametrize("dtype", [torch.float16], ids=["fp16"])
-@pytest.mark.parametrize("enable_cuda_graph", [False], ids=["noCG"])
 class TestAutoTensorParallelism(DistributedTest):
     world_size = [2]
 
@@ -495,12 +491,11 @@ def test(
         query,
         inf_kwargs,
         assert_fn,
-        invalid_test,
         dtype,
-        enable_cuda_graph,
     ):
-        if invalid_test:
-            pytest.skip(invalid_test)
+        invalid_test_msg = validate_test(model_w_task, dtype, enable_cuda_graph=False, enable_triton=False)
+        if invalid_test_msg:
+            pytest.skip(invalid_test_msg)
 
         model, task = model_w_task
         local_rank = int(os.getenv("LOCAL_RANK", "0"))

In our inference unit tests, we were seeing duplicated tests due to the use of `invalid_test` fixture. Because this fixture used `model_w_task, dtype, enable_cuda_graph, enable_triton` fixtures, any test that used this fixture but did not use all of those fixtures would have duplicated tests generated. A short example to show the problem:

```python
@pytest.fixture
def invalid_test(model_w_task, dtype):
    pass

@pytest.fixture(params=[("gpt2", "text-generation")], ids=["gpt2"])
def model_w_task(request):
    return request.param

@pytest.fixture(params=["fp16", "fp32"])
def dtype(request):
    return request.param

class TestExample(DistributedTest):
    def test(self, model_w_task):
        pass
```

The generated tests would be:
```
test[gpt2-fp16]
test[gpt2-fp32]
```

So despite the test not using the `dtype` fixture directly, the tests were still generated.

Resolved that issue by making the invalid test check a function called inside the tests.