Fix Multi-GPU not working on exllama_hf
diff --git a/modules/exllama_hf.py b/modules/exllama_hf.py
index 27cac374ad..64de7a5ff4 100644
--- a/modules/exllama_hf.py
+++ b/modules/exllama_hf.py
@@ -38,7 +38,6 @@ def prepare_inputs_for_generation(self, input_ids, **kwargs):
 
     @property
     def device(self) -> torch.device:
-        # TODO: May cause problem on multi-gpu inference?
         return torch.device(0)
 
     def __call__(self, *args, **kwargs):
@@ -50,7 +49,7 @@ def __call__(self, *args, **kwargs):
         if cache is None:
             cache = ExLlamaCache(self.ex_model)
             self.ex_model.forward(torch.tensor([seq[:-1]], dtype=torch.long), cache, preprocess_only=True)
-        logits = self.ex_model.forward(torch.tensor([seq[-1:]], dtype=torch.long), cache).to(self.device)
+        logits = self.ex_model.forward(torch.tensor([seq[-1:]], dtype=torch.long), cache).to(kwargs['input_ids'].device)
         return CausalLMOutputWithPast(logits=logits, past_key_values=cache if use_cache else None)
 
     @classmethod
@@ -72,11 +71,14 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
         assert weight_path is not None, f'could not find weight in "{pretrained_model_name_or_path}"'
 
         config.model_path = str(weight_path)
+
+        if shared.args.gpu_split:
+            config.set_auto_map(shared.args.gpu_split)
+            config.gpu_peer_fix = True
         
         # This slowes down a bit but align better with autogptq generation.
         # TODO: Should give user choice to tune the exllama config
-        config.act_order = True
-        config.fused_attn = False
-        config.fused_mlp_thd = 0
+        # config.fused_attn = False
+        # config.fused_mlp_thd = 0
 
-        return ExllamaHF(config)
\ No newline at end of file
+        return ExllamaHF(config)

After https://github.com/oobabooga/text-generation-webui/pull/2777, new exllama_hf doesn't support gpu_split, since it wasn't added in the code. 

It would try to load all on the GPU0, and then get:

```
Traceback (most recent call last):
  File "F:\ChatIAs\oobabooga\text-generation-webui\server.py", line 62, in load_model_wrapper
    shared.model, shared.tokenizer = load_model(shared.model_name, loader)
  File "F:\ChatIAs\oobabooga\text-generation-webui\modules\models.py", line 66, in load_model
    output = load_func_map[loader](model_name)
  File "F:\ChatIAs\oobabooga\text-generation-webui\modules\models.py", line 285, in ExLlama_HF_loader
    return ExllamaHF.from_pretrained(model_name)
  File "F:\ChatIAs\oobabooga\text-generation-webui\modules\exllama_hf.py", line 82, in from_pretrained
    return ExllamaHF(config)
  File "F:\ChatIAs\oobabooga\text-generation-webui\modules\exllama_hf.py", line 27, in __init__
    self.ex_model = ExLlama(self.ex_config)
  File "F:\ChatIAs\oobabooga\text-generation-webui\repositories\exllama\model.py", line 711, in __init__
    tensor = tensor.to(device, non_blocking = True)
torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 86.00 MiB. GPU 0 has a total capacty of 23.99 GiB of which 0 bytes is free. Of the allocated memory 22.92 GiB is allocated by PyTorch, and 39.80 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
```

This adds the configuration to let it load in more than 1 GPU. Now it loads without issues on 2x4090.

Also, since last PR, fixes the next issue.

```
Traceback (most recent call last):
  File "F:\ChatIAs\oobabooga\text-generation-webui\modules\callbacks.py", line 74, in gentask
    ret = self.mfunc(callback=_callback, *args, **self.kwargs)
  File "F:\ChatIAs\oobabooga\text-generation-webui\modules\text_generation.py", line 262, in generate_with_callback
    shared.model.generate(**kwargs)
  File "F:\ChatIAs\oobabooga\venv\lib\site-packages\torch\utils\_contextlib.py", line 115, in decorate_context
    return func(*args, **kwargs)
  File "F:\ChatIAs\oobabooga\venv\lib\site-packages\transformers\generation\utils.py", line 1572, in generate
    return self.sample(
  File "F:\ChatIAs\oobabooga\venv\lib\site-packages\transformers\generation\utils.py", line 2632, in sample
    next_token_scores = logits_processor(input_ids, next_token_logits)
  File "F:\ChatIAs\oobabooga\venv\lib\site-packages\transformers\generation\logits_process.py", line 92, in __call__
    scores = processor(input_ids, scores)
  File "F:\ChatIAs\oobabooga\venv\lib\site-packages\transformers\generation\logits_process.py", line 203, in __call__
    score = torch.gather(scores, 1, input_ids)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cuda:1! (when checking argument for argument index in method wrapper_CUDA_gather)
```

~~Speeds are reduced a bit (15 tokens/s to 12-13 tokens/s at max context on 65B).~~ Fixed after latest exllama commits.

Closes https://github.com/oobabooga/text-generation-webui/issues/2809