Potenial improvements to password generator and battleship
diff --git a/.gitignore b/.gitignore
index 9753953afa..412d32f9c1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ build/
 .env
 .env.sh
 venv/
+.venv/
 ENV/
 
 # IDE-specific files
diff --git a/gpt_engineer/API/api.py b/gpt_engineer/API/api.py
index b2a1d41af0..be6b5ba0a0 100644
--- a/gpt_engineer/API/api.py
+++ b/gpt_engineer/API/api.py
@@ -10,8 +10,8 @@
 import pathlib
 from typing import Callable, Optional, Coroutine, Any
 
-from gpt_engineer.db import DB
-from gpt_engineer.main import main
+from gpt_engineer.core.db import DB
+from gpt_engineer.cli.main import main
 from gpt_engineer.API.agent import base_router, Agent
 from gpt_engineer.API.db import NotFoundException, not_found_exception_handler
 
diff --git a/gpt_engineer/cli/file_selector.py b/gpt_engineer/cli/file_selector.py
index b4f7997c09..02860b1820 100644
--- a/gpt_engineer/cli/file_selector.py
+++ b/gpt_engineer/cli/file_selector.py
@@ -50,6 +50,7 @@
 from gpt_engineer.core.db import DB, DBs
 
 IGNORE_FOLDERS = {"site-packages", "node_modules", "venv"}
+REFERENCE_FILE_LIST_NAME = "file_to_reference_list.txt"
 FILE_LIST_NAME = "file_list.txt"
 
 
@@ -321,6 +322,57 @@ def is_in_ignoring_extensions(path: Path) -> bool:
     return is_hidden and is_pycache
 
 
+def scan_for_reference_files(metadata_db: DB, workspace_db: DB) -> List[str]:
+    """
+    Scans the root directory for reference files and updates the file list in project metadata.
+
+    This function scans the root directory of the workspace for files referenced in the code
+    and updates the file list in the project metadata database. It ensures that the list of
+    reference files is up to date.
+
+    Parameters:
+    - metadata_db (DB): The project metadata database where the reference file list is stored.
+    - workspace_db (DB): The workspace database representing the root directory.
+
+    Returns:
+    - List[str]: A list of file paths found in the workspace.
+    """
+
+    root_directory = workspace_db.path
+    existing_files = []
+
+    # Files to ignore
+    ignore_files = ["run.sh", "README.md", "pre-execution-files.txt", "prompt"]
+
+    # Directories to ignore
+    ignore_directories = ["env", "venv", "preprompts"]
+
+    # Walk through the root directory and find files referenced in the code
+    for dirpath, dirnames, filenames in os.walk(root_directory):
+        # Ignore directories starting with "." and specified directories
+        dirnames[:] = [
+            d
+            for d in dirnames
+            if not d.startswith(".")
+            and not d.startswith("__")
+            and d not in ignore_directories
+        ]
+
+        for filename in filenames:
+            file_path = os.path.join(dirpath, filename)
+
+            # Check if the file should be ignored
+            if filename not in ignore_files:
+                if file_path not in existing_files:
+                    existing_files.append(file_path)
+
+    # Update the referenced file list in the project metadata
+    metadata_db[REFERENCE_FILE_LIST_NAME] = "\n".join(
+        str(file_path) for file_path in existing_files
+    )
+    return existing_files
+
+
 def ask_for_files(metadata_db: DB, workspace_db: DB) -> None:
     """
     Ask user to select files to improve.
diff --git a/gpt_engineer/core/chat_to_files.py b/gpt_engineer/core/chat_to_files.py
index 2a166fabac..0ac32aad44 100644
--- a/gpt_engineer/core/chat_to_files.py
+++ b/gpt_engineer/core/chat_to_files.py
@@ -137,7 +137,9 @@ def overwrite_files(chat: str, dbs: DBs) -> None:
             dbs.workspace[file_name] = file_content
 
 
-def get_code_strings(workspace: DB, metadata_db: DB) -> dict[str, str]:
+def get_code_strings(
+    workspace: DB, metadata_db: DB, file_list_name: str = FILE_LIST_NAME
+) -> dict[str, str]:
     """
     Read file_list.txt and return file names and their content.
 
@@ -159,7 +161,7 @@ def get_all_files_in_dir(directory):
         for dir in dirs:
             yield from get_all_files_in_dir(os.path.join(root, dir))
 
-    files_paths = metadata_db[FILE_LIST_NAME].strip().split("\n")
+    files_paths = metadata_db[file_list_name].strip().split("\n")
     files = []
 
     for full_file_path in files_paths:
diff --git a/gpt_engineer/core/steps.py b/gpt_engineer/core/steps.py
index ea39b8ce4d..d5c7dbdb88 100644
--- a/gpt_engineer/core/steps.py
+++ b/gpt_engineer/core/steps.py
@@ -64,7 +64,12 @@
     to_files_and_memory,
 )
 from gpt_engineer.core.db import DBs
-from gpt_engineer.cli.file_selector import FILE_LIST_NAME, ask_for_files
+from gpt_engineer.cli.file_selector import (
+    REFERENCE_FILE_LIST_NAME,
+    FILE_LIST_NAME,
+    ask_for_files,
+    scan_for_reference_files,
+)
 from gpt_engineer.cli.learning import human_review_input
 
 # Type hint for chat messages
@@ -180,7 +185,14 @@ def simple_gen(ai: AI, dbs: DBs) -> List[Message]:
     The function assumes the `ai.start` method and the `to_files` utility are correctly
     set up and functional. Ensure these prerequisites are in place before invoking `simple_gen`.
     """
-    messages = ai.start(setup_sys_prompt(dbs), dbs.input["prompt"], step_name=curr_fn())
+
+    # use an enhanced prompt
+    if "enhanced_prompt" in dbs.memory:
+        input_prompt = dbs.memory["enhanced_prompt"]
+    else:
+        input_prompt = dbs.input["prompt"]
+
+    messages = ai.start(setup_sys_prompt(dbs), input_prompt, step_name=curr_fn())
     to_files_and_memory(messages[-1].content.strip(), dbs)
     return messages
 
@@ -634,6 +646,80 @@ def human_review(ai: AI, dbs: DBs):
     return []
 
 
+def enhance_prompt_add_reference_files(ai: AI, dbs: DBs):
+    """
+    Scans the root directory for existing files referenced in the generated code.
+
+    This function scans the root directory for any files that may already exist and
+    are referenced in the code generated for the input prompt. It then updates the
+    file list in the database to include these files.
+
+    Parameters:
+    - dbs (DBs): An instance containing the database configurations and project metadata.
+      The function will update the file list in the project metadata.
+
+    Returns:
+    - list: Returns an empty list, indicating that there's no subsequent interaction with the LLM.
+    """
+    reference_files = scan_for_reference_files(dbs.project_metadata, dbs.workspace)
+
+    files_info = get_code_strings(
+        dbs.workspace, dbs.project_metadata, REFERENCE_FILE_LIST_NAME
+    )  # this has file names relative to the workspace path
+
+    enhanced_prompt = (
+        dbs.input["prompt"]
+        + "\n Here is a list of all the existing files present in the root directory your code will be added to: \n"
+    )
+
+    # Add files as input
+    for file_name, file_str in files_info.items():
+        enhanced_prompt += format_file_to_input(file_name, file_str)
+
+    dbs.memory["enhanced_prompt"] = enhanced_prompt
+
+    return []
+
+
+def enhance_prompt_add_strict_requirements(ai: AI, dbs: DBs) -> List[Message]:
+    """
+    Enhances the promp by adding a set of strict functional requirements aimed
+    at helping it pass tests written against the outputted code.
+
+    This function takes a user-provided prompt and asks the AI model to generate
+    a set of strict functional requirements for the described scenario or system.
+    The AI's response is appended to the original prompt.
+
+    Parameters:
+    - ai (AI): An instance of the AI model.
+    - dbs (DBs): An instance containing the database configurations and user prompts.
+
+    Returns:
+    - List[Message]: A list of message objects encapsulating the AI's generated output.
+
+    Note:
+    - The function assumes the `ai.start` method is correctly set up and functional.
+      Ensure these prerequisites before invoking `convert_to_strict_requirements`.
+    """
+    system_prompt = "Your being shown a prompt which will be passed to an LLM to make it generate code. \
+        The LLMs response to the prompt is being tested to see how it performs. \
+        Every aspect of the prompt will have a corresponding test applied to the LLMs output. \
+        With this in mind, generate a set of strict functional requirements which can be appended to the prompt to improve the LLMs performance. \
+        If some aspect of the prompt seems vague and colloquial e.g. the program 'should' do this or that - Interpret these vague requirements as strict requirements e.g. the program 'must' do this or that. \
+        Output requirements which ensure no reasonable test written against this prompt would fail."
+
+    user_prompt = dbs.input["prompt"]
+    messages = ai.start(system_prompt, user_prompt, step_name=curr_fn())
+
+    dbs.memory["enhanced_prompt"] = (
+        dbs.input["prompt"]
+        + "\n Here are a set of strict functional requirements to consider when completing this task: \n"
+        + messages[-1].content.strip()
+    )
+
+    return messages
+
+
 class Config(str, Enum):
     """
     Enumeration representing different configuration modes for the code processing system.
@@ -669,6 +755,8 @@ class Config(str, Enum):
 
 STEPS = {
     Config.DEFAULT: [
+        # enhance_prompt_add_strict_requirements,
+        # enhance_prompt_add_reference_files,
         simple_gen,
         gen_entrypoint,
         execute_entrypoint,
@@ -689,6 +777,8 @@ class Config(str, Enum):
         gen_entrypoint,
     ],
     Config.SIMPLE: [
+        # enhance_prompt_add_strict_requirements, This seems to add some minor improvements for the password generator but given the exta call the the LLM adds a lot of time  its not worth it.
+        # enhance_prompt_add_reference_files, This seems to add a fairly major improvement to the battleships test - but it breaks every other test
         simple_gen,
         gen_entrypoint,
         execute_entrypoint,
diff --git a/gpt_engineer/preprompts/file_format b/gpt_engineer/preprompts/file_format
index 45a611855a..e7c5711762 100644
--- a/gpt_engineer/preprompts/file_format
+++ b/gpt_engineer/preprompts/file_format
@@ -18,3 +18,5 @@ print("Hello World")
 ```
 
 Do not comment on what every file does. Please note that the code should be fully functional. No placeholders.
+
+Files provided to you in the prompt may also be provided in this format.
diff --git a/gpt_engineer/preprompts/generate b/gpt_engineer/preprompts/generate
index 6e9725879a..763ee1380d 100644
--- a/gpt_engineer/preprompts/generate
+++ b/gpt_engineer/preprompts/generate
@@ -12,4 +12,4 @@ Ensure to implement all code, if you are unsure, write a plausible implementatio
 Include module dependency or package manager dependency definition file.
 Before you finish, double check that all parts of the architecture is present in the files.
 
-When you are done, write finish with "this concludes a fully working implementation".
+When you are done, finish by writing "this concludes a fully working implementation".
diff --git a/gpt_engineer/preprompts/philosophy b/gpt_engineer/preprompts/philosophy
index 4adf2779b0..a52b542dfd 100644
--- a/gpt_engineer/preprompts/philosophy
+++ b/gpt_engineer/preprompts/philosophy
@@ -1,9 +1,16 @@
+Important: The code in your response will be tested. It must be a very detailed, fully working implementation. Never just provide a basic implementation.
+In long answers, always prioritise sending code snippets over helpful text.
+Assume the root folder is the source folder - do not prefix file paths with src/
 Keep the program as short and simple as possible, keeping all code in one file if appropriate.
 Always use the programming language the user asks for.
 For Python, you always create an appropriate requirements.txt file.
 For NodeJS, you always create an appropriate package.json file.
 Always add a comment briefly describing the purpose of the function definition.
 Add comments explaining very complex bits of logic.
+Make sure to consider any references you have missed in your own source folder, which the code you have generated is dependent on and import those dependencies
+Make sure any interfaces or abstract classes you implement are implemented correctly.
+Make sure any tests provided in the root folder will pass.
+
 
 
 Python toolbelt preferences:
