feat: add coder check and give more time (#1127)

* feat: introduce max_seconds_multiplier for timeout management across components

* avoid using assert in test

* hot fix a very small bug

* runner multiply twice

* revert feedback change

* add a switch to longer timeout

Author: peteryang1

rdagent/app/data_science/conf.py

@@ -118,6 +118,9 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
     enable_planner: bool = False
 
     model_architecture_suggestion_time_percent: float = 0.75
+    allow_longer_timeout: bool = False
+    coder_longer_timeout_multiplier: int = 3
+    runner_longer_timeout_multiplier: int = 2
 
     #### hypothesis critique and rewrite
     enable_hypo_critique_rewrite: bool = True

rdagent/components/coder/CoSTEER/__init__.py

@@ -28,6 +28,7 @@ def __init__(
         es: EvolvingStrategy,
         evolving_version: int,
         *args,
+        max_seconds: int | None = None,
         with_knowledge: bool = True,
         with_feedback: bool = True,
         knowledge_self_gen: bool = True,
@@ -37,7 +38,7 @@ def __init__(
     ) -> None:
         super().__init__(*args, **kwargs)
         self.max_loop = settings.max_loop if max_loop is None else max_loop
-        self.max_seconds = settings.max_seconds
+        self.max_seconds = max_seconds
         self.knowledge_base_path = (
             Path(settings.knowledge_base_path) if settings.knowledge_base_path is not None else None
         )
@@ -105,7 +106,7 @@ def develop(self, exp: Experiment) -> Experiment:
             logger.log_object(evo_exp.sub_workspace_list, tag="evolving code")
             for sw in evo_exp.sub_workspace_list:
                 logger.info(f"evolving workspace: {sw}")
-            if (datetime.now() - start_datetime).seconds > self.max_seconds:
+            if self.max_seconds is not None and (datetime.now() - start_datetime).seconds > self.max_seconds:
                 logger.info(f"Reached max time limit {self.max_seconds} seconds, stop evolving")
                 break
             if RD_Agent_TIMER_wrapper.timer.started and RD_Agent_TIMER_wrapper.timer.is_timeout():

��rdagent/components/coder/CoSTEER/config.py

@@ -33,7 +33,7 @@ class Config:
     new_knowledge_base_path: Union[str, None] = None
     """Path to the new knowledge base"""
 
-    max_seconds: int = 10**6
+    max_seconds_multiplier: int = 10**6
 
 
 CoSTEER_SETTINGS = CoSTEERSettings()

rdagent/components/coder/data_science/conf.py

@@ -19,7 +19,7 @@ class DSCoderCoSTEERSettings(CoSTEERSettings):
     class Config:
         env_prefix = "DS_Coder_CoSTEER_"
 
-    max_seconds: int = DS_RD_SETTING.debug_timeout * 4
+    max_seconds_multiplier: int = 4
     env_type: str = "docker"
     # TODO: extract a function for env and conf.
 
@@ -28,7 +28,7 @@ class Config:
     Extra evaluators
 
     The evaluator follows the following assumptions:
-    - It runs after previous evaluator (So the running results are alreadly there)
+    - It runs after previous evaluator (So the running results are already there)
 
     It is not a complete feature due to it is only implemented in DS Pipeline & Coder.
 

rdagent/components/coder/data_science/ensemble/__init__.py

@@ -160,5 +160,6 @@ def __init__(
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/ensemble/eval.py

@@ -47,7 +47,10 @@ def evaluate(
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         fname = "test/ensemble_test.txt"
         test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()

rdagent/components/coder/data_science/feature/__init__.py

@@ -138,5 +138,6 @@ def __init__(
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/feature/eval.py

@@ -43,7 +43,10 @@ def evaluate(
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         # TODO: do we need to clean the generated temporary content?
         fname = "test/feature_test.py"

rdagent/components/coder/data_science/model/__init__.py

@@ -170,5 +170,6 @@ def __init__(
             evolving_version=2,
             scen=scen,
             max_loop=DS_RD_SETTING.coder_max_loop,
+            max_seconds=scen.real_debug_timeout() * settings.max_seconds_multiplier,
             **kwargs,
         )

rdagent/components/coder/data_science/model/eval.py

@@ -57,7 +57,10 @@ def evaluate(
                 final_decision=False,
             )
 
-        env = get_ds_env(extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()})
+        env = get_ds_env(
+            extra_volumes={self.scen.debug_path: T("scenarios.data_science.share:scen.input_path").r()},
+            running_timeout_period=self.scen.real_debug_timeout(),
+        )
 
         if_model_removed = False